From: Shravan Rangarajuvenkata (shrarang) <shrarang@cisco.com>
Date: Sat, 11 Apr 2020 23:59:13 +0000 (+0000)
Subject: Merge pull request #2144 in SNORT/snort3 from ~SHRARANG/snort3:appid_remove_unused_co... 
X-Git-Tag: 3.0.1-2~20
X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=0ff0f628773c464c26a230dd543a729bd18457cb;p=thirdparty%2Fsnort3.git

Merge pull request #2144 in SNORT/snort3 from ~SHRARANG/snort3:appid_remove_unused_code to master

Squashed commit of the following:

commit 08f71fa3082ed8e77f15403f1b0d2553f85c08de
Author: Shravan Rangaraju <shrarang@cisco.com>
Date:   Thu Apr 9 12:29:40 2020 -0400

    appid: remove unused code
---

diff --git a/src/network_inspectors/appid/CMakeLists.txt b/src/network_inspectors/appid/CMakeLists.txt
index 90afa09ef..d333b2262 100644
--- a/src/network_inspectors/appid/CMakeLists.txt
+++ b/src/network_inspectors/appid/CMakeLists.txt
@@ -5,7 +5,6 @@ set (APPID_INCLUDES
     appid_session_api.h
     appid_types.h
     application_ids.h
-    http_xff_fields.h
     tp_appid_module_api.h
     tp_appid_session_api.h
     tp_appid_types.h
@@ -144,8 +143,6 @@ set ( UTIL_APPID_SOURCES
     appid_utils/ip_funcs.h
     appid_utils/sf_mlmp.cc
     appid_utils/sf_mlmp.h
-    appid_utils/sf_multi_mpse.cc
-    appid_utils/sf_multi_mpse.h
 )
 
 set ( APPID_SOURCES
@@ -199,7 +196,6 @@ set ( APPID_SOURCES
     lua_detector_util.h
     service_state.cc
     service_state.h
-    http_xff_fields.h
     tp_appid_utils.cc
     tp_lib_handler.cc
     tp_lib_handler.h
diff --git a/src/network_inspectors/appid/app_info_table.cc b/src/network_inspectors/appid/app_info_table.cc
index b6220ad13..75b21b54f 100644
--- a/src/network_inspectors/appid/app_info_table.cc
+++ b/src/network_inspectors/appid/app_info_table.cc
@@ -456,7 +456,7 @@ void AppInfoManager::load_odp_config(OdpContext& odp_ctxt, const char* path)
             {
                 if (!(strcasecmp(conf_val, "enabled")))
                 {
-                    odp_ctxt.tp_allow_probes = 1;
+                    odp_ctxt.tp_allow_probes = true;
                 }
             }
             else if (!(strcasecmp(conf_key, "tp_client_app")))
@@ -491,7 +491,7 @@ void AppInfoManager::load_odp_config(OdpContext& odp_ctxt, const char* path)
             {
                 if (!(strcasecmp(conf_val, "disabled")))
                 {
-                    odp_ctxt.chp_body_collection_disabled = 1;
+                    odp_ctxt.chp_body_collection_disabled = true;
                     continue;
                 }
             }
@@ -499,7 +499,7 @@ void AppInfoManager::load_odp_config(OdpContext& odp_ctxt, const char* path)
             {
                 if (!(strcasecmp(conf_val, "disabled")))
                 {
-                    odp_ctxt.ftp_userid_disabled = 1;
+                    odp_ctxt.ftp_userid_disabled = true;
                     continue;
                 }
             }
diff --git a/src/network_inspectors/appid/appid_config.h b/src/network_inspectors/appid/appid_config.h
index 859fc8c95..7bce597b0 100644
--- a/src/network_inspectors/appid/appid_config.h
+++ b/src/network_inspectors/appid/appid_config.h
@@ -85,15 +85,15 @@ public:
     bool is_host_port_app_cache_runtime = false;
     bool check_host_port_app_cache = false;
     bool check_host_cache_unknown_ssl = false;
-    uint32_t ftp_userid_disabled = 0;
-    uint32_t chp_body_collection_disabled = 0;
+    bool ftp_userid_disabled = false;
+    bool chp_body_collection_disabled = false;
     uint32_t chp_body_collection_max = 0;
     uint32_t rtmp_max_packets = 15;
     uint32_t max_tp_flow_depth = 5;
-    uint32_t tp_allow_probes = 0;
+    bool tp_allow_probes = false;
     uint32_t host_port_app_cache_lookup_interval = 10;
     uint32_t host_port_app_cache_lookup_range = 100000;
-    uint32_t http_response_version_enabled = 0;
+    bool http_response_version_enabled = false;
     bool allow_port_wildcard_host_cache = false;
     bool recheck_for_portservice_appid = false;
 
diff --git a/src/network_inspectors/appid/appid_http_session.cc b/src/network_inspectors/appid/appid_http_session.cc
index d64670dc3..293a964e3 100644
--- a/src/network_inspectors/appid/appid_http_session.cc
+++ b/src/network_inspectors/appid/appid_http_session.cc
@@ -32,7 +32,6 @@
 #include "appid_debug.h"
 #include "appid_session.h"
 #include "detector_plugins/http_url_patterns.h"
-#include "http_xff_fields.h"
 #include "tp_lib_handler.h"
 #define PORT_MAX 65535
 
@@ -50,8 +49,6 @@ AppIdHttpSession::AppIdHttpSession(AppIdSession& asd)
 
 AppIdHttpSession::~AppIdHttpSession()
 {
-    delete xff_addr;
-
     for ( int i = 0; i < NUM_METADATA_FIELDS; i++)
         delete meta_data[i];
     if (tun_dest)
@@ -723,95 +720,6 @@ int AppIdHttpSession::process_http_packet(AppidSessionDirection direction,
     return 0;
 }
 
-// FIXIT-E - Implement this function when (reconfigurable) XFF is supported.
-void AppIdHttpSession::update_http_xff_address(struct XffFieldValue* xff_fields,
-    uint32_t numXffFields, AppidChangeBits& change_bits)
-{
-    UNUSED(xff_fields);
-    UNUSED(numXffFields);
-    UNUSED(change_bits);
-#if 0
-    // When this is implemented, do change_bits.set(APPID_XFF_BIT) soon after xff_addr is changed
-    static const char* defaultXffPrecedence[] =
-    {
-        HTTP_XFF_FIELD_X_FORWARDED_FOR,
-        HTTP_XFF_FIELD_TRUE_CLIENT_IP
-    };
-
-    // XFF precedence configuration cannot change for a session. Do not get it again if we already
-    // got it.
-    char** xffPrecedence = _dpd.sessionAPI->get_http_xff_precedence(p->stream_session, p->flags,
-        &numXffFields);
-    if (!xffPrecedence)
-    {
-        xffPrecedence = defaultXffPrecedence;
-        numXffFields = sizeof(defaultXffPrecedence) / sizeof(defaultXffPrecedence[0]);
-    }
-
-    xffPrecedence = malloc(numXffFields * sizeof(char*));
-
-    for (unsigned j = 0; j < numXffFields; j++)
-        xffPrecedence[j] = strndup(xffPrecedence[j], UINT8_MAX);
-
-    if (appidDebug->is_active())
-    {
-        for (unsigned i = 0; i < numXffFields; i++)
-            LogMessage("AppIdDbg %s XFF %s : %s\n", appidDebug->get_debug_session(),
-                xff_fields[i].field.c_str(), xff_fields[i].value.empty() ? "(empty)" :
-                xff_fields[i].value);
-    }
-
-    // xffPrecedence array is sorted based on precedence
-    for (unsigned i = 0; (i < numXffFields) && xffPrecedence[i]; i++)
-    {
-        for (unsigned j = 0; j < numXffFields; j++)
-        {
-            if (xff_addr)
-            {
-                delete xff_addr;
-                xff_addr = nullptr;
-            }
-
-            if (strncasecmp(xff_fields[j].field.c_str(), xffPrecedence[i], UINT8_MAX) == 0)
-            {
-                if (xff_fields[j].value.empty())
-                    return;
-
-                // For a comma-separated list of addresses, pick the last address
-                // FIXIT-L: change to select last address port from 2.9.10-42..not tested
-
-                // FIXIT_H: - this code is wrong. We can't have
-                // tmp-xff_fields[j].value when tmp=0.
-
-                // xff_addr = new SfIp();
-                // char* xff_addr_str = nullptr;
-                // char* tmp = strchr(xff_fields[j].value, ',');
-
-                // if (tmp)
-                // {
-                //     xff_addr_str = tmp + 1;
-                // }
-                // else
-                // {
-                //     xff_fields[j].value[tmp - xff_fields[j].value] = '\0';
-                //     xff_addr_str = xff_fields[j].value;
-                // }
-
-                // if (xff_addr->set(xff_addr_str) != SFIP_SUCCESS)
-                // {
-                //     delete xff_addr;
-                //     xff_addr = nullptr;
-                // }
-                break;
-            }
-        }
-
-        if (xff_addr)
-            break;
-    }
-#endif
-}
-
 void AppIdHttpSession::update_url(AppidChangeBits& change_bits)
 {
     const std::string* host = meta_data[REQ_HOST_FID];
@@ -837,17 +745,5 @@ void AppIdHttpSession::clear_all_fields()
         delete meta_data[i];
         meta_data[i] = nullptr;
     }
-    if (xff_addr)
-    {
-        delete xff_addr;
-        xff_addr = nullptr;
-    }
-    if (xffPrecedence)
-    {
-        for (unsigned i = 0; i < numXffFields; i++)
-            delete xffPrecedence[i];
-        delete xffPrecedence;
-        xffPrecedence = NULL;
-    }
 }
 
diff --git a/src/network_inspectors/appid/appid_http_session.h b/src/network_inspectors/appid/appid_http_session.h
index 59ec8d8ca..5ede3da0e 100644
--- a/src/network_inspectors/appid/appid_http_session.h
+++ b/src/network_inspectors/appid/appid_http_session.h
@@ -31,7 +31,6 @@
 
 #include "appid_types.h"
 #include "application_ids.h"
-#include "http_xff_fields.h"
 
 class AppIdSession;
 class ChpMatchDescriptor;
@@ -101,13 +100,9 @@ public:
 
     int process_http_packet(AppidSessionDirection direction, AppidChangeBits& change_bits,
         HttpPatternMatchers& http_matchers);
-    void update_http_xff_address(struct XffFieldValue*, uint32_t, AppidChangeBits&);
 
     void update_url(AppidChangeBits& change_bits);
 
-    snort::SfIp* get_xff_addr()
-    { return xff_addr; }
-
     const std::string* get_field(HttpFieldIds id)
     { return meta_data[id]; }
 
@@ -236,9 +231,6 @@ protected:
     int num_matches = 0;
     int num_scans = 0;
     bool skip_simple_detect = false;
-    snort::SfIp* xff_addr = nullptr;
-    const char** xffPrecedence = nullptr;
-    unsigned numXffFields = 0;
     int ptype_req_counts[NUM_HTTP_FIELDS] = { 0 };
     int ptype_scan_counts[NUM_HTTP_FIELDS] = { 0 };
     const TunnelDest* tun_dest = nullptr;
diff --git a/src/network_inspectors/appid/appid_utils/sf_mlmp.h b/src/network_inspectors/appid/appid_utils/sf_mlmp.h
index 4f6f77c76..351cf9a3a 100644
--- a/src/network_inspectors/appid/appid_utils/sf_mlmp.h
+++ b/src/network_inspectors/appid/appid_utils/sf_mlmp.h
@@ -25,6 +25,12 @@
 #include <cstddef>
 #include <cstdint>
 
+struct tMlpPattern
+{
+    const uint8_t* pattern;
+    size_t patternSize;
+};
+
 struct tMlmpPattern
 {
     /*binary pattern */
diff --git a/src/network_inspectors/appid/appid_utils/sf_multi_mpse.cc b/src/network_inspectors/appid/appid_utils/sf_multi_mpse.cc
deleted file mode 100644
index 3beeb83c1..000000000
--- a/src/network_inspectors/appid/appid_utils/sf_multi_mpse.cc
+++ /dev/null
@@ -1,439 +0,0 @@
-//--------------------------------------------------------------------------
-// Copyright (C) 2014-2020 Cisco and/or its affiliates. All rights reserved.
-// Copyright (C) 2005-2013 Sourcefire, Inc.
-//
-// This program is free software; you can redistribute it and/or modify it
-// under the terms of the GNU General Public License Version 2 as published
-// by the Free Software Foundation.  You may not use, modify or distribute
-// this program under any other version of the GNU General Public License.
-//
-// This program is distributed in the hope that it will be useful, but
-// WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-// General Public License for more details.
-//
-// You should have received a copy of the GNU General Public License along
-// with this program; if not, write to the Free Software Foundation, Inc.,
-// 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
-//--------------------------------------------------------------------------
-
-// sf_multi_mpse.cc author Sourcefire Inc.
-
-#ifdef HAVE_CONFIG_H
-#include "config.h"
-#endif
-
-#include "sf_multi_mpse.h"
-
-#include "search_engines/search_tool.h"
-#include "utils/util.h"
-
-using namespace snort;
-
-struct tPatternRootNode;
-struct tPatternList
-{
-    tMlpPattern pattern;
-    void* userData;             /*client/service info */
-
-    tPatternList* nextPattern;
-    tPatternRootNode* nextLevelMatcher;
-};
-
-/*Root node */
-struct tPatternRootNode
-{
-    SearchTool* patternTree;
-    tPatternList* patternList;
-    tPatternList* lastPattern;
-    unsigned int level;        /*some searches may be specific to levels. Increments from 1 at top
-                                  level, */
-};
-
-/*Used to track matched patterns. */
-struct MatchedPattern
-{
-    tPatternList* patternNode;
-    size_t match_start_pos;
-    unsigned int level;
-};
-
-static int compareAppUrlPatterns(const void* p1, const void* p2);
-static int createTreesRecusively(void* root);
-static void destroyTreesRecursively(void* root);
-static void dumpTreesRecursively(void* root, int level);
-static int addPatternRecursively(void* root, const tMlpPattern** inputPatternList, void* metaData,
-    int level);
-static int longest_pattern_match(void* id, void*, int match_end_pos, void* data,
-    void*);
-static int url_pattern_match(void* id, void*, int match_end_pos, void* data, void*);
-
-void* mlpCreate()
-{
-    tPatternRootNode* root = (tPatternRootNode*)snort_calloc(sizeof(tPatternRootNode));
-    root->level = 0;
-    return root;
-}
-
-/*last pattern should be nullptr */
-int mlpAddPattern(void* root, const tMlpPattern** inputPatternList, void* metaData)
-{
-    return addPatternRecursively(root, inputPatternList, metaData, 0);
-}
-
-int mlpProcessPatterns(void* root)
-{
-    int rvalue;
-
-    rvalue = createTreesRecusively(root);
-    if (rvalue)
-        destroyTreesRecursively(root);
-    return rvalue;
-}
-
-void* mlpMatchPatternLongest(void* root, tMlpPattern** inputPatternList)
-{
-    return mlpMatchPatternCustom(root, inputPatternList, longest_pattern_match);
-}
-
-void* mlpMatchPatternUrl(void* root, tMlpPattern** inputPatternList)
-{
-    return mlpMatchPatternCustom(root, inputPatternList, url_pattern_match);
-}
-
-static inline bool match_is_domain_pattern(MatchedPattern mp, const uint8_t* data)
-{
-    if (!data)
-        return false;
-
-    return mp.level != 0 or
-           mp.match_start_pos == 0 or
-           data[mp.match_start_pos-1] == '.';
-}
-
-void* mlpMatchPatternCustom(void* root, tMlpPattern** inputPatternList, int (* callback)(void*,
-    void*, int, void*, void*))
-{
-    MatchedPattern mp = { nullptr,0,0 };
-    void* data = nullptr;
-    void* tmpData = nullptr;
-    tPatternList* patternNode;
-    tPatternRootNode* rootNode = (tPatternRootNode*)root;
-    tMlpPattern* pattern = *inputPatternList;
-
-    if (!rootNode || !pattern || !pattern->pattern)
-        return nullptr;
-
-    mp.level = rootNode->level;
-
-    rootNode->patternTree->find_all((const char*)pattern->pattern,
-        pattern->patternSize,
-        callback,
-        false,
-        &mp);
-
-    patternNode = mp.patternNode;
-    if (patternNode)
-    {
-        if (!match_is_domain_pattern(mp, pattern->pattern))
-            return nullptr;
-
-        data = patternNode->userData;
-        tmpData = mlpMatchPatternCustom(patternNode->nextLevelMatcher, ++inputPatternList,
-            callback);
-        if (tmpData)
-            data = tmpData;
-    }
-
-    return data;
-}
-
-void mlpDestroy(void* root)
-{
-    destroyTreesRecursively(root);
-}
-
-void mlpDump(void* root)
-{
-    dumpTreesRecursively(root, 0);
-}
-
-/*alphabetically ordering */
-static int compareAppUrlPatterns(const void* p1, const void* p2)
-{
-    const tMlpPattern* pat1 = (const tMlpPattern*)p1;
-    const tMlpPattern* pat2 = (const tMlpPattern*)p2;
-    int rValue;
-    size_t minSize;
-
-    /*first compare patterns by the smaller pattern size, if same then size wins */
-    minSize = (pat1->patternSize > pat2->patternSize) ? pat2->patternSize : pat1->patternSize;
-
-    rValue = memcmp(pat1->pattern, pat2->pattern, minSize);
-    if (rValue)
-        return rValue;
-
-    return ((int)pat1->patternSize - (int)pat2->patternSize);
-}
-
-/* Pattern trees are not freed on error because in case of error, caller
- * should call detroyTreesRecursively.
- */
-static int createTreesRecusively(void* root)
-{
-    tPatternRootNode* rootNode = (tPatternRootNode*)root;
-    SearchTool* patternMatcher;
-    tPatternList* patternNode;
-
-    /* set up the MPSE for url patterns */
-    if (!(patternMatcher = rootNode->patternTree = new SearchTool("ac_full", true)))
-        return -1;
-
-    for (patternNode = rootNode->patternList;
-        patternNode;
-        patternNode = patternNode->nextPattern)
-    {
-        /*recursion into next lower level */
-        if (patternNode->nextLevelMatcher)
-        {
-            if (createTreesRecusively(patternNode->nextLevelMatcher))
-                return -1;
-        }
-
-        patternMatcher->add(patternNode->pattern.pattern,
-            patternNode->pattern.patternSize,
-            patternNode,
-            false);
-    }
-
-    patternMatcher->prep();
-
-    return 0;
-}
-
-static void destroyTreesRecursively(void* root)
-{
-    tPatternRootNode* rootNode = (tPatternRootNode*)root;
-    tPatternList* patternNode;
-
-    while ((patternNode = rootNode->patternList))
-    {
-        /*recursion into next lower level */
-        if (patternNode->nextLevelMatcher)
-        {
-            destroyTreesRecursively(patternNode->nextLevelMatcher);
-        }
-        rootNode->patternList = patternNode->nextPattern;
-        snort_free(patternNode);
-    }
-
-    delete rootNode->patternTree;
-    snort_free(rootNode);
-}
-
-static void dumpTreesRecursively(void* root, int level)
-{
-    tPatternRootNode* rootNode = (tPatternRootNode*)root;
-    tPatternList* patternNode;
-    char* offset;
-
-    offset = (char*)snort_calloc(4*level+2);
-    if (!offset)
-        return;
-    memset(offset, ' ', 4*level+1);
-    offset[4*level] = '\0';
-
-    for (patternNode = rootNode->patternList;
-        patternNode;
-        patternNode = patternNode->nextPattern)
-    {
-        printf("%sPattern %s, size %u, userData %p\n", offset,
-            (const char*)patternNode->pattern.pattern,
-            (uint32_t)patternNode->pattern.patternSize,
-            patternNode->userData);
-
-        /*recursion into next lower level */
-        if (patternNode->nextLevelMatcher)
-        {
-            dumpTreesRecursively(patternNode->nextLevelMatcher, (level+1));
-        }
-    }
-    snort_free(offset);
-}
-
-static int longest_pattern_match(void* id, void*, int match_end_pos, void* data,
-    void*)
-{
-    tPatternList* target = (tPatternList*)id;
-    MatchedPattern* match = (MatchedPattern*)data;
-    int newMatchWins = 0;
-
-    /*printf("LongestMatcher: level %d, match_end_pos: %d, matched %s\n", matches->level, match_end_pos,
-       target->pattern.pattern); */
-
-    /*first match */
-    if (!match->patternNode)
-        newMatchWins = 1;
-    /*subsequent longer match */
-    else if (match->patternNode->pattern.patternSize < target->pattern.patternSize)
-        newMatchWins = 1;
-
-    if (newMatchWins)
-    {
-        /*printf("new pattern wins\n"); */
-        match->patternNode = target;
-        match->match_start_pos = match_end_pos - target->pattern.patternSize;
-    }
-
-    return 0;
-}
-
-static int url_pattern_match(void* id, void*, int match_end_pos, void* data, void*)
-{
-    tPatternList* target = (tPatternList*)id;
-    MatchedPattern* match = (MatchedPattern*)data;
-    int newMatchWins = 0;
-
-    /*printf("UrlMatcher: level %d, match_end_pos: %d, matched %s\n", match->level, match_end_pos,
-      target->pattern.pattern);
-      first match */
-    if (!match->patternNode)
-        newMatchWins = 1;
-
-    /*subsequent longer match */
-    else if (match->patternNode->pattern.patternSize < target->pattern.patternSize)
-        newMatchWins = 1;
-    else if (match->patternNode->pattern.patternSize == target->pattern.patternSize)
-    {
-        /*host part matching towards later part is better. This is not designed to prevent
-          mis-identifying
-          url 'www.spoof_for_google.google.com.phishing.com' as google. */
-        if ((match->level == 0) &&
-            (match->match_start_pos < (unsigned int)match_end_pos - target->pattern.patternSize))
-            newMatchWins = 1;
-        /*path part matching towards lower position is better */
-        if ((match->level == 1) &&
-            (match->match_start_pos > (unsigned int)match_end_pos - target->pattern.patternSize))
-            newMatchWins = 1;
-    }
-
-    if (newMatchWins)
-    {
-        /*printf("new pattern wins\n"); */
-        match->patternNode = target;
-        match->match_start_pos = match_end_pos - target->pattern.patternSize;
-    }
-
-    return 0;
-}
-
-static int addPatternRecursively(void* root, const tMlpPattern** inputPatternList, void* metaData,
-    int level)
-{
-    tPatternRootNode* rootNode = (tPatternRootNode*)root;
-    tPatternList* prevNode = nullptr;
-    tPatternList* patternList;
-    tPatternList* newNode;
-    const tMlpPattern* nextPattern;
-    const tMlpPattern* patterns = *inputPatternList;
-
-    if (!rootNode || !patterns || !patterns->pattern)
-        return -1;
-
-    for (patternList = rootNode->patternList;
-        patternList;
-        prevNode = patternList, patternList = patternList->nextPattern)
-    {
-        int rvalue = compareAppUrlPatterns(patterns, patternList);
-
-        if (rvalue < 0)
-            continue;
-
-        if (rvalue == 0)
-        {
-            nextPattern = *(inputPatternList+1);
-
-            if (!nextPattern || !nextPattern->pattern)
-            {
-                /*overriding any previous userData. */
-                patternList->userData = metaData;
-                return 0;
-            }
-            return addPatternRecursively(patternList->nextLevelMatcher, inputPatternList+1,
-                metaData, level+1);
-        }
-        break;
-    }
-
-    /*allocate and initialize a new node */
-    newNode = (tPatternList*)snort_calloc(sizeof(tPatternList));
-    newNode->pattern.pattern = patterns->pattern;
-    newNode->pattern.patternSize = patterns->patternSize;
-    newNode->nextLevelMatcher = (tPatternRootNode*)snort_calloc(sizeof(tPatternRootNode));
-    newNode->nextLevelMatcher->level = rootNode->level+1;
-
-    /*insert the new node */
-    if (!prevNode)
-    {
-        /*insert as first node since either this is the only node, or this is lexically smallest.
-           */
-        newNode->nextPattern = rootNode->patternList;
-        rootNode->patternList = newNode;
-    }
-    else
-    {
-        /*insert after previous node since either there is either a biggest node after prevNode or
-          newNode is lexically largest. */
-        newNode->nextPattern = prevNode->nextPattern;
-        prevNode->nextPattern = newNode;
-    }
-
-    /*move down the new node */
-    nextPattern = *(inputPatternList+1);
-    if (!nextPattern || !nextPattern->pattern)
-    {
-        newNode->userData = metaData;
-    }
-    else
-    {
-        addPatternRecursively(newNode->nextLevelMatcher, inputPatternList+1, metaData, level+1);
-    }
-
-    return 0;
-}
-
-/**returns pattern tree at the level where inputPatternList runs out.
- */
-void* mlpGetPatternMatcherTree(void* root, tMlpPattern** inputPatternList)
-{
-    MatchedPattern mp = { nullptr,0,0 };
-    tPatternList* patternNode;
-    tPatternRootNode* rootNode = (tPatternRootNode*)root;
-    tMlpPattern* pattern = *inputPatternList;
-
-    if (!rootNode || !pattern || !pattern->pattern)
-        return nullptr;
-
-    mp.level = rootNode->level;
-
-    rootNode->patternTree->find_all((const char*)pattern->pattern,
-        pattern->patternSize,
-        longest_pattern_match,
-        false,
-        &mp);
-
-    patternNode = mp.patternNode;
-    if (patternNode)
-    {
-        ++inputPatternList;
-        if (*inputPatternList && (*inputPatternList)->pattern)
-        {
-            return mlpMatchPatternCustom(patternNode->nextLevelMatcher, inputPatternList,
-                longest_pattern_match);
-        }
-        return patternNode->nextLevelMatcher;
-    }
-
-    return nullptr;
-}
-
diff --git a/src/network_inspectors/appid/appid_utils/sf_multi_mpse.h b/src/network_inspectors/appid/appid_utils/sf_multi_mpse.h
deleted file mode 100644
index 6a1cfd855..000000000
--- a/src/network_inspectors/appid/appid_utils/sf_multi_mpse.h
+++ /dev/null
@@ -1,46 +0,0 @@
-//--------------------------------------------------------------------------
-// Copyright (C) 2014-2020 Cisco and/or its affiliates. All rights reserved.
-// Copyright (C) 2005-2013 Sourcefire, Inc.
-//
-// This program is free software; you can redistribute it and/or modify it
-// under the terms of the GNU General Public License Version 2 as published
-// by the Free Software Foundation.  You may not use, modify or distribute
-// this program under any other version of the GNU General Public License.
-//
-// This program is distributed in the hope that it will be useful, but
-// WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-// General Public License for more details.
-//
-// You should have received a copy of the GNU General Public License along
-// with this program; if not, write to the Free Software Foundation, Inc.,
-// 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
-//--------------------------------------------------------------------------
-
-// sf_multi_mpse.h author Sourcefire Inc.
-
-#ifndef SF_MULTI_MPSE_H
-#define SF_MULTI_MPSE_H
-
-#include <cstddef>
-#include <cstdint>
-
-struct tMlpPattern
-{
-    const uint8_t* pattern;
-    size_t patternSize;
-};
-
-void* mlpCreate();
-int mlpAddPattern(void* root, const tMlpPattern**, void* metaData);
-int mlpProcessPatterns(void* root);
-void* mlpMatchPatternLongest(void* root, tMlpPattern**);
-void* mlpMatchPatternUrl(void* root, tMlpPattern**);
-void* mlpMatchPatternCustom(void* root, tMlpPattern**,
-    int (* callback)(void*, void*, int, void*, void*));
-void mlpDestroy(void* root);
-void mlpDump(void* root);
-void* mlpGetPatternMatcherTree(void* root, tMlpPattern**);
-
-#endif
-
diff --git a/src/network_inspectors/appid/detector_plugins/http_url_patterns.h b/src/network_inspectors/appid/detector_plugins/http_url_patterns.h
index b7bfb06cc..d1e524373 100644
--- a/src/network_inspectors/appid/detector_plugins/http_url_patterns.h
+++ b/src/network_inspectors/appid/detector_plugins/http_url_patterns.h
@@ -32,7 +32,6 @@
 
 #include "appid_http_session.h"
 #include "appid_utils/sf_mlmp.h"
-#include "appid_utils/sf_multi_mpse.h"
 #include "application_ids.h"
 
 namespace snort
diff --git a/src/network_inspectors/appid/detector_plugins/sip_patterns.h b/src/network_inspectors/appid/detector_plugins/sip_patterns.h
index ed87e75e0..ae9688423 100644
--- a/src/network_inspectors/appid/detector_plugins/sip_patterns.h
+++ b/src/network_inspectors/appid/detector_plugins/sip_patterns.h
@@ -22,7 +22,6 @@
 #define SIP_PATTERNS_H
 
 #include "appid_utils/sf_mlmp.h"
-#include "appid_utils/sf_multi_mpse.h"
 #include "application_ids.h"
 
 class OdpContext;
diff --git a/src/network_inspectors/appid/detector_plugins/test/detector_plugins_mock.h b/src/network_inspectors/appid/detector_plugins/test/detector_plugins_mock.h
index 2d4dd50cb..5a03142ef 100644
--- a/src/network_inspectors/appid/detector_plugins/test/detector_plugins_mock.h
+++ b/src/network_inspectors/appid/detector_plugins/test/detector_plugins_mock.h
@@ -154,8 +154,6 @@ AppIdHttpSession::AppIdHttpSession(AppIdSession& asd)
 
 AppIdHttpSession::~AppIdHttpSession()
 {
-    delete xff_addr;
-
     for ( int i = 0; i < NUM_METADATA_FIELDS; i++)
     {
         if ( meta_data[i] )
diff --git a/src/network_inspectors/appid/detector_plugins/test/http_url_patterns_test.cc b/src/network_inspectors/appid/detector_plugins/test/http_url_patterns_test.cc
index 09fcddaef..29a9e4034 100644
--- a/src/network_inspectors/appid/detector_plugins/test/http_url_patterns_test.cc
+++ b/src/network_inspectors/appid/detector_plugins/test/http_url_patterns_test.cc
@@ -25,7 +25,6 @@
 
 #include "protocols/protocol_ids.h"
 #include "framework/module.cc"
-#include "network_inspectors/appid/appid_utils/sf_multi_mpse.h"
 #include "network_inspectors/appid/appid_utils/sf_mlmp.cc"
 #include "utils/util_cstring.cc"
 #include "detector_plugins_mock.h"
diff --git a/src/network_inspectors/appid/http_xff_fields.h b/src/network_inspectors/appid/http_xff_fields.h
deleted file mode 100644
index e7d3563a9..000000000
--- a/src/network_inspectors/appid/http_xff_fields.h
+++ /dev/null
@@ -1,39 +0,0 @@
-//--------------------------------------------------------------------------
-// Copyright (C) 2014-2020 Cisco and/or its affiliates. All rights reserved.
-// Copyright (C) 2005-2013 Sourcefire, Inc.
-//
-// This program is free software; you can redistribute it and/or modify it
-// under the terms of the GNU General Public License Version 2 as published
-// by the Free Software Foundation.  You may not use, modify or distribute
-// this program under any other version of the GNU General Public License.
-//
-// This program is distributed in the hope that it will be useful, but
-// WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-// General Public License for more details.
-//
-// You should have received a copy of the GNU General Public License along
-// with this program; if not, write to the Free Software Foundation, Inc.,
-// 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
-//--------------------------------------------------------------------------
-
-// http_xff_fields.h author Sourcefire Inc.
-
-#ifndef HTTP_XFF_FIELDS_H
-#define HTTP_XFF_FIELDS_H
-
-#include <string>
-
-// FIXIT-L refactor
-#define HTTP_XFF_FIELD_X_FORWARDED_FOR  "X-Forwarded-For"
-#define HTTP_XFF_FIELD_TRUE_CLIENT_IP   "True-Client-IP"
-
-#define HTTP_MAX_XFF_FIELDS 8
-
-struct XffFieldValue
-{
-    std::string field;
-    std::string value;
-};
-
-#endif
diff --git a/src/network_inspectors/appid/test/appid_discovery_test.cc b/src/network_inspectors/appid/test/appid_discovery_test.cc
index d899b8ab7..d832a7a32 100644
--- a/src/network_inspectors/appid/test/appid_discovery_test.cc
+++ b/src/network_inspectors/appid/test/appid_discovery_test.cc
@@ -318,7 +318,7 @@ TEST(appid_discovery_tests, event_published_when_ignoring_flow)
 
     // Detect changes in service, client, payload, and misc appid
     CHECK_EQUAL(databus_publish_called, true);
-    STRCMP_EQUAL(test_log, "Published change_bits == 0000000001111");
+    STRCMP_EQUAL(test_log, "Published change_bits == 000000001111");
     delete asd;
     delete flow;
 }
@@ -348,7 +348,7 @@ TEST(appid_discovery_tests, event_published_when_processing_flow)
 
     // Detect changes in service, client, payload, and misc appid
     CHECK_EQUAL(databus_publish_called, true);
-    STRCMP_EQUAL(test_log, "Published change_bits == 0000000001111");
+    STRCMP_EQUAL(test_log, "Published change_bits == 000000001111");
     delete asd;
     delete flow;
 }
@@ -441,10 +441,10 @@ TEST(appid_discovery_tests, change_bits_to_string)
     change_bits.set();
     change_bits_to_string(change_bits, str);
     STRCMP_EQUAL(str.c_str(), "service, client, payload, misc, referred, host,"
-        " tls-host, url, user-agent, response, referrer, xff, client-version");
+        " tls-host, url, user-agent, response, referrer, client-version");
 
     // Failure of this test is a reminder that enum is changed, hence translator needs update
-    CHECK_EQUAL(APPID_MAX_BIT, 13);
+    CHECK_EQUAL(APPID_MAX_BIT, 12);
 }
 
 int main(int argc, char** argv)
diff --git a/src/network_inspectors/appid/test/appid_mock_http_session.h b/src/network_inspectors/appid/test/appid_mock_http_session.h
index c4dbe64ea..21809ce71 100644
--- a/src/network_inspectors/appid/test/appid_mock_http_session.h
+++ b/src/network_inspectors/appid/test/appid_mock_http_session.h
@@ -34,8 +34,6 @@ AppIdHttpSession::AppIdHttpSession(AppIdSession& session)
 
 AppIdHttpSession::~AppIdHttpSession()
 {
-    delete xff_addr;
-
     for ( int i = 0; i < NUM_METADATA_FIELDS; i++)
     {
         if ( meta_data[i] )
@@ -45,7 +43,6 @@ AppIdHttpSession::~AppIdHttpSession()
 
 int AppIdHttpSession::process_http_packet(AppidSessionDirection, AppidChangeBits&, HttpPatternMatchers&) { return 0; }
 
-char const* APPID_UT_XFF_IP_ADDR = "192.168.0.1";
 char const* CONTENT_TYPE = "html/text";
 char const* COOKIE = "this is my request cookie content";
 char const* NEW_COOKIE = "request new cookie content is chocolate chip";
@@ -83,10 +80,6 @@ public:
     MockAppIdHttpSession(AppIdSession& asd)
         : AppIdHttpSession(asd)
     {
-        SfIp* ip = new SfIp;
-        ip->pton(AF_INET, APPID_UT_XFF_IP_ADDR);
-        xff_addr = ip;
-
         meta_data[REQ_AGENT_FID] = new std::string(USERAGENT);
         meta_data[REQ_HOST_FID] = new std::string(HOST);
         meta_data[REQ_REFERER_FID] = new std::string(REFERER);
diff --git a/src/network_inspectors/appid/tp_appid_module_api.h b/src/network_inspectors/appid/tp_appid_module_api.h
index ba0fdb5a5..f2a1d382f 100644
--- a/src/network_inspectors/appid/tp_appid_module_api.h
+++ b/src/network_inspectors/appid/tp_appid_module_api.h
@@ -25,28 +25,20 @@
 #include <string>
 #include "tp_appid_types.h"
 
-#define THIRD_PARTY_APPID_API_VERSION 4
+#define THIRD_PARTY_APPID_API_VERSION 5
 
 class ThirdPartyConfig
 {
 public:
-    unsigned chp_body_collection_max;
-    unsigned ftp_userid_disabled : 1;
-    unsigned chp_body_collection_disabled : 1;
-    unsigned tp_allow_probes : 1;
-    unsigned http_upgrade_reporting_enabled : 1;
-    unsigned http_response_version_enabled : 1;
+    uint32_t chp_body_collection_max = 0;
+    bool ftp_userid_disabled = false;
+    bool chp_body_collection_disabled = false;
+    bool tp_allow_probes = false;
+    bool http_upgrade_reporting_enabled = false;
+    bool http_response_version_enabled = false;
     std::string tp_appid_config;
-    std::vector<std::string> xff_fields;
     bool tp_appid_stats_enable = false;
     bool tp_appid_config_dump = false;
-
-    ThirdPartyConfig()
-    {
-        xff_fields.clear();
-        xff_fields.emplace_back(HTTP_XFF_FIELD_X_FORWARDED_FOR);
-        xff_fields.emplace_back(HTTP_XFF_FIELD_TRUE_CLIENT_IP);
-    }
 };
 
 class ThirdPartyAppIdContext
diff --git a/src/network_inspectors/appid/tp_appid_types.h b/src/network_inspectors/appid/tp_appid_types.h
index 91348401e..9e839e4ed 100644
--- a/src/network_inspectors/appid/tp_appid_types.h
+++ b/src/network_inspectors/appid/tp_appid_types.h
@@ -25,8 +25,6 @@
 #include <cstdint>
 #include <string>
 
-#include "http_xff_fields.h"
-
 using std::string;
 
 enum TPFlags
@@ -152,9 +150,6 @@ class ThirdPartyAppIDAttributeData
     // FIXIT-L: make these private too. Figure out how these get set in tp.
 
 public:
-    XffFieldValue xffFieldValue[HTTP_MAX_XFF_FIELDS];
-    uint8_t numXffFields = 0;
-
     ThirdPartyAppIDAttributeData() { }
 
     ~ThirdPartyAppIDAttributeData()
diff --git a/src/network_inspectors/appid/tp_appid_utils.cc b/src/network_inspectors/appid/tp_appid_utils.cc
index b26281ede..4cede1748 100644
--- a/src/network_inspectors/appid/tp_appid_utils.cc
+++ b/src/network_inspectors/appid/tp_appid_utils.cc
@@ -375,10 +375,6 @@ static inline void process_http_session(AppIdSession& asd,
         hsession->set_field(RSP_BODY_FID, field, change_bits);
     }
 
-    if (attribute_data.numXffFields)
-        hsession->update_http_xff_address(attribute_data.xffFieldValue,
-            attribute_data.numXffFields, change_bits);
-
     if (!hsession->is_chp_finished() || hsession->is_chp_hold_flow())
     {
         asd.set_session_flags(APPID_SESSION_CHP_INSPECTING);
diff --git a/src/network_inspectors/appid/tp_lib_handler.cc b/src/network_inspectors/appid/tp_lib_handler.cc
index 06616711e..ade27ab2d 100644
--- a/src/network_inspectors/appid/tp_lib_handler.cc
+++ b/src/network_inspectors/appid/tp_lib_handler.cc
@@ -103,9 +103,9 @@ ThirdPartyAppIdContext* TPLibHandler::create_tp_appid_ctxt(const AppIdConfig& co
         odp_ctxt.chp_body_collection_disabled;
     tp_config.tp_allow_probes = odp_ctxt.tp_allow_probes;
     if (odp_ctxt.http2_detection_enabled)
-        tp_config.http_upgrade_reporting_enabled = 1;
+        tp_config.http_upgrade_reporting_enabled = true;
     else
-        tp_config.http_upgrade_reporting_enabled = 0;
+        tp_config.http_upgrade_reporting_enabled = false;
     tp_config.http_response_version_enabled = odp_ctxt.http_response_version_enabled;
 
     ThirdPartyAppIdContext* tp_appid_ctxt = self->tp_appid_create_ctxt(tp_config);
diff --git a/src/pub_sub/appid_events.h b/src/pub_sub/appid_events.h
index 9fd48119a..8bf3cf410 100644
--- a/src/pub_sub/appid_events.h
+++ b/src/pub_sub/appid_events.h
@@ -46,7 +46,6 @@ enum AppidChangeBit
     APPID_USERAGENT_BIT,
     APPID_RESPONSE_BIT,
     APPID_REFERER_BIT,
-    APPID_XFF_BIT,
 
     // other
     APPID_VERSION_BIT,
@@ -82,8 +81,6 @@ inline void change_bits_to_string(AppidChangeBits& change_bits, std::string& str
         --n? str.append("response, ") : str.append("response");
     if (change_bits.test(APPID_REFERER_BIT))
         --n? str.append("referrer, ") : str.append("referrer");
-    if (change_bits.test(APPID_XFF_BIT))
-        --n? str.append("xff, ") : str.append("xff");
     if (change_bits.test(APPID_VERSION_BIT))
         --n? str.append("client-version, ") : str.append("client-version");
     if (n != 0) // make sure all bits from AppidChangeBit enum get translated