From: Tom Peters (thopeter) <thopeter@cisco.com>
Date: Wed, 5 Apr 2017 14:45:54 +0000 (-0400)
Subject: Merge pull request #850 in SNORT/snort3 from appid_syncup_291042_tp187691 to master
X-Git-Tag: 3.0.0-233~34
X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=78076d7633fead0819f3996049071eaf44adefdf;p=thirdparty%2Fsnort3.git

Merge pull request #850 in SNORT/snort3 from appid_syncup_291042_tp187691 to master

Squashed commit of the following:

commit f60f7d82a30affc5738d653d14fb8aace5141188
Author: davis mcpherson <davmcphe.cisco.com>
Date:   Tue Apr 4 16:05:30 2017 -0400

    updates based on PR code review comments

commit b62183ee7da3b454d5720e5a0fe320cc8dad60f9
Author: davis mcpherson <davmcphe.cisco.com>
Date:   Mon Mar 27 15:05:21 2017 -0400

    refactor failed service detection handling to make functions that handle failure member functions of the ServiceDiscoveryState class

    refactor to port 2.9 changes to the process for selecting service detectors for a flow and managing the service discovery process
---

diff --git a/src/network_inspectors/appid/appid_api.cc b/src/network_inspectors/appid/appid_api.cc
index a166f410b..7ee4df8b4 100644
--- a/src/network_inspectors/appid/appid_api.cc
+++ b/src/network_inspectors/appid/appid_api.cc
@@ -160,7 +160,7 @@ bool AppIdApi::is_appid_inspecting_session(AppIdSession* appIdSession)
 {
     if (appIdSession && appIdSession->common.flow_type == APPID_FLOW_TYPE_NORMAL)
     {
-        if (appIdSession->rna_service_state != RNA_STATE_FINISHED ||
+        if (appIdSession->service_disco_state != APPID_DISCO_STATE_FINISHED ||
             !is_third_party_appid_done(appIdSession->tpsession) ||
             appIdSession->get_session_flags(APPID_SESSION_HTTP_SESSION | APPID_SESSION_CONTINUE) ||
             (appIdSession->get_session_flags(APPID_SESSION_ENCRYPTED) &&
@@ -169,9 +169,9 @@ bool AppIdApi::is_appid_inspecting_session(AppIdSession* appIdSession)
         {
             return true;
         }
-        if (appIdSession->rna_client_state != RNA_STATE_FINISHED &&
+        if (appIdSession->client_disco_state != APPID_DISCO_STATE_FINISHED &&
             (!appIdSession->get_session_flags(APPID_SESSION_CLIENT_DETECTED) ||
-            (appIdSession->rna_service_state != RNA_STATE_STATEFUL
+            (appIdSession->service_disco_state != APPID_DISCO_STATE_STATEFUL
             && appIdSession->get_session_flags(APPID_SESSION_CLIENT_GETS_SERVER_PACKETS))))
         {
             return true;
@@ -544,11 +544,11 @@ uint32_t AppIdApi::consume_ha_state(Flow* flow, const uint8_t* buf, uint8_t, IpP
                 {
                     asd->set_session_flags(APPID_SESSION_CONTINUE);
                 }
-                asd->rna_service_state = RNA_STATE_STATEFUL;
+                asd->service_disco_state = APPID_DISCO_STATE_STATEFUL;
             }
             else
-                asd->rna_service_state = RNA_STATE_FINISHED;
-            asd->rna_client_state = RNA_STATE_FINISHED;
+                asd->service_disco_state = APPID_DISCO_STATE_FINISHED;
+            asd->client_disco_state = APPID_DISCO_STATE_FINISHED;
             if (thirdparty_appid_module)
                 thirdparty_appid_module->session_state_set(asd->tpsession, TP_STATE_HA);
         }
diff --git a/src/network_inspectors/appid/appid_config.h b/src/network_inspectors/appid/appid_config.h
index eee8238e7..4b2536094 100644
--- a/src/network_inspectors/appid/appid_config.h
+++ b/src/network_inspectors/appid/appid_config.h
@@ -124,14 +124,6 @@ public:
     bool http2_detection_enabled = false;
 };
 
-// App ID Active Configuration
-enum RnaFwConfigState
-{
-    RNA_FW_CONFIG_STATE_UNINIT,
-    RNA_FW_CONFIG_STATE_INIT,
-    RNA_FW_CONFIG_STATE_PENDING,
-};
-
 typedef std::array<SF_LIST*, APP_ID_PORT_ARRAY_SIZE> AppIdPortExclusions;
 
 class AppIdConfig
diff --git a/src/network_inspectors/appid/appid_detector.h b/src/network_inspectors/appid/appid_detector.h
index 12573a600..b18bd6e97 100644
--- a/src/network_inspectors/appid/appid_detector.h
+++ b/src/network_inspectors/appid/appid_detector.h
@@ -35,6 +35,17 @@ struct Packet;
 
 #define STATE_ID_MAX_VALID_COUNT 5
 
+enum DetectorType
+{
+    DETECTOR_TYPE_DECODER =  0,
+    DETECTOR_TYPE_NETFLOW,
+    DETECTOR_TYPE_PORT,
+    DETECTOR_TYPE_DERIVED,
+    DETECTOR_TYPE_CONFLICT,
+    DETECTOR_TYPE_PATTERN,
+    DETECTOR_TYPE_NOT_SET
+};
+
 typedef std::vector<AppRegistryEntry> FlowApplicationInfo;
 
 struct AppIdFlowContentPattern
@@ -116,8 +127,6 @@ public:  // FIXIT-M - make this protected:
     bool provides_user = false;
     unsigned flow_data_index = 0;
     unsigned detectorType = DETECTOR_TYPE_NOT_SET;
-    unsigned ref_count = 1;
-    unsigned current_ref_count = 0;
     bool isCustom = false;
 
     AppIdFlowContentPatterns tcp_patterns;
@@ -126,12 +135,6 @@ public:  // FIXIT-M - make this protected:
     ServiceDetectorPorts service_ports;
 };
 
-inline uint32_t get_service_detect_level(AppIdSession* asd)
-{
-    if (asd->get_session_flags(APPID_SESSION_DECRYPTED))
-        return 1;
-    return 0;
-}
 
 #if defined(WORDS_BIGENDIAN)
 #define LETOHS(p)   BYTE_SWAP_16(*((uint16_t*)(p)))
diff --git a/src/network_inspectors/appid/appid_discovery.cc b/src/network_inspectors/appid/appid_discovery.cc
index c9f183a08..6ccc8ee5d 100644
--- a/src/network_inspectors/appid/appid_discovery.cc
+++ b/src/network_inspectors/appid/appid_discovery.cc
@@ -17,7 +17,7 @@
 // 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 //--------------------------------------------------------------------------
 
-// client_app_bit.cc author Sourcefire Inc.
+// appid_discovery.cc author Sourcefire Inc.
 
 #ifdef HAVE_CONFIG_H
 #include "config.h"
@@ -45,7 +45,6 @@
 #include "log/messages.h"
 #include "protocols/packet.h"
 #include "protocols/tcp.h"
-#include "time/packet_time.h"
 
 ProfileStats tpPerfStats;
 
@@ -630,8 +629,8 @@ void AppIdDiscovery::do_application_discovery(Packet* p)
 
     if (asd->get_session_flags(APPID_SESSION_IGNORE_FLOW))
     {
-        if (asd->session_logging_enabled && !asd->get_session_flags(
-            APPID_SESSION_IGNORE_FLOW_LOGGED))
+        if ( asd->session_logging_enabled &&
+             !asd->get_session_flags(APPID_SESSION_IGNORE_FLOW_LOGGED) )
         {
             asd->set_session_flags(APPID_SESSION_IGNORE_FLOW_LOGGED);
             LogMessage("AppIdDbg %s Ignoring connection with service %d\n",
@@ -661,20 +660,7 @@ void AppIdDiscovery::do_application_discovery(Packet* p)
                 port = p->ptrs.sp;
             }
 
-            ServiceDiscoveryState* id_state = AppIdServiceState::get(ip, IpProtocol::TCP, port,
-                get_service_detect_level(asd));
-
-            if (id_state)
-            {
-                if (!id_state->reset_time)
-                    id_state->reset_time = packet_time();
-                else if ((packet_time() - id_state->reset_time) >= 60)
-                {
-                    AppIdServiceState::remove(ip, IpProtocol::TCP, port,
-                        get_service_detect_level(asd));
-                    asd->set_session_flags(APPID_SESSION_SERVICE_DELETED);
-                }
-            }
+            AppIdServiceState::check_reset(asd, ip, port);
         }
 
         asd->previous_tcp_flags = p->ptrs.tcph->th_flags;
@@ -704,7 +690,7 @@ void AppIdDiscovery::do_application_discovery(Packet* p)
             {
             case 1:
                 asd->client_app_id = hv->appId;
-                asd->rna_client_state = RNA_STATE_FINISHED;
+                asd->client_disco_state = APPID_DISCO_STATE_FINISHED;
                 break;
             case 2:
                 asd->payload_app_id = hv->appId;
@@ -712,8 +698,8 @@ void AppIdDiscovery::do_application_discovery(Packet* p)
             default:
                 asd->serviceAppId = hv->appId;
                 asd->sync_with_snort_id(hv->appId, p);
-                asd->rna_service_state = RNA_STATE_FINISHED;
-                asd->rna_client_state = RNA_STATE_FINISHED;
+                asd->service_disco_state = APPID_DISCO_STATE_FINISHED;
+                asd->client_disco_state = APPID_DISCO_STATE_FINISHED;
                 asd->set_session_flags(APPID_SESSION_SERVICE_DETECTED);
                 if (thirdparty_appid_module)
                     thirdparty_appid_module->session_delete(asd->tpsession, 1);
@@ -812,7 +798,7 @@ void AppIdDiscovery::do_application_discovery(Packet* p)
     /* exceptions for rexec and any other service detector that needs to see SYN and SYN/ACK */
     if (asd->get_session_flags(APPID_SESSION_REXEC_STDERR))
     {
-        ServiceDiscovery::get_instance().AppIdDiscoverService(p, direction, asd);
+        ServiceDiscovery::get_instance().identify_service(asd, p, direction);
         if (asd->serviceAppId == APP_ID_DNS &&
             asd->config->mod_config->dns_host_reporting &&
             asd->dsession && asd->dsession->host )
diff --git a/src/network_inspectors/appid/appid_discovery.h b/src/network_inspectors/appid/appid_discovery.h
index c2aa62ec1..820d816b3 100644
--- a/src/network_inspectors/appid/appid_discovery.h
+++ b/src/network_inspectors/appid/appid_discovery.h
@@ -17,7 +17,7 @@
 // 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 //--------------------------------------------------------------------------
 
-// client_app_config.h author Sourcefire Inc.
+// appid_discovery.h author Sourcefire Inc.
 
 #ifndef APPID_DISCOVERY_H
 #define APPID_DISCOVERY_H
diff --git a/src/network_inspectors/appid/appid_inspector.cc b/src/network_inspectors/appid/appid_inspector.cc
index b667e0ee9..e8593dd49 100644
--- a/src/network_inspectors/appid/appid_inspector.cc
+++ b/src/network_inspectors/appid/appid_inspector.cc
@@ -136,6 +136,7 @@ void AppIdInspector::tinit()
 {
     appid_stats_manager = AppIdStatistics::initialize_manager(*config);
     HostPortCache::initialize();
+    AppIdServiceState::initialize();
     init_appid_forecast();
     HttpPatternMatchers* http_matchers = HttpPatternMatchers::get_instance();
     AppIdDiscovery::initialize_plugins();
@@ -148,7 +149,6 @@ void AppIdInspector::tinit()
     SipUdpClientDetector::finalize_sip_ua();
     ssl_detector_process_patterns();
     dns_host_detector_process_patterns();
-    AppIdServiceState::initialize(config->memcap);
 }
 
 void AppIdInspector::tterm()
@@ -160,9 +160,9 @@ void AppIdInspector::tterm()
     service_ssl_clean();
     free_length_app_cache();
 
+    AppIdServiceState::clean();
     AppIdSession::release_free_list_flow_data();
     LuaDetectorManager::terminate();
-    AppIdServiceState::clean();
     AppIdDiscovery::release_plugins();
     delete HttpPatternMatchers::get_instance();
 }
diff --git a/src/network_inspectors/appid/appid_session.cc b/src/network_inspectors/appid/appid_session.cc
index c2801f48b..8cba2f58d 100644
--- a/src/network_inspectors/appid/appid_session.cc
+++ b/src/network_inspectors/appid/appid_session.cc
@@ -99,14 +99,14 @@ AppIdSession* AppIdSession::allocate_session(const Packet* p, IpProtocol proto,
     if ( ( proto == IpProtocol::TCP || proto == IpProtocol::UDP ) && ( p->ptrs.sp != p->ptrs.dp ) )
         port = (direction == APP_ID_FROM_INITIATOR) ? p->ptrs.sp : p->ptrs.dp;
 
-    AppIdSession* data = new AppIdSession(proto, ip, port);
-
-    data->flow = p->flow;
-    data->stats.firstPktsecond = p->pkth->ts.tv_sec;
-    data->set_session_logging_state(p, direction);
-    data->snort_id = snortId_for_unsynchronized;
-    p->flow->set_flow_data(data);
-    return data;
+    AppIdSession* asd = new AppIdSession(proto, ip, port);
+
+    asd->flow = p->flow;
+    asd->stats.firstPktsecond = p->pkth->ts.tv_sec;
+    asd->set_session_logging_state(p, direction);
+    asd->snort_id = snortId_for_unsynchronized;
+    p->flow->set_flow_data(asd);
+    return asd;
 }
 
 AppIdSession::AppIdSession(IpProtocol proto, const SfIp* ip, uint16_t port)
@@ -138,8 +138,21 @@ AppIdSession::~AppIdSession()
         if ( stats_mgr )
             stats_mgr->update(this);
 
-        if (flow)
-            FailInProcessService(this, config);
+        // fail any service detection that is in process for this flow
+        if (flow &&
+            !get_session_flags(APPID_SESSION_SERVICE_DETECTED | APPID_SESSION_UDP_REVERSED) )
+        {
+            ServiceDiscoveryState* sds =
+                AppIdServiceState::get(&service_ip, protocol, service_port, is_decrypted());
+            if( sds )
+            {
+                sds->invalid_client_count += STATE_ID_INCONCLUSIVE_SERVICE_WEIGHT;
+                if (flow->server_ip.fast_eq6(service_ip))
+                    sds->set_service_id_failed(this, &flow->client_ip);
+                else
+                    sds->set_service_id_failed(this, &flow->server_ip);
+            }
+        }
     }
 
     delete_shared_data();
@@ -256,7 +269,7 @@ void AppIdSession::reinit_shared_data()
 
         service_ip.clear();
         service_port = 0;
-        rna_service_state = RNA_STATE_NONE;
+        service_disco_state = APPID_DISCO_STATE_NONE;
         service_detector = nullptr;
         free_flow_data_by_mask(APPID_SESSION_DATA_SERVICE_MODSTATE_BIT);
     }
@@ -268,7 +281,7 @@ void AppIdSession::reinit_shared_data()
         snort_free(client_version);
         client_version = nullptr;
     }
-    rna_client_state = RNA_STATE_NONE;
+    client_disco_state = APPID_DISCO_STATE_NONE;
     free_flow_data_by_mask(APPID_SESSION_DATA_CLIENT_MODSTATE_BIT);
 
     //3rd party cleaning
@@ -350,18 +363,15 @@ bool AppIdSession::is_ssl_decryption_enabled()
 
 void AppIdSession::check_app_detection_restart()
 {
-    if (get_session_flags(APPID_SESSION_DECRYPTED))
-        return;
-
-    if (!is_ssl_decryption_enabled())
+    if (get_session_flags(APPID_SESSION_DECRYPTED) || !is_ssl_decryption_enabled() )
         return;
 
     AppId serviceAppId = pick_service_app_id();
     bool isSsl = isSslServiceAppId(serviceAppId);
 
-    // A asd could either:
-    // 1. Start of as SSL - captured with isSsl flag, OR
-    // 2. It could start of as a non-SSL asd and later change to SSL. For example, FTP->FTPS.
+    // A session could either:
+    // 1. Start off as SSL - captured with isSsl flag, OR
+    // 2. It could start off as a non-SSL session and later change to SSL. For example, FTP->FTPS.
     //    In this case APPID_SESSION_ENCRYPTED flag is set by the protocol state machine.
     if (get_session_flags(APPID_SESSION_ENCRYPTED) || isSsl)
     {
@@ -376,9 +386,9 @@ void AppIdSession::check_app_detection_restart()
             LogMessage("AppIdDbg %s SSL decryption is available, restarting app Detection\n",
                 session_logging_id);
 
-        // APPID_SESSION_ENCRYPTED is set upon receiving a command which upgrades the asd to
+        // APPID_SESSION_ENCRYPTED is set upon receiving a command which upgrades the session to
         // SSL. Next packet after the command will have encrypted traffic.  In the case of a
-        // asd which starts as SSL, current packet itself is encrypted. Set the special flag
+        // session which starts as SSL, current packet itself is encrypted. Set the special flag
         // APPID_SESSION_APP_REINSPECT_SSL which allows reinspection of this pcaket.
         if (isSsl)
             set_session_flags(APPID_SESSION_APP_REINSPECT_SSL);
@@ -964,7 +974,7 @@ void AppIdSession::stop_rna_service_inspection(Packet* p, int direction)
         service_port = p->ptrs.sp;
     }
 
-    rna_service_state = RNA_STATE_FINISHED;
+    service_disco_state = APPID_DISCO_STATE_FINISHED;
     set_session_flags(APPID_SESSION_SERVICE_DETECTED);
     clear_session_flags(APPID_SESSION_CONTINUE);
 }
diff --git a/src/network_inspectors/appid/appid_session.h b/src/network_inspectors/appid/appid_session.h
index c93eea330..9c38142c3 100644
--- a/src/network_inspectors/appid/appid_session.h
+++ b/src/network_inspectors/appid/appid_session.h
@@ -78,12 +78,12 @@ enum AppIdFlowStatusCodes
 #define MIN_SFTP_PACKET_COUNT   30
 #define MAX_SFTP_PACKET_COUNT   55
 
-enum RNA_INSPECTION_STATE
+enum APPID_DISCOVERY_STATE
 {
-    RNA_STATE_NONE = 0,
-    RNA_STATE_DIRECT,
-    RNA_STATE_STATEFUL,
-    RNA_STATE_FINISHED
+    APPID_DISCO_STATE_NONE = 0,
+    APPID_DISCO_STATE_DIRECT,
+    APPID_DISCO_STATE_STATEFUL,
+    APPID_DISCO_STATE_FINISHED
 };
 
 enum APPID_SESSION_DIRECTION
@@ -237,7 +237,8 @@ public:
     uint8_t previous_tcp_flags = 0;
 
     // AppId matching service side
-    RNA_INSPECTION_STATE rna_service_state = RNA_STATE_NONE;
+    APPID_DISCOVERY_STATE service_disco_state = APPID_DISCO_STATE_NONE;
+    SESSION_SERVICE_ID_STATE service_search_state = START;
     AppId serviceAppId = APP_ID_NONE;
     AppId portServiceAppId = APP_ID_NONE;
     ServiceDetector* service_detector = nullptr;
@@ -246,17 +247,15 @@ public:
     RNAServiceSubtype* subtype = nullptr;
     char* netbios_name = nullptr;
     std::vector<ServiceDetector*> service_candidates;
-    unsigned int num_candidate_services_tried = 0;
     bool got_incompatible_services = false;
 
     // AppId matching client side
-    RNA_INSPECTION_STATE rna_client_state = RNA_STATE_NONE;
+    APPID_DISCOVERY_STATE client_disco_state = APPID_DISCO_STATE_NONE;
     AppId client_app_id = APP_ID_NONE;
     AppId client_service_app_id = APP_ID_NONE;
     char* client_version = nullptr;
     ClientDetector* client_detector = nullptr;
     std::map<std::string, ClientDetector*> client_candidates;
-    unsigned int num_candidate_clients_tried = 0;
     bool tried_reverse_service = false;
 
     // AppId matching payload
@@ -332,11 +331,15 @@ public:
         common.flags &= ~flags;
     }
 
-    inline uint64_t get_session_flags(uint64_t flags)
+    uint64_t get_session_flags(uint64_t flags)
     {
         return (common.flags & flags);
     }
 
+    bool is_decrypted()
+    {
+       return get_session_flags(APPID_SESSION_DECRYPTED) == APPID_SESSION_DECRYPTED;
+    }
     char session_logging_id[MAX_SESSION_LOGGING_ID_LEN];
     bool session_logging_enabled = false;
 
diff --git a/src/network_inspectors/appid/client_plugins/client_discovery.cc b/src/network_inspectors/appid/client_plugins/client_discovery.cc
index e65cbd9a7..73d43ff8a 100644
--- a/src/network_inspectors/appid/client_plugins/client_discovery.cc
+++ b/src/network_inspectors/appid/client_plugins/client_discovery.cc
@@ -270,7 +270,7 @@ int get_detector_candidates_list(Packet* p, int direction, AppIdSession* asd)
         if (!asd->get_session_flags(APPID_SESSION_CLIENT_DETECTED))
             create_detector_candidates_list(p, direction, asd);
     }
-    else if ( asd->rna_service_state != RNA_STATE_STATEFUL
+    else if ( asd->service_disco_state != APPID_DISCO_STATE_STATEFUL
         && asd->get_session_flags(APPID_SESSION_CLIENT_GETS_SERVER_PACKETS))
         create_detector_candidates_list(p, direction, asd);
 
@@ -321,19 +321,19 @@ bool ClientDiscovery::do_client_discovery(AppIdSession& asd, int direction, Pack
     bool isTpAppidDiscoveryDone = false;
     AppInfoTableEntry* entry;
 
-    if (asd.rna_client_state != RNA_STATE_FINISHED)
+    if (asd.client_disco_state != APPID_DISCO_STATE_FINISHED)
     {
         Profile clientMatchPerfStats_profile_context(clientMatchPerfStats);
-        uint32_t prevRnaClientState = asd.rna_client_state;
+        uint32_t prevRnaClientState = asd.client_disco_state;
         bool was_http2 = asd.is_http2;
         bool was_service = asd.get_session_flags(APPID_SESSION_SERVICE_DETECTED) ? true : false;
         //decision to directly call validator or go through elaborate service_state tracking
         //is made once at the beginning of sesssion.
-        if (asd.rna_client_state == RNA_STATE_NONE && p->dsize && direction ==
+        if (asd.client_disco_state == APPID_DISCO_STATE_NONE && p->dsize && direction ==
             APP_ID_FROM_INITIATOR)
         {
             if (p->flow->get_session_flags() & SSNFLAG_MIDSTREAM)
-                asd.rna_client_state = RNA_STATE_FINISHED;
+                asd.client_disco_state = APPID_DISCO_STATE_FINISHED;
             else if (is_third_party_appid_available(asd.tpsession) && ( asd.tp_app_id >
                 APP_ID_NONE )
                 && ( asd.tp_app_id < SF_APPID_MAX ) )
@@ -347,36 +347,37 @@ bool ClientDiscovery::do_client_discovery(AppIdSession& asd, int direction, Pack
                     //tp has positively identified appId, Dig deeper only if sourcefire
                     // detector identifies additional information
                     asd.client_detector = entry->client_detector;
-                    asd.rna_client_state = RNA_STATE_DIRECT;
+                    asd.client_disco_state = APPID_DISCO_STATE_DIRECT;
                 }
                 else
                 {
                     asd.set_session_flags(APPID_SESSION_CLIENT_DETECTED);
-                    asd.rna_client_state = RNA_STATE_FINISHED;
+                    asd.client_disco_state = APPID_DISCO_STATE_FINISHED;
                 }
             }
             else if (asd.get_session_flags(APPID_SESSION_HTTP_SESSION))
-                asd.rna_client_state = RNA_STATE_FINISHED;
+                asd.client_disco_state = APPID_DISCO_STATE_FINISHED;
             else
-                asd.rna_client_state = RNA_STATE_STATEFUL;
+                asd.client_disco_state = APPID_DISCO_STATE_STATEFUL;
         }
         //stop rna inspection as soon as tp has classified a valid AppId later in the session
-        if ((asd.rna_client_state == RNA_STATE_STATEFUL || asd.rna_client_state ==
-            RNA_STATE_DIRECT)
-            && asd.rna_client_state == prevRnaClientState && !asd.get_session_flags(
-            APPID_SESSION_NO_TPI)
-            && is_third_party_appid_available(asd.tpsession) && asd.tp_app_id > APP_ID_NONE &&
-            asd.tp_app_id < SF_APPID_MAX)
+        if ( (asd.client_disco_state == APPID_DISCO_STATE_STATEFUL ||
+              asd.client_disco_state == APPID_DISCO_STATE_DIRECT) &&
+              asd.client_disco_state == prevRnaClientState &&
+              !asd.get_session_flags( APPID_SESSION_NO_TPI)  &&
+              is_third_party_appid_available(asd.tpsession) &&
+              asd.tp_app_id > APP_ID_NONE && asd.tp_app_id < SF_APPID_MAX)
         {
             entry = asd.app_info_mgr->get_app_info_entry(asd.tp_app_id);
             if (!(entry && entry->client_detector && entry->client_detector == asd.client_detector
                 && (entry->flags & (APPINFO_FLAG_CLIENT_ADDITIONAL | APPINFO_FLAG_CLIENT_USER))))
             {
-                asd.rna_client_state = RNA_STATE_FINISHED;
+                asd.client_disco_state = APPID_DISCO_STATE_FINISHED;
                 asd.set_session_flags(APPID_SESSION_CLIENT_DETECTED);
             }
         }
-        if (asd.rna_client_state == RNA_STATE_DIRECT)
+
+        if (asd.client_disco_state == APPID_DISCO_STATE_DIRECT)
         {
             int ret = APPID_INPROCESS;
             if (direction == APP_ID_FROM_INITIATOR)
@@ -387,7 +388,7 @@ bool ClientDiscovery::do_client_discovery(AppIdSession& asd, int direction, Pack
                     ret = exec_client_detectors(asd, p, direction);
                 }
             }
-            else if (asd.rna_service_state != RNA_STATE_STATEFUL
+            else if (asd.service_disco_state != APPID_DISCO_STATE_STATEFUL
                 && asd.get_session_flags(APPID_SESSION_CLIENT_GETS_SERVER_PACKETS))
             {
                 ret = exec_client_detectors(asd, p, direction);
@@ -398,11 +399,11 @@ bool ClientDiscovery::do_client_discovery(AppIdSession& asd, int direction, Pack
             case APPID_INPROCESS:
                 break;
             default:
-                asd.rna_client_state = RNA_STATE_FINISHED;
+                asd.client_disco_state = APPID_DISCO_STATE_FINISHED;
                 break;
             }
         }
-        else if (asd.rna_client_state == RNA_STATE_STATEFUL)
+        else if (asd.client_disco_state == APPID_DISCO_STATE_STATEFUL)
         {
             get_detector_candidates_list(p, direction, &asd);
             isTpAppidDiscoveryDone = true;
@@ -415,7 +416,7 @@ bool ClientDiscovery::do_client_discovery(AppIdSession& asd, int direction, Pack
                     if (!asd.get_session_flags(APPID_SESSION_CLIENT_DETECTED))
                         ret = exec_client_detectors(asd, p, direction);
                 }
-                else if (asd.rna_service_state != RNA_STATE_STATEFUL
+                else if (asd.service_disco_state != APPID_DISCO_STATE_STATEFUL
                     && asd.get_session_flags(APPID_SESSION_CLIENT_GETS_SERVER_PACKETS))
                     ret = exec_client_detectors(asd, p, direction);
 
diff --git a/src/network_inspectors/appid/detector_plugins/detector_http.cc b/src/network_inspectors/appid/detector_plugins/detector_http.cc
index e6ba6b63a..90adefe6f 100644
--- a/src/network_inspectors/appid/detector_plugins/detector_http.cc
+++ b/src/network_inspectors/appid/detector_plugins/detector_http.cc
@@ -66,10 +66,10 @@ HttpClientDetector::~HttpClientDetector()
 int HttpClientDetector::validate(AppIdDiscoveryArgs& args)
 {
     add_app(args.asd, APP_ID_HTTP, APP_ID_HTTP + GENERIC_APP_OFFSET, nullptr);
-    args.asd->rna_client_state = RNA_STATE_FINISHED;
+    args.asd->client_disco_state = APPID_DISCO_STATE_FINISHED;
     http_service_detector->add_service(args.asd, args.pkt, args.dir, APP_ID_HTTP, nullptr, nullptr,
         nullptr);
-    args.asd->rna_service_state = RNA_STATE_FINISHED;
+    args.asd->service_disco_state = APPID_DISCO_STATE_FINISHED;
     args.asd->set_session_flags(APPID_SESSION_CLIENT_DETECTED | APPID_SESSION_SERVICE_DETECTED);
     args.asd->clear_session_flags(APPID_SESSION_CONTINUE);
     args.asd->is_http2 = true;
@@ -84,7 +84,6 @@ HttpServiceDetector::HttpServiceDetector(ServiceDiscovery* sd)
     handler = sd;
     name = "HTTP";
     proto = IpProtocol::TCP;
-    current_ref_count = 1;
     detectorType = DETECTOR_TYPE_DECODER;
 
     appid_registry =
diff --git a/src/network_inspectors/appid/detector_plugins/detector_imap.cc b/src/network_inspectors/appid/detector_plugins/detector_imap.cc
index fa0098eed..ed44d58ff 100644
--- a/src/network_inspectors/appid/detector_plugins/detector_imap.cc
+++ b/src/network_inspectors/appid/detector_plugins/detector_imap.cc
@@ -820,7 +820,6 @@ ImapServiceDetector::ImapServiceDetector(ServiceDiscovery* sd)
     proto = IpProtocol::TCP;
     provides_user = true;
     detectorType = DETECTOR_TYPE_DECODER;
-    current_ref_count = 1;
 
     tcp_patterns =
     {
diff --git a/src/network_inspectors/appid/detector_plugins/detector_kerberos.cc b/src/network_inspectors/appid/detector_plugins/detector_kerberos.cc
index 3b87f96a4..e4d0d3450 100644
--- a/src/network_inspectors/appid/detector_plugins/detector_kerberos.cc
+++ b/src/network_inspectors/appid/detector_plugins/detector_kerberos.cc
@@ -457,7 +457,6 @@ KerberosServiceDetector::KerberosServiceDetector(ServiceDiscovery* sd)
     proto = IpProtocol::TCP;
     provides_user = true;
     detectorType = DETECTOR_TYPE_DECODER;
-    current_ref_count = 1;
 
     tcp_patterns =
     {
diff --git a/src/network_inspectors/appid/detector_plugins/detector_pattern.cc b/src/network_inspectors/appid/detector_plugins/detector_pattern.cc
index d6c430ba9..85dec27a7 100644
--- a/src/network_inspectors/appid/detector_plugins/detector_pattern.cc
+++ b/src/network_inspectors/appid/detector_plugins/detector_pattern.cc
@@ -430,7 +430,6 @@ PatternServiceDetector::PatternServiceDetector(ServiceDiscovery* sd)
     name = "pattern";
     proto = IpProtocol::TCP;
     detectorType = DETECTOR_TYPE_PATTERN;
-    current_ref_count = 1;
 
     service_pattern_detector = this;
     handler->register_detector(name, this, proto);
diff --git a/src/network_inspectors/appid/detector_plugins/detector_pop3.cc b/src/network_inspectors/appid/detector_plugins/detector_pop3.cc
index 04d25e64d..ae0262e91 100644
--- a/src/network_inspectors/appid/detector_plugins/detector_pop3.cc
+++ b/src/network_inspectors/appid/detector_plugins/detector_pop3.cc
@@ -757,7 +757,6 @@ Pop3ServiceDetector::Pop3ServiceDetector(ServiceDiscovery* sd)
     proto = IpProtocol::TCP;
     provides_user = true;
     detectorType = DETECTOR_TYPE_DECODER;
-    current_ref_count = 1;
 
     tcp_patterns =
     {
diff --git a/src/network_inspectors/appid/detector_plugins/detector_sip.cc b/src/network_inspectors/appid/detector_plugins/detector_sip.cc
index 75263c134..1794ff146 100644
--- a/src/network_inspectors/appid/detector_plugins/detector_sip.cc
+++ b/src/network_inspectors/appid/detector_plugins/detector_sip.cc
@@ -405,7 +405,6 @@ SipServiceDetector::SipServiceDetector(ServiceDiscovery* sd)
     proto = IpProtocol::TCP;
     detectorType = DETECTOR_TYPE_DECODER;
     provides_user = true;
-    current_ref_count = 1;
 
     tcp_patterns =
     {
diff --git a/src/network_inspectors/appid/detector_plugins/detector_smtp.cc b/src/network_inspectors/appid/detector_plugins/detector_smtp.cc
index 79bf89026..b8231029c 100644
--- a/src/network_inspectors/appid/detector_plugins/detector_smtp.cc
+++ b/src/network_inspectors/appid/detector_plugins/detector_smtp.cc
@@ -654,7 +654,6 @@ SmtpServiceDetector::SmtpServiceDetector(ServiceDiscovery* sd)
     name = "smtp";
     proto = IpProtocol::TCP;
     detectorType = DETECTOR_TYPE_DECODER;
-    current_ref_count = 1;
 
     tcp_patterns =
     {
diff --git a/src/network_inspectors/appid/lua_detector_api.cc b/src/network_inspectors/appid/lua_detector_api.cc
index d54ebbb81..dedf64d72 100644
--- a/src/network_inspectors/appid/lua_detector_api.cc
+++ b/src/network_inspectors/appid/lua_detector_api.cc
@@ -334,7 +334,6 @@ static int service_add_ports(lua_State* L)
         return 1;
     }
 
-    ++ud->ref_count;
     lua_pushnumber(L, 0);
     return 1;
 }
@@ -2236,8 +2235,8 @@ static int create_future_flow(lua_State* L)
         fp->payload_app_id = payload_app_id;
         fp->set_session_flags(APPID_SESSION_SERVICE_DETECTED | APPID_SESSION_NOT_A_SERVICE |
             APPID_SESSION_PORT_SERVICE_DONE);
-        fp->rna_service_state = RNA_STATE_FINISHED;
-        fp->rna_client_state  = RNA_STATE_FINISHED;
+        fp->service_disco_state = APPID_DISCO_STATE_FINISHED;
+        fp->client_disco_state  = APPID_DISCO_STATE_FINISHED;
 
         return 1;
     }
diff --git a/src/network_inspectors/appid/lua_detector_module.cc b/src/network_inspectors/appid/lua_detector_module.cc
index 59f500c0f..456a29b39 100644
--- a/src/network_inspectors/appid/lua_detector_module.cc
+++ b/src/network_inspectors/appid/lua_detector_module.cc
@@ -487,7 +487,6 @@ void LuaDetectorManager::activate_lua_detectors()
                 detector->name.c_str(), lua_tostring(L, -1));
 
         ++num_active_lua_detectors;
-        detector->current_ref_count = detector->ref_count;
     }
 
     lua_tracker_size = compute_lua_tracker_size(MAX_MEMORY_FOR_LUA_DETECTORS,
diff --git a/src/network_inspectors/appid/service_plugins/service_battle_field.cc b/src/network_inspectors/appid/service_plugins/service_battle_field.cc
index 17d89099e..74989cc9f 100644
--- a/src/network_inspectors/appid/service_plugins/service_battle_field.cc
+++ b/src/network_inspectors/appid/service_plugins/service_battle_field.cc
@@ -59,7 +59,6 @@ BattleFieldServiceDetector::BattleFieldServiceDetector(ServiceDiscovery* sd)
     name = "BattleField";
     proto = IpProtocol::TCP;
     detectorType = DETECTOR_TYPE_DECODER;
-    current_ref_count =  1;
 
     tcp_patterns =
     {
diff --git a/src/network_inspectors/appid/service_plugins/service_bgp.cc b/src/network_inspectors/appid/service_plugins/service_bgp.cc
index 3c7f47e29..25c000b6c 100644
--- a/src/network_inspectors/appid/service_plugins/service_bgp.cc
+++ b/src/network_inspectors/appid/service_plugins/service_bgp.cc
@@ -99,7 +99,6 @@ BgpServiceDetector::BgpServiceDetector(ServiceDiscovery* sd)
     name = "bgp";
     proto = IpProtocol::TCP;
     detectorType = DETECTOR_TYPE_DECODER;
-    current_ref_count =  1;
 
     tcp_patterns =
     {
diff --git a/src/network_inspectors/appid/service_plugins/service_bit.cc b/src/network_inspectors/appid/service_plugins/service_bit.cc
index bfa35db84..83ecb6292 100644
--- a/src/network_inspectors/appid/service_plugins/service_bit.cc
+++ b/src/network_inspectors/appid/service_plugins/service_bit.cc
@@ -71,7 +71,6 @@ BitServiceDetector::BitServiceDetector(ServiceDiscovery* sd)
     name = "bit";
     proto = IpProtocol::TCP;
     detectorType = DETECTOR_TYPE_DECODER;
-    current_ref_count =  1;
 
     tcp_patterns =
     {
diff --git a/src/network_inspectors/appid/service_plugins/service_bootp.cc b/src/network_inspectors/appid/service_plugins/service_bootp.cc
index 939e2e7ae..48415b5bf 100644
--- a/src/network_inspectors/appid/service_plugins/service_bootp.cc
+++ b/src/network_inspectors/appid/service_plugins/service_bootp.cc
@@ -81,7 +81,6 @@ BootpServiceDetector::BootpServiceDetector(ServiceDiscovery* sd)
     name = "bootp";
     proto = IpProtocol::TCP;
     detectorType = DETECTOR_TYPE_DECODER;
-    current_ref_count =  1;
 
     appid_registry =
     {
@@ -214,13 +213,9 @@ int BootpServiceDetector::validate(AppIdDiscoveryArgs& args)
         goto fail;
 
     if (dir == APP_ID_FROM_INITIATOR)
-    {
         asd->set_session_flags(APPID_SESSION_UDP_REVERSED);
-    }
     else
-    {
         asd->clear_session_flags(APPID_SESSION_UDP_REVERSED);
-    }
 
     if (size > sizeof(ServiceBOOTPHeader) + 4)
     {
diff --git a/src/network_inspectors/appid/service_plugins/service_dcerpc.cc b/src/network_inspectors/appid/service_plugins/service_dcerpc.cc
index a204c8efd..c056b2382 100644
--- a/src/network_inspectors/appid/service_plugins/service_dcerpc.cc
+++ b/src/network_inspectors/appid/service_plugins/service_dcerpc.cc
@@ -44,7 +44,6 @@ DceRpcServiceDetector::DceRpcServiceDetector(ServiceDiscovery* sd)
     name = "dcerpc";
     proto = IpProtocol::TCP;
     detectorType = DETECTOR_TYPE_DECODER;
-    current_ref_count =  1;
 
     appid_registry =
     {
diff --git a/src/network_inspectors/appid/service_plugins/service_detector.cc b/src/network_inspectors/appid/service_plugins/service_detector.cc
index d16bb0f01..f0372d63a 100644
--- a/src/network_inspectors/appid/service_plugins/service_detector.cc
+++ b/src/network_inspectors/appid/service_plugins/service_detector.cc
@@ -73,38 +73,12 @@ int ServiceDetector::service_inprocess(AppIdSession* asd, const Packet* pkt, int
         asd->get_session_flags(APPID_SESSION_IGNORE_HOST | APPID_SESSION_UDP_REVERSED))
         return APPID_SUCCESS;
 
-    const SfIp* ip = pkt->ptrs.ip_api.get_src();
-    uint16_t port = asd->service_port ? asd->service_port : pkt->ptrs.sp;
-    ServiceDiscoveryState* id_state = AppIdServiceState::get(ip, asd->protocol, port,
-        get_service_detect_level(asd));
-    if ( !id_state )
+    if (!asd->service_ip.is_set())
     {
-        id_state = AppIdServiceState::add(ip, asd->protocol, port, get_service_detect_level(asd));
-        id_state->state = SERVICE_ID_NEW;
-        id_state->service = this;
-        asd->service_ip = *ip;
-        asd->service_port = port;
+        asd->service_ip = *(pkt->ptrs.ip_api.get_src());
+        if (!asd->service_port)
+            asd->service_port = pkt->ptrs.sp;
     }
-    else
-    {
-        if (!asd->service_ip.is_set())
-        {
-            asd->service_ip = *(pkt->ptrs.ip_api.get_src());
-            if (!asd->service_port)
-                asd->service_port = pkt->ptrs.sp;
-        }
-    }
-
-    APPID_LOG_IP_FILTER_PORTS(pkt->ptrs.dp, pkt->ptrs.sp,asd->service_ip,
-        "Inprocess: %s:%u:%u %p %d\n", ipstr,
-        (unsigned)asd->protocol, (unsigned)asd->service_port, (void*)id_state,
-        (int)id_state->state);
-
-    APPID_LOG_FILTER_PORTS(pkt->ptrs.dp, pkt->ptrs.sp,
-        "Service for protocol %u on port %u is in process (%u->%u), %s\n",
-        (unsigned)asd->protocol, (unsigned)asd->service_port, (unsigned)pkt->ptrs.sp,
-        (unsigned)pkt->ptrs.dp,
-        svc_element->name ? svc_element->name : "UNKNOWN");
 
     return APPID_SUCCESS;
 }
@@ -112,7 +86,6 @@ int ServiceDetector::service_inprocess(AppIdSession* asd, const Packet* pkt, int
 int ServiceDetector::add_service(AppIdSession* asd, const Packet* pkt, int dir, AppId appId,
     const char* vendor, const char* version)
 {
-    ServiceDiscoveryState* id_state = nullptr;
     uint16_t port = 0;
     const SfIp* ip = nullptr;
 
@@ -165,64 +138,13 @@ int ServiceDetector::add_service(AppIdSession* asd, const Packet* pkt, int dir,
         }
     }
 
-    // If UDP reversed, ensure we have the correct host tracker entry.
-    if (asd->get_session_flags(APPID_SESSION_UDP_REVERSED))
-        id_state = AppIdServiceState::get(ip, asd->protocol, port, get_service_detect_level(asd));
-
-    if ( !id_state )
-    {
-        id_state = AppIdServiceState::add(ip, asd->protocol, port, get_service_detect_level(asd));
-        asd->service_ip = *ip;
-        asd->service_port = port;
-    }
-    else
-    {
-        if (!asd->service_ip.is_set())
-        {
-            asd->service_ip = *ip;
-            asd->service_port = port;
-        }
-
-        APPID_LOG_FILTER_PORTS(pkt->ptrs.dp, pkt->ptrs.sp,
-            "Service %d for protocol %u on port %u (%u->%u) is valid\n",
-            (int)appId, (unsigned)asd->protocol, (unsigned)asd->service_port,
-            (unsigned)pkt->ptrs.sp, (unsigned)pkt->ptrs.dp);
-    }
-    id_state->reset_time = 0;
-    if (id_state->state != SERVICE_ID_VALID)
-    {
-        id_state->state = SERVICE_ID_VALID;
-        id_state->valid_count = 0;
-        id_state->detract_count = 0;
-        id_state->last_detract.clear();
-        id_state->invalid_client_count = 0;
-        id_state->last_invalid_client.clear();
-    }
-    id_state->service = this;
-
-    APPID_LOG_IP_FILTER_PORTS(pkt->ptrs.dp, pkt->ptrs.sp, asd->service_ip,
-        "Valid: %s:%u:%u %p %d\n",
-        ipstr, (unsigned)asd->protocol, (unsigned)asd->service_port,
-        (void*)id_state, (int)id_state->state);
-
-    if (!id_state->valid_count)
-    {
-        id_state->valid_count++;
-        id_state->invalid_client_count = 0;
-        id_state->last_invalid_client.clear();
-        id_state->detract_count = 0;
-        id_state->last_detract.clear();
-    }
-    else if (id_state->valid_count < STATE_ID_MAX_VALID_COUNT)
-        id_state->valid_count++;
-
-    /* Done looking for this session. */
-    id_state->searching = false;
+    asd->service_ip = *ip;
+    asd->service_port = port;
+    ServiceDiscoveryState* sds = AppIdServiceState::get(ip, asd->protocol, port, asd->is_decrypted());
+    if ( !sds )
+       sds = AppIdServiceState::add(ip, asd->protocol, port, asd->is_decrypted());
+    sds->set_service_id_valid(this);
 
-    APPID_LOG_FILTER_PORTS(pkt->ptrs.dp, pkt->ptrs.sp,
-        "Service %d for protocol %u on port %u (%u->%u) is valid\n",
-        (int)appId, (unsigned)asd->protocol, (unsigned)asd->service_port,
-        (unsigned)pkt->ptrs.sp, (unsigned)pkt->ptrs.dp);
     return APPID_SUCCESS;
 }
 
@@ -230,14 +152,6 @@ int ServiceDetector::add_service_consume_subtype(AppIdSession* asd, const Packet
     AppId appId, const char* vendor, const char* version, RNAServiceSubtype* subtype)
 {
     asd->subtype = subtype;
-    if (!current_ref_count)
-    {
-        APPID_LOG_FILTER_PORTS(pkt->ptrs.dp, pkt->ptrs.sp,
-            "Service %d for protocol %u on port %u (%u->%u) is valid, but skipped\n",
-            (int)appId, (unsigned)asd->protocol, (unsigned)asd->service_port,
-            (unsigned)pkt->ptrs.sp, (unsigned)pkt->ptrs.dp);
-        return APPID_SUCCESS;
-    }
     return add_service(asd, pkt, dir, appId, vendor, version);
 }
 
@@ -246,15 +160,6 @@ int ServiceDetector::add_service(AppIdSession* asd, const Packet* pkt, int dir,
 {
     RNAServiceSubtype* new_subtype = nullptr;
 
-    if (!current_ref_count)
-    {
-        APPID_LOG_FILTER_PORTS(pkt->ptrs.dp, pkt->ptrs.sp,
-            "Service %d for protocol %u on port %u (%u->%u) is valid, but skipped\n",
-            (int)appId, (unsigned)asd->protocol, (unsigned)asd->service_port,
-            (unsigned)pkt->ptrs.sp, (unsigned)pkt->ptrs.dp);
-        return APPID_SUCCESS;
-    }
-
     for (; subtype; subtype = subtype->next)
     {
         RNAServiceSubtype* tmp_subtype = (RNAServiceSubtype*)snort_calloc(
@@ -296,7 +201,7 @@ void ServiceDetector::PopulateExpectedFlow(AppIdSession* parent,
         APPID_SESSION_INITIATOR_CHECKED |
         APPID_SESSION_DISCOVER_APP |
         APPID_SESSION_DISCOVER_USER));
-    expected->rna_service_state = RNA_STATE_FINISHED;
-    expected->rna_client_state = RNA_STATE_FINISHED;
+    expected->service_disco_state = APPID_DISCO_STATE_FINISHED;
+    expected->client_disco_state = APPID_DISCO_STATE_FINISHED;
 }
 
diff --git a/src/network_inspectors/appid/service_plugins/service_direct_connect.cc b/src/network_inspectors/appid/service_plugins/service_direct_connect.cc
index 770e4991b..55f8eabb2 100644
--- a/src/network_inspectors/appid/service_plugins/service_direct_connect.cc
+++ b/src/network_inspectors/appid/service_plugins/service_direct_connect.cc
@@ -58,7 +58,6 @@ DirectConnectServiceDetector::DirectConnectServiceDetector(ServiceDiscovery* sd)
     name = "DirectConnect";
     proto = IpProtocol::TCP;
     detectorType = DETECTOR_TYPE_DECODER;
-    current_ref_count =  1;
 
     tcp_patterns =
     {
diff --git a/src/network_inspectors/appid/service_plugins/service_discovery.cc b/src/network_inspectors/appid/service_plugins/service_discovery.cc
index f36034f61..c211a116e 100644
--- a/src/network_inspectors/appid/service_plugins/service_discovery.cc
+++ b/src/network_inspectors/appid/service_plugins/service_discovery.cc
@@ -161,10 +161,7 @@ int ServiceDiscovery::add_service_port(AppIdDetector* detector, const ServiceDet
     if (pp.proto == IpProtocol::TCP)
     {
         if (pp.port == 21 && !ftp_service)
-        {
             ftp_service = service;
-            detector->ref_count++;
-        }
 
         tcp_services[ pp.port ].push_back(service);
     }
@@ -182,134 +179,9 @@ int ServiceDiscovery::add_service_port(AppIdDetector* detector, const ServiceDet
         return 0;
     }
 
-    service->ref_count++;
     return 0;
 }
 
-/* Handle some exception cases on failure:
- *  - valid_count: If we have a detector that should be valid, but it keeps
- *    failing, consider restarting the detector search.
- *  - invalid_client_count: If our service detector search had trouble
- *    simply because of unrecognized client data, then consider retrying
- *    the search again. */
-static void HandleFailure(AppIdSession* asd, ServiceDiscoveryState* id_state,
-    const SfIp* client_ip, unsigned timeout)
-{
-    /* If we had a valid detector, check for too many fails.  If so, start
-     * search sequence again. */
-    if (id_state->state == SERVICE_ID_VALID)
-    {
-        /* Too many invalid clients?  If so, count it as an invalid detect. */
-        if (id_state->invalid_client_count >= STATE_ID_INVALID_CLIENT_THRESHOLD)
-        {
-            if (id_state->valid_count <= 1)
-            {
-                id_state->state = SERVICE_ID_NEW;
-                id_state->invalid_client_count = 0;
-                id_state->last_invalid_client.clear();
-                id_state->valid_count = 0;
-                id_state->detract_count = 0;
-                id_state->last_detract.clear();
-            }
-            else
-            {
-                id_state->valid_count--;
-                id_state->last_invalid_client = *client_ip;
-                id_state->invalid_client_count = 0;
-            }
-        }
-        /* Just a plain old fail.  If too many of these happen, start
-         * search process over. */
-        else if (id_state->invalid_client_count == 0)
-        {
-            if (id_state->last_detract.fast_eq6(*client_ip))
-                id_state->detract_count++;
-            else
-                id_state->last_detract = *client_ip;
-
-            if (id_state->detract_count >= STATE_ID_NEEDED_DUPE_DETRACT_COUNT)
-            {
-                if (id_state->valid_count <= 1)
-                {
-                    id_state->state = SERVICE_ID_NEW;
-                    id_state->invalid_client_count = 0;
-                    id_state->last_invalid_client.clear();
-                    id_state->valid_count = 0;
-                    id_state->detract_count = 0;
-                    id_state->last_detract.clear();
-                }
-                else
-                    id_state->valid_count--;
-            }
-        }
-    }
-    /* If we were port/pattern searching and timed out, just restart over next
-     * time. */
-    else if ( timeout && asd->service_candidates.size() )
-    {
-        id_state->state = SERVICE_ID_NEW;
-    }
-    else if ( (asd->service_candidates.size() )
-        && (id_state->state == SERVICE_ID_BRUTE_FORCE) )
-    {
-        /* If we're getting some invalid clients, defer brute force search and keep retrying
-         * port/pattern search until we either find something or until we
-         * just see too many invalid clients. */
-        if (    (id_state->invalid_client_count > 0)
-            && (id_state->invalid_client_count < STATE_ID_INVALID_CLIENT_THRESHOLD) )
-        {
-            id_state->state = SERVICE_ID_NEW;
-        }
-    }
-
-    /* Done looking for this session. */
-    id_state->searching = false;
-}
-
-/**Changes in_process service state to failed state when a flow is terminated.
- *
- * RNA used to repeat the same service detector if the detector remained in process till the flow terminated. Thus RNA
- * got stuck on this one detector and never tried another service detector. This function will treat such a detector
- * as returning incompatibleData when the flow is terminated. The intent here to make RNA try other service detectors but
- * unlike incompatibleData status, we dont want to undermine confidence in the service.
- *
- * @note Packet may be nullptr when this function is called upon session timeout.
- */
-void FailInProcessService(AppIdSession* asd, const AppIdConfig*)
-{
-    if (asd->get_session_flags(APPID_SESSION_SERVICE_DETECTED | APPID_SESSION_UDP_REVERSED))
-        return;
-
-    ServiceDiscoveryState* id_state = AppIdServiceState::get(&asd->service_ip, asd->protocol,
-        asd->service_port, get_service_detect_level(asd));
-
-    APPID_LOG_FILTER_SERVICE_PORT(asd->service_port,
-        "FailInProcess %" PRIx64 ", %08X:%u proto %u\n", asd->common.flags,
-        asd->common.initiator_ip.ip32[3], (unsigned)asd->service_port,
-        (unsigned)asd->protocol);
-
-    if (!id_state || (id_state->service && !id_state->service->current_ref_count))
-        return;
-
-    APPID_LOG_FILTER_SERVICE_PORT(asd->service_port,
-        "FailInProcess: State %s for protocol %u on port %u, count %u, %s\n",
-        service_id_state_name[id_state->state], (unsigned)asd->protocol,
-        (unsigned)asd->service_port, id_state->invalid_client_count,
-        (id_state->service) ? id_state->service->name.c_str() : "UNKNOWN");
-
-    id_state->invalid_client_count += STATE_ID_INCONCLUSIVE_SERVICE_WEIGHT;
-    if (asd->flow->server_ip.fast_eq6(asd->service_ip))
-        HandleFailure(asd, id_state, &asd->flow->client_ip, 1);
-    else
-        HandleFailure(asd, id_state, &asd->flow->server_ip, 1);
-
-    APPID_LOG_FILTER_SERVICE_PORT(asd->service_port,
-        "FailInProcess: Changed State to %s for protocol %u on port %u, count %u, %s\n",
-        service_id_state_name[id_state->state], (unsigned)asd->protocol,
-        (unsigned)asd->service_port, id_state->invalid_client_count,
-        (id_state->service) ? id_state->service->name.c_str() : "UNKNOWN");
-}
-
 static int AppIdPatternPrecedence(const void* a, const void* b)
 {
     const ServiceMatch* sm1 = (ServiceMatch*)a;
@@ -368,33 +240,27 @@ void ServiceDiscovery::match_services_by_pattern(AppIdSession* asd, const Packet
     else
         patterns = udp_patterns;
 
-    if (!patterns)
+    if (patterns)
     {
-        APPID_LOG_SERVICE("Pattern bailing due to no patterns\n");
-        return;
-    }
-
-    /*FRE didn't search */
-    ServiceMatch* match_list = nullptr;
-    patterns->find_all((char*)pkt->data, pkt->dsize, &pattern_match, false, (void*)&match_list);
+        ServiceMatch* match_list = nullptr;
+        patterns->find_all((char*)pkt->data, pkt->dsize, &pattern_match, false, (void*)&match_list);
 
-    std::vector<ServiceMatch*> smOrderedList;
-    for (ServiceMatch* sm = match_list; sm; sm = sm->next)
-        smOrderedList.push_back(sm);
+        std::vector<ServiceMatch*> smOrderedList;
+        for (ServiceMatch* sm = match_list; sm; sm = sm->next)
+            smOrderedList.push_back(sm);
 
-    if (smOrderedList.size() )
-    {
-        std::sort(smOrderedList.begin(), smOrderedList.end(), AppIdPatternPrecedence);
-        for ( auto& sm : smOrderedList )
+        if (smOrderedList.size() )
         {
-            if ( std::find(asd->service_candidates.begin(), asd->service_candidates.end(),
-                sm->service)
-                == asd->service_candidates.end() )
+            std::sort(smOrderedList.begin(), smOrderedList.end(), AppIdPatternPrecedence);
+            for ( auto& sm : smOrderedList )
             {
-                asd->service_candidates.push_back(sm->service);
-                asd->num_candidate_services_tried++;
+                if ( std::find(asd->service_candidates.begin(), asd->service_candidates.end(),
+                    sm->service) == asd->service_candidates.end() )
+                {
+                    asd->service_candidates.push_back(sm->service);
+                }
+                snort_free(sm);
             }
-            snort_free(sm);
         }
     }
 }
@@ -426,47 +292,33 @@ static inline uint16_t sslPortRemap(uint16_t port)
 void ServiceDiscovery::get_port_based_services(IpProtocol protocol, uint16_t port,
     AppIdSession* asd)
 {
-    if (get_service_detect_level(asd) == 1)
+    ServiceDiscovery& sd = ServiceDiscovery::get_instance();
+
+    if ( asd->is_decrypted() )
     {
         unsigned mapped_port = sslPortRemap(port);
         if (mapped_port)
         {
-            if ( ServiceDiscovery::get_instance().tcp_services.find(mapped_port) !=
-                ServiceDiscovery::get_instance().tcp_services.end() )
-            {
-                asd->service_candidates =
-                    ServiceDiscovery::get_instance().tcp_services[mapped_port];
-                asd->num_candidate_services_tried +=
-                    ServiceDiscovery::get_instance().tcp_services[mapped_port].size();
-            }
+            if ( sd.tcp_services.find(mapped_port) != sd.tcp_services.end() )
+                asd->service_candidates = sd.tcp_services[mapped_port];
         }
     }
-    else if (protocol == IpProtocol::TCP)
+    else if ( protocol == IpProtocol::TCP )
     {
-        if ( ServiceDiscovery::get_instance().tcp_services.find(port) !=
-            ServiceDiscovery::get_instance().tcp_services.end() )
-        {
-            asd->service_candidates = ServiceDiscovery::get_instance().tcp_services[port];
-            asd->num_candidate_services_tried +=
-                ServiceDiscovery::get_instance().tcp_services[port].size();
-        }
+        if ( sd.tcp_services.find(port) != sd.tcp_services.end() )
+            asd->service_candidates = sd.tcp_services[port];
     }
     else
     {
-        if ( ServiceDiscovery::get_instance().udp_services.find(port) !=
-            ServiceDiscovery::get_instance().udp_services.end() )
-        {
-            asd->service_candidates = ServiceDiscovery::get_instance().udp_services[port];
-            asd->num_candidate_services_tried +=
-                ServiceDiscovery::get_instance().udp_services[port].size();
-        }
+        if ( sd.udp_services.find(port) != sd.udp_services.end() )
+            asd->service_candidates = sd.udp_services[port];
     }
 }
 
 /* This function should be called to find the next service detector to try when
  * we have not yet found a valid detector in the host tracker.  It will try
  * both port and/or pattern (but not brute force - that should be done outside
- * of this function).  This includes UDP reversed services.  A valid id_state
+ * of this function).  This includes UDP reversed services.  A valid sds
  * (even if just initialized to the NEW state) should exist before calling this
  * function.  The state coming out of this function will reflect the state in
  * which the next detector was found.  If nothing is found, it'll indicate that
@@ -477,25 +329,25 @@ void ServiceDiscovery::get_port_based_services(IpProtocol protocol, uint16_t por
  * through the main port/pattern search (and returning which detector to add
  * next to the list of detectors to try (even if only 1)). */
 void ServiceDiscovery::get_next_service(const Packet* p, const int dir,
-    AppIdSession* asd, ServiceDiscoveryState* id_state)
+    AppIdSession* asd, ServiceDiscoveryState* sds)
 {
     auto proto = asd->protocol;
 
     /* If NEW, just advance onto trying ports. */
-    if (id_state->state == SERVICE_ID_NEW)
+    if (asd->service_search_state == SESSION_SERVICE_ID_STATE::START)
     {
-        id_state->state = SERVICE_ID_PORT;
-        id_state->service   = nullptr;
+        asd->service_search_state = SESSION_SERVICE_ID_STATE::PORT;
+        sds->service   = nullptr;
     }
 
     /* See if there are any port detectors to try.  If not, move onto patterns. */
-    if (id_state->state == SERVICE_ID_PORT)
+    if (asd->service_search_state == SESSION_SERVICE_ID_STATE::PORT)
         get_port_based_services(proto, (uint16_t)((dir ==  APP_ID_FROM_RESPONDER) ? p->ptrs.sp :
             p->ptrs.dp), asd);
 
-    id_state->state = SERVICE_ID_PATTERN;
+    asd->service_search_state = SESSION_SERVICE_ID_STATE::PATTERN;
 
-    if (id_state->state == SERVICE_ID_PATTERN)
+    if (asd->service_search_state == SESSION_SERVICE_ID_STATE::PATTERN)
     {
         /* If we haven't found anything yet, try to see if we get any hits
          * first with UDP reversed services before moving onto pattern matches. */
@@ -505,10 +357,10 @@ void ServiceDiscovery::get_next_service(const Packet* p, const int dir,
                 && (proto == IpProtocol::UDP) && !asd->tried_reverse_service )
             {
                 asd->tried_reverse_service = true;
-                ServiceDiscoveryState* id_state = AppIdServiceState::get(p->ptrs.ip_api.get_src(),
-                    proto, p->ptrs.sp, get_service_detect_level(asd));
-                if ( id_state && id_state->service )
-                    asd->service_candidates.push_back(id_state->service);
+                ServiceDiscoveryState* sds = AppIdServiceState::get(p->ptrs.ip_api.get_src(),
+                    proto, p->ptrs.sp, asd->is_decrypted());
+                if ( sds && sds->service )
+                    asd->service_candidates.push_back(sds->service);
                 else if ( udp_reversed_services[p->ptrs.sp].size() )
                 {
                     asd->service_candidates.insert(asd->service_candidates.end(),
@@ -525,7 +377,7 @@ void ServiceDiscovery::get_next_service(const Packet* p, const int dir,
         else
         {
             match_services_by_pattern(asd, p, proto);
-            id_state->state = SERVICE_ID_BRUTE_FORCE;
+            sds->state = SERVICE_ID_STATE::SEARCHING_BRUTE_FORCE;
             return;
         }
     }
@@ -533,7 +385,7 @@ void ServiceDiscovery::get_next_service(const Packet* p, const int dir,
     /* Don't do anything if it was in VALID or BRUTE FORCE. */
 }
 
-int ServiceDiscovery::AppIdDiscoverService(Packet* p, const int dir, AppIdSession* asd)
+int ServiceDiscovery::identify_service(AppIdSession* asd, Packet* p, int dir)
 {
     const SfIp* ip = nullptr;
     int ret = APPID_NOMATCH;
@@ -561,27 +413,36 @@ int ServiceDiscovery::AppIdDiscoverService(Packet* p, const int dir, AppIdSessio
     }
 
     /* Get host tracker state. */
-    ServiceDiscoveryState* id_state = AppIdServiceState::get(ip, proto, port,
-        get_service_detect_level(asd));
-    if ( !id_state )
-        id_state = AppIdServiceState::add(ip, proto, port, get_service_detect_level(asd));
+    ServiceDiscoveryState* sds = AppIdServiceState::get(ip, proto, port, asd->is_decrypted());
+    if ( !sds )
+        sds = AppIdServiceState::add(ip, proto, port, asd->is_decrypted());
 
-    if ( !asd->service_detector )
+    if( asd->service_search_state == SESSION_SERVICE_ID_STATE::START )
     {
-        /* If a valid service already exists in host tracker, give it a try. */
-        if (id_state->service && (id_state->state == SERVICE_ID_VALID))
-            asd->service_detector = id_state->service;
-
-        /* If we've gotten to brute force, give next detector a try. */
-        else if ((id_state->state == SERVICE_ID_BRUTE_FORCE)
-            && (asd->num_candidate_services_tried == 0)
-            && !id_state->searching )
+        asd->service_search_state = SESSION_SERVICE_ID_STATE::PORT;
+
+        if( sds->state == SERVICE_ID_STATE::FAILED )
         {
-            if (!id_state->brute_force_mgr)
-                id_state->brute_force_mgr = new AppIdDetectorList(proto);
+            fail_service(asd, p, dir, nullptr);
+            return APPID_NOMATCH;
+        }
+        if ( !asd->service_detector )
+        {
+            /* If a valid service already exists in host tracker, give it a try. */
+            if ( sds->state == SERVICE_ID_STATE::VALID )
+                asd->service_detector = sds->service;
+            else if ( ( sds->state == SERVICE_ID_STATE::SEARCHING_BRUTE_FORCE )
+                            && (!asd->service_candidates.size() ) )
+            {
+                if (!sds->brute_force_mgr)
+                    sds->brute_force_mgr = new AppIdDetectorList(proto);
 
-            id_state->service = id_state->brute_force_mgr->next();
-            asd->service_detector = id_state->service;
+                sds->service = sds->brute_force_mgr->next();
+                if( sds->service )
+                    asd->service_detector = sds->service;
+                else
+                    sds->state = SERVICE_ID_STATE::FAILED;
+            }
         }
     }
 
@@ -602,26 +463,11 @@ int ServiceDiscovery::AppIdDiscoverService(Packet* p, const int dir, AppIdSessio
     }
     else
     {
-        if ( !asd->service_candidates.size() )
-        {
-            asd->num_candidate_services_tried = 0;
-
-            /* This is our first time in for this session, and we're about to
-             * search for a service, because we don't have any solid history on
-             * this IP/port yet.  If some other session is also currently
-             * searching on this host tracker entry, reset state here, so that
-             * we can start search over again with this session. */
-            if (id_state->searching)
-                id_state->state = SERVICE_ID_NEW;
-            id_state->searching = true;
-        }
-
         /* See if we've got more detector(s) to add to the candidate list. */
-        if ( (id_state->state == SERVICE_ID_NEW)
-            || (id_state->state == SERVICE_ID_PORT)
-            || ((id_state->state == SERVICE_ID_PATTERN) && (dir == APP_ID_FROM_RESPONDER)) )
+        if ( (asd->service_search_state == PORT)
+            || ((asd->service_search_state == PATTERN) && (dir == APP_ID_FROM_RESPONDER)) )
         {
-            get_next_service(p, dir, asd, id_state);
+            get_next_service(p, dir, asd, sds);
         }
 
         /* Run all of the detectors that we currently have. */
@@ -656,8 +502,7 @@ int ServiceDiscovery::AppIdDiscoverService(Packet* p, const int dir, AppIdSessio
         if (ret != APPID_SUCCESS)
         {
             if ( (asd->service_candidates.size() == 0)
-                && ((asd->num_candidate_services_tried >= MAX_CANDIDATE_SERVICES)
-                || (id_state->state == SERVICE_ID_BRUTE_FORCE) ) )
+                && (sds->state == SERVICE_ID_STATE::SEARCHING_BRUTE_FORCE) )
             {
                 fail_service(asd, p, dir, nullptr);
                 ret = APPID_NOMATCH;
@@ -667,7 +512,7 @@ int ServiceDiscovery::AppIdDiscoverService(Packet* p, const int dir, AppIdSessio
 
     if (asd->service_detector != nullptr)
     {
-        id_state->reset_time = 0;
+        sds->reset_time = 0;
     }
     else if (dir == APP_ID_FROM_RESPONDER)    // bidirectional exchange unknown service
     {
@@ -689,19 +534,19 @@ int ServiceDiscovery::AppIdDiscoverService(Packet* p, const int dir, AppIdSessio
 
         if (asd->got_incompatible_services)
         {
-            if (id_state->invalid_client_count < STATE_ID_INVALID_CLIENT_THRESHOLD)
+            if (sds->invalid_client_count < STATE_ID_INVALID_CLIENT_THRESHOLD)
             {
-                if (id_state->last_invalid_client.fast_equals_raw(*tmp_ip))
-                    id_state->invalid_client_count++;
+                if (sds->last_invalid_client.fast_equals_raw(*tmp_ip))
+                    sds->invalid_client_count++;
                 else
                 {
-                    id_state->invalid_client_count += 3;
-                    id_state->last_invalid_client = *tmp_ip;
+                    sds->invalid_client_count += 3;
+                    sds->last_invalid_client = *tmp_ip;
                 }
             }
         }
 
-        HandleFailure(asd, id_state, tmp_ip, 0);
+        sds->set_service_id_failed(asd, tmp_ip);
     }
 
     return ret;
@@ -720,13 +565,12 @@ bool ServiceDiscovery::do_service_discovery(AppIdSession& asd, IpProtocol protoc
     AppInfoTableEntry* entry = nullptr;
     bool isTpAppidDiscoveryDone = false;
 
-    if (asd.rna_service_state != RNA_STATE_FINISHED)
+    if (asd.service_disco_state != APPID_DISCO_STATE_FINISHED)
     {
         Profile serviceMatchPerfStats_profile_context(serviceMatchPerfStats);
-        uint32_t prevRnaServiceState = asd.rna_service_state;
-        //decision to directly call validator or go through elaborate service_state tracking
-        //is made once at the beginning of sesssion.
-        if (asd.rna_service_state == RNA_STATE_NONE && p->dsize)
+        uint32_t prevRnaServiceState = asd.service_disco_state;
+
+        if (asd.service_disco_state == APPID_DISCO_STATE_NONE && p->dsize)
         {
             if (p->flow->get_session_flags() & SSNFLAG_MIDSTREAM)
             {
@@ -735,20 +579,19 @@ bool ServiceDiscovery::do_service_discovery(AppIdSession& asd, IpProtocol protoc
                     && !(p->ptrs.tcph->is_fin() || p->ptrs.tcph->is_rst()))
                 {
                     asd.set_session_flags(APPID_SESSION_CLIENT_DETECTED |
-                        APPID_SESSION_NOT_A_SERVICE
-                        | APPID_SESSION_SERVICE_DETECTED);
+                        APPID_SESSION_NOT_A_SERVICE | APPID_SESSION_SERVICE_DETECTED);
                     if (!AddFTPServiceState(&asd))
                     {
                         asd.set_session_flags(APPID_SESSION_CONTINUE);
                         if (p->ptrs.dp != 21)
                             asd.set_session_flags(APPID_SESSION_RESPONDER_SEEN);
                     }
-                    asd.rna_service_state = RNA_STATE_STATEFUL;
+                    asd.service_disco_state = APPID_DISCO_STATE_STATEFUL;
                 }
                 else
                 {
                     asd.set_session_flags(APPID_SESSION_MID | APPID_SESSION_SERVICE_DETECTED);
-                    asd.rna_service_state = RNA_STATE_FINISHED;
+                    asd.service_disco_state = APPID_DISCO_STATE_FINISHED;
                 }
             }
             else if (is_third_party_appid_available(asd.tpsession))
@@ -757,31 +600,30 @@ bool ServiceDiscovery::do_service_discovery(AppIdSession& asd, IpProtocol protoc
                 {
                     //tp has positively identified appId, Dig deeper only if sourcefire
                     // detector identifies additional information or flow is UDP reveresed.
-                    if ((entry = asd.app_info_mgr->get_app_info_entry(asd.tp_app_id)) &&
-                        entry->service_detector
-                        && ((entry->flags & APPINFO_FLAG_SERVICE_ADDITIONAL)
-                        || ((entry->flags & APPINFO_FLAG_SERVICE_UDP_REVERSED)
-                        && protocol == IpProtocol::UDP
-                        && asd.get_session_flags(
-                        APPID_SESSION_INITIATOR_MONITORED
-                        | APPID_SESSION_RESPONDER_MONITORED))))
+                    if ( ( entry = asd.app_info_mgr->get_app_info_entry(asd.tp_app_id)) &&
+                           entry->service_detector &&
+                         ( ( entry->flags & APPINFO_FLAG_SERVICE_ADDITIONAL ) ||
+                           ( ( entry->flags & APPINFO_FLAG_SERVICE_UDP_REVERSED ) &&
+                             protocol == IpProtocol::UDP &&
+                             asd.get_session_flags(APPID_SESSION_INITIATOR_MONITORED |
+                                 APPID_SESSION_RESPONDER_MONITORED) ) ) )
                     {
                         asd.free_flow_data_by_mask(APPID_SESSION_DATA_SERVICE_MODSTATE_BIT);
                         asd.service_detector = entry->service_detector;
-                        asd.rna_service_state = RNA_STATE_STATEFUL;
+                        asd.service_disco_state = APPID_DISCO_STATE_STATEFUL;
                     }
                     else
                         asd.stop_rna_service_inspection(p, direction);
                 }
                 else
-                    asd.rna_service_state = RNA_STATE_STATEFUL;
+                    asd.service_disco_state = APPID_DISCO_STATE_STATEFUL;
             }
             else
-                asd.rna_service_state = RNA_STATE_STATEFUL;
+                asd.service_disco_state = APPID_DISCO_STATE_STATEFUL;
         }
         //stop rna inspection as soon as tp has classified a valid AppId later in the session
-        if (asd.rna_service_state == RNA_STATE_STATEFUL &&
-            prevRnaServiceState == RNA_STATE_STATEFUL &&
+        if (asd.service_disco_state == APPID_DISCO_STATE_STATEFUL &&
+            prevRnaServiceState == APPID_DISCO_STATE_STATEFUL &&
             !asd.get_session_flags(APPID_SESSION_NO_TPI) &&
             is_third_party_appid_available(asd.tpsession) &&
             asd.tp_app_id > APP_ID_NONE && asd.tp_app_id < SF_APPID_MAX)
@@ -796,14 +638,14 @@ bool ServiceDiscovery::do_service_discovery(AppIdSession& asd, IpProtocol protoc
             }
         }
 
-        if (asd.rna_service_state == RNA_STATE_STATEFUL)
+        if (asd.service_disco_state == APPID_DISCO_STATE_STATEFUL)
         {
-            AppIdDiscoverService(p, direction, &asd);
+            identify_service(&asd, p, direction);
             isTpAppidDiscoveryDone = true;
             //to stop executing validator after service has been detected by RNA.
             if (asd.get_session_flags(APPID_SESSION_SERVICE_DETECTED |
                 APPID_SESSION_CONTINUE) == APPID_SESSION_SERVICE_DETECTED)
-                asd.rna_service_state = RNA_STATE_FINISHED;
+                asd.service_disco_state = APPID_DISCO_STATE_FINISHED;
 
             if (asd.serviceAppId == APP_ID_DNS && asd.config->mod_config->dns_host_reporting
                 && asd.dsession && asd.dsession->host)
@@ -819,9 +661,8 @@ bool ServiceDiscovery::do_service_discovery(AppIdSession& asd, IpProtocol protoc
                 asd.examine_ssl_metadata(p);
 
             if (asd.tp_app_id <= APP_ID_NONE && asd.get_session_flags(
-                APPID_SESSION_SERVICE_DETECTED |
-                APPID_SESSION_NOT_A_SERVICE | APPID_SESSION_IGNORE_HOST) ==
-                APPID_SESSION_SERVICE_DETECTED)
+                APPID_SESSION_SERVICE_DETECTED | APPID_SESSION_NOT_A_SERVICE |
+                APPID_SESSION_IGNORE_HOST) == APPID_SESSION_SERVICE_DETECTED)
             {
                 asd.sync_with_snort_id(asd.serviceAppId, p);
             }
@@ -842,76 +683,35 @@ int ServiceDiscovery::incompatible_data(AppIdSession* asd, const Packet* pkt, in
 {
     const SfIp* ip = pkt->ptrs.ip_api.get_src();
     uint16_t port = asd->service_port ? asd->service_port : pkt->ptrs.sp;
-    ServiceDiscoveryState* id_state = AppIdServiceState::get(ip, asd->protocol, port,
-        get_service_detect_level(asd));
+    ServiceDiscoveryState* sds = AppIdServiceState::get(ip, asd->protocol, port,
+        asd->is_decrypted());
 
-    if ( service )
-        asd->free_flow_data_by_id(service->flow_data_index);
+    asd->free_flow_data_by_id(service->flow_data_index);
 
-    /* If we're still working on a port/pattern list of detectors, then ignore
-     * individual fails until we're done looking at everything. */
-    if ( (asd->service_detector == nullptr) && (id_state != nullptr) )
-    {
-        if (asd->service_candidates.size())
-            return APPID_SUCCESS;
-        else if ((asd->num_candidate_services_tried >= MAX_CANDIDATE_SERVICES)
-            || (id_state->state == SERVICE_ID_BRUTE_FORCE) )
-            return APPID_SUCCESS;
-    }
+    // ignore fails while searching with port/pattern selected detectors
+    if ( !asd->service_detector && asd->service_candidates.size() )
+        return APPID_SUCCESS;
 
     asd->set_session_flags(APPID_SESSION_SERVICE_DETECTED);
     asd->clear_session_flags(APPID_SESSION_CONTINUE);
-
     asd->serviceAppId = APP_ID_NONE;
 
-    if ( asd->get_session_flags(APPID_SESSION_IGNORE_HOST | APPID_SESSION_UDP_REVERSED)
-        || (service && !service->current_ref_count) )
+    if ( asd->get_session_flags(APPID_SESSION_IGNORE_HOST | APPID_SESSION_UDP_REVERSED ) )
         return APPID_SUCCESS;
 
-    if (dir == APP_ID_FROM_INITIATOR)
+    if ( dir == APP_ID_FROM_INITIATOR )
     {
         asd->set_session_flags(APPID_SESSION_INCOMPATIBLE);
         return APPID_SUCCESS;
     }
 
-    if ( !id_state )
+    if (!asd->service_ip.is_set())
     {
-        id_state = AppIdServiceState::add(ip, asd->protocol, port, get_service_detect_level(asd));
-        id_state->state = SERVICE_ID_NEW;
-        id_state->service = service;
         asd->service_ip = *ip;
-        asd->service_port = port;
+        if (!asd->service_port)
+            asd->service_port = port;
     }
-    else
-    {
-        if (!asd->service_ip.is_set())
-        {
-            asd->service_ip = *(pkt->ptrs.ip_api.get_src());
-            if (!asd->service_port)
-                asd->service_port = pkt->ptrs.sp;
-
-            APPID_LOG_FILTER_PORTS(pkt->ptrs.dp, pkt->ptrs.sp,
-                "service_IC: Changed State to %s for protocol %u on port %u (%u->%u), count %u, %s\n",
-                service_id_state_name[id_state->state], (unsigned)asd->protocol,
-                (unsigned)asd->service_port,
-                (unsigned)pkt->ptrs.sp, (unsigned)pkt->ptrs.dp, id_state->invalid_client_count,
-                (id_state->svc && id_state->svc->name) ? id_state->svc->name : "UNKNOWN");
-        }
-        id_state->reset_time = 0;
-    }
-
-    APPID_LOG_FILTER_PORTS(pkt->ptrs.dp, pkt->ptrs.sp,
-        "service_IC: State %s for protocol %u on port %u (%u->%u), count %u, %s\n",
-        service_id_state_name[id_state->state], (unsigned)asd->protocol,
-        (unsigned)asd->service_port,
-        (unsigned)pkt->ptrs.sp, (unsigned)pkt->ptrs.dp, id_state->invalid_client_count,
-        (id_state->svc && id_state->svc->name) ? id_state->svc->name : "UNKNOWN");
-
-    APPID_LOG_IP_FILTER_PORTS(pkt->ptrs.dp, pkt->ptrs.sp, asd->service_ip,
-        "Incompat: %s:%u:%u %p %d %s\n",
-        ipstr, (unsigned)asd->protocol, (unsigned)asd->service_port, (void*)id_state,
-        (int)id_state->state,
-        (id_state->svc && id_state->svc->name) ? id_state->svc->name : "UNKNOWN");
+    sds->reset_time = 0;
 
     return APPID_SUCCESS;
 }
@@ -921,22 +721,12 @@ int ServiceDiscovery::fail_service(AppIdSession* asd, const Packet* pkt, int dir
 {
     const SfIp* ip = pkt->ptrs.ip_api.get_src();
     uint16_t port = asd->service_port ? asd->service_port : pkt->ptrs.sp;
-    ServiceDiscoveryState* id_state = AppIdServiceState::get(ip, asd->protocol, port,
-        get_service_detect_level(asd));
 
     if ( service )
         asd->free_flow_data_by_id(service->flow_data_index);
 
-    /* If we're still working on a port/pattern list of detectors, then ignore
-     * individual fails until we're done looking at everything. */
-    if ( (asd->service_detector == nullptr) && (id_state != nullptr) )
-    {
-        if ( asd->service_candidates.size() )
-            return APPID_SUCCESS;
-        else if ( (asd->num_candidate_services_tried >= MAX_CANDIDATE_SERVICES)
-            || (id_state->state == SERVICE_ID_BRUTE_FORCE) )
+    if ( !asd->service_detector && asd->service_candidates.size() )
             return APPID_SUCCESS;
-    }
 
     asd->serviceAppId = APP_ID_NONE;
     asd->set_session_flags(APPID_SESSION_SERVICE_DETECTED);
@@ -945,8 +735,7 @@ int ServiceDiscovery::fail_service(AppIdSession* asd, const Packet* pkt, int dir
     /* detectors should be careful in marking session UDP_REVERSED otherwise the same detector
      * gets all future flows. UDP_REVERSE should be marked only when detector positively
      * matches opposite direction patterns. */
-    if ( asd->get_session_flags(APPID_SESSION_IGNORE_HOST | APPID_SESSION_UDP_REVERSED)
-        || (service && !service->current_ref_count) )
+    if ( asd->get_session_flags(APPID_SESSION_IGNORE_HOST | APPID_SESSION_UDP_REVERSED) )
         return APPID_SUCCESS;
 
     /* For subsequent packets, avoid marking service failed on client packet,
@@ -957,35 +746,21 @@ int ServiceDiscovery::fail_service(AppIdSession* asd, const Packet* pkt, int dir
         return APPID_SUCCESS;
     }
 
-    if ( !id_state )
+    if (!asd->service_ip.is_set())
     {
-        id_state = AppIdServiceState::add(ip, asd->protocol, port, get_service_detect_level(asd));
-        id_state->state = SERVICE_ID_NEW;
-        id_state->service = service;
         asd->service_ip = *ip;
-        asd->service_port = port;
+        if (!asd->service_port)
+            asd->service_port = port;
     }
-    else
+
+    ServiceDiscoveryState* sds = AppIdServiceState::get(ip, asd->protocol, port,
+        asd->is_decrypted());
+    if ( !sds )
     {
-        if (!asd->service_ip.is_set())
-        {
-            asd->service_ip = *(pkt->ptrs.ip_api.get_src());
-            if (!asd->service_port)
-                asd->service_port = pkt->ptrs.sp;
-        }
+        sds = AppIdServiceState::add(ip, asd->protocol, port, asd->is_decrypted());
+        sds->service = service;
     }
-    id_state->reset_time = 0;
-
-    APPID_LOG_FILTER_PORTS(pkt->ptrs.dp, pkt->ptrs.sp,
-        "service_fail: State %s for protocol %u on port %u (%u->%u), count %u, valid count %u, currSvc %s\n",
-        service_id_state_name[id_state->state], (unsigned)asd->protocol,
-        (unsigned)asd->service_port, (unsigned)pkt->ptrs.sp, (unsigned)pkt->ptrs.dp,
-        id_state->invalid_client_count, id_state->valid_count, name.c_str());
-
-    APPID_LOG_IP_FILTER_PORTS(pkt->ptrs.dp, pkt->ptrs.sp, asd->service_ip,
-        "Fail: %s:%u:%u %p %d %s\n",
-        ipstr, (unsigned)asd->protocol, (unsigned)asd->service_port, (void*)id_state,
-        (int)id_state->state, name.c_str());
+    sds->reset_time = 0;
 
     return APPID_SUCCESS;
 }
diff --git a/src/network_inspectors/appid/service_plugins/service_discovery.h b/src/network_inspectors/appid/service_plugins/service_discovery.h
index ec06fe139..ff423b65e 100644
--- a/src/network_inspectors/appid/service_plugins/service_discovery.h
+++ b/src/network_inspectors/appid/service_plugins/service_discovery.h
@@ -36,9 +36,6 @@ class AppIdSession;
 class ServiceDetector;
 class ServiceDiscoveryState;
 
-#define MAX_CANDIDATE_SERVICES 10
-#define RNA_SERVICE_MAX_PORT 65536
-
 #define STATE_ID_INCONCLUSIVE_SERVICE_WEIGHT 3
 #define STATE_ID_INVALID_CLIENT_THRESHOLD    9
 #define STATE_ID_NEEDED_DUPE_DETRACT_COUNT   3
@@ -49,6 +46,18 @@ enum SERVICE_HOST_INFO_CODE
     SERVICE_HOST_INFO_NETBIOS_NAME = 1
 };
 
+/* Service state stored per flow, which acts based on global SERVICE_ID_STATE
+ * at the beginning of the flow, then independently do service discovery, and
+ * synchronize findings at the end of service discovery by the flow.
+ */
+enum SESSION_SERVICE_ID_STATE
+{
+    START = 0,
+    PORT,
+    PATTERN,
+    PENDING
+};
+
 void FailInProcessService(AppIdSession*, const AppIdConfig*);
 int AddFTPServiceState(AppIdSession*);
 
@@ -66,7 +75,7 @@ public:
     ServiceDetector* get_next_udp_detector(AppIdDetectorsIterator&);
 
     bool do_service_discovery(AppIdSession&, IpProtocol, int, AppId, AppId,  Packet*);
-    int AppIdDiscoverService(Packet*, const int dir, AppIdSession*);
+    int identify_service(AppIdSession*, Packet*, int dir);
     int fail_service(AppIdSession*, const Packet*, int dir, ServiceDetector*);
     int incompatible_data(AppIdSession*, const Packet*, int dir, ServiceDetector*);
 
@@ -82,84 +91,4 @@ private:
     void match_services_by_pattern(AppIdSession*, const Packet*, IpProtocol);
 };
 
-class AppIdDetectorList
-{
-public:
-    AppIdDetectorList(IpProtocol proto)
-    {
-        if (proto == IpProtocol::TCP)
-            detectors = &ServiceDiscovery::get_instance().tcp_detectors;
-        else
-            detectors = &ServiceDiscovery::get_instance().udp_detectors;
-        dit = detectors->begin();
-    }
-
-    ServiceDetector* next()
-    {
-        ServiceDetector* detector = nullptr;
-
-        if ( dit != detectors->end())
-            detector = (ServiceDetector*)(dit++)->second;
-        return detector;
-    }
-
-    void reset()
-    {
-        dit = detectors->begin();
-    }
-
-private:
-    AppIdDetectors* detectors;
-    AppIdDetectorsIterator dit;
-};
-
-//#define SERVICE_DEBUG 1
-//#define SERVICE_DEBUG_PORT  80
-
-#ifdef SERVICE_DEBUG
-static const char* service_id_state_name[] =
-{
-    "NEW",
-    "VALID",
-    "PORT",
-    "PATTERN",
-    "BRUTE_FORCE"
-};
-
-#ifdef SERVICE_DEBUG_PORT
-#define APPID_LOG_SERVICE(fmt) fprintf(SF_DEBUG_FILE, fmt)
-#define APPID_LOG_FILTER_PORTS(dp, sp, fmt, ...) \
-    if (dp == SERVICE_DEBUG_PORT || sp == SERVICE_DEBUG_PORT) \
-        fprintf(SF_DEBUG_FILE, fmt, __VA_ARGS__)
-#define APPID_LOG_FILTER_SERVICE_PORT(port, fmt, ...) \
-    if (port == SERVICE_DEBUG_PORT) \
-        fprintf(SF_DEBUG_FILE, fmt, __VA_ARGS__)
-#define APPID_LOG_IP_FILTER_PORTS(dp, sp, ip, fmt, ...) \
-    if (dp == SERVICE_DEBUG_PORT || sp == SERVICE_DEBUG_PORT) \
-    { \
-        char ipstr[INET6_ADDRSTRLEN]; \
-        sfip_ntop(&ip, ipstr, sizeof(ipstr)); \
-        fprintf(SF_DEBUG_FILE, fmt, __VA_ARGS__); \
-    }
-#else
-#define APPID_LOG_SERVICE(fmt) fprintf(SF_DEBUG_FILE, fmt)
-#define APPID_LOG_FILTER_PORTS(dp, sp, fmt, ...) fprintf(SF_DEBUG_FILE, fmt, __VA_ARGS__)
-#define APPID_LOG_FILTER_SERVICE_PORT(port, fmt, ...) \
-    UNUSED(port); \
-    fprintf(SF_DEBUG_FILE, fmt, __VA_ARGS__)
-#define APPID_LOG_IP_FILTER_PORTS(dp, sp, ip, fmt, ...) \
-    { \
-        char ipstr[INET6_ADDRSTRLEN]; \
-        sfip_ntop(&ip, ipstr, sizeof(ipstr)); \
-        fprintf(SF_DEBUG_FILE, fmt, __VA_ARGS__); \
-    }
 #endif
-#else
-#define APPID_LOG_SERVICE(fmt)
-#define APPID_LOG_FILTER_PORTS(dp, sp, fmt, ...)
-#define APPID_LOG_FILTER_SERVICE_PORT(port, fmt, ...) UNUSED(port);
-#define APPID_LOG_IP_FILTER_PORTS(dp, sp, ip, fmt, ...) UNUSED(ip);
-#endif
-
-#endif
-
diff --git a/src/network_inspectors/appid/service_plugins/service_flap.cc b/src/network_inspectors/appid/service_plugins/service_flap.cc
index 4d0f916e6..5f74aaa29 100644
--- a/src/network_inspectors/appid/service_plugins/service_flap.cc
+++ b/src/network_inspectors/appid/service_plugins/service_flap.cc
@@ -82,7 +82,6 @@ FlapServiceDetector::FlapServiceDetector(ServiceDiscovery* sd)
     name = "flap";
     proto = IpProtocol::TCP;
     detectorType = DETECTOR_TYPE_DECODER;
-    current_ref_count =  1;
 
     tcp_patterns =
     {
diff --git a/src/network_inspectors/appid/service_plugins/service_ftp.cc b/src/network_inspectors/appid/service_plugins/service_ftp.cc
index f4cb239e6..c6bdf0e79 100644
--- a/src/network_inspectors/appid/service_plugins/service_ftp.cc
+++ b/src/network_inspectors/appid/service_plugins/service_ftp.cc
@@ -90,7 +90,6 @@ FtpServiceDetector::FtpServiceDetector(ServiceDiscovery* sd)
     name = "ftp";
     proto = IpProtocol::TCP;
     detectorType = DETECTOR_TYPE_DECODER;
-    current_ref_count =  1;
     ftp_data_app_id = add_appid_protocol_reference("ftp-data");
 
     tcp_patterns =
diff --git a/src/network_inspectors/appid/service_plugins/service_irc.cc b/src/network_inspectors/appid/service_plugins/service_irc.cc
index 30aba8c59..5891edd79 100644
--- a/src/network_inspectors/appid/service_plugins/service_irc.cc
+++ b/src/network_inspectors/appid/service_plugins/service_irc.cc
@@ -70,7 +70,6 @@ IrcServiceDetector::IrcServiceDetector(ServiceDiscovery* sd)
     name = "irc";
     proto = IpProtocol::TCP;
     detectorType = DETECTOR_TYPE_DECODER;
-    current_ref_count =  1;
 
     appid_registry =
     {
diff --git a/src/network_inspectors/appid/service_plugins/service_lpr.cc b/src/network_inspectors/appid/service_plugins/service_lpr.cc
index 99322be5a..7d012a631 100644
--- a/src/network_inspectors/appid/service_plugins/service_lpr.cc
+++ b/src/network_inspectors/appid/service_plugins/service_lpr.cc
@@ -67,7 +67,6 @@ LprServiceDetector::LprServiceDetector(ServiceDiscovery* sd)
     name = "lpr";
     proto = IpProtocol::TCP;
     detectorType = DETECTOR_TYPE_DECODER;
-    current_ref_count =  1;
 
     appid_registry =
     {
diff --git a/src/network_inspectors/appid/service_plugins/service_mdns.cc b/src/network_inspectors/appid/service_plugins/service_mdns.cc
index d36c32eba..e1d1e8ad7 100644
--- a/src/network_inspectors/appid/service_plugins/service_mdns.cc
+++ b/src/network_inspectors/appid/service_plugins/service_mdns.cc
@@ -93,7 +93,6 @@ MdnsServiceDetector::MdnsServiceDetector(ServiceDiscovery* sd)
     name = "MDNS";
     proto = IpProtocol::UDP;
     detectorType = DETECTOR_TYPE_DECODER;
-    current_ref_count = 1;
 
     appid_registry =
     {
diff --git a/src/network_inspectors/appid/service_plugins/service_mysql.cc b/src/network_inspectors/appid/service_plugins/service_mysql.cc
index a2ab8d482..788561c06 100644
--- a/src/network_inspectors/appid/service_plugins/service_mysql.cc
+++ b/src/network_inspectors/appid/service_plugins/service_mysql.cc
@@ -52,7 +52,6 @@ MySqlServiceDetector::MySqlServiceDetector(ServiceDiscovery* sd)
     name = "mysql";
     proto = IpProtocol::TCP;
     detectorType = DETECTOR_TYPE_DECODER;
-    current_ref_count =  1;
 
     appid_registry =
     {
diff --git a/src/network_inspectors/appid/service_plugins/service_netbios.cc b/src/network_inspectors/appid/service_plugins/service_netbios.cc
index 1118c9007..476775fbe 100644
--- a/src/network_inspectors/appid/service_plugins/service_netbios.cc
+++ b/src/network_inspectors/appid/service_plugins/service_netbios.cc
@@ -30,8 +30,6 @@
 #include "dcerpc.h"
 #include "protocols/packet.h"
 
-/*#define RNA_DEBUG_NETBIOS   1 */
-
 #define NBSS_PORT   139
 
 #define NBNS_NB 32
@@ -439,7 +437,6 @@ NbnsServiceDetector::NbnsServiceDetector(ServiceDiscovery* sd)
     name = "nbns";
     proto = IpProtocol::TCP;
     detectorType = DETECTOR_TYPE_DECODER;
-    current_ref_count =  1;
 
     appid_registry =
     {
@@ -627,7 +624,7 @@ static inline void smb_domain_skip_string(const uint8_t** data, uint16_t* size,
 }
 
 static inline void smb_find_domain(const uint8_t* data, uint16_t size, const int,
-    AppIdSession* asd, const Packet* pkt)
+    AppIdSession* asd)
 {
     const ServiceSMBHeader* smb;
     const ServiceSMBAndXResponse* resp;
@@ -728,16 +725,7 @@ static inline void smb_find_domain(const uint8_t* data, uint16_t size, const int
                 }
                 data++;
                 if (*data != 0)
-                {
-#ifdef RNA_DEBUG_NETBIOS
-                    _dpd.errMsg("Failed command %02X  %u  0x%08X:%u->0x%08X:%u",
-                        smb->command, byte_count, pkt->src_ip.s_addr, pkt->ptrs.sp,
-                        pkt->dst_ip.s_addr, pkt->ptrs.dp);
-#else
-                    UNUSED(pkt);
-#endif
                     return;
-                }
                 data++;
             }
         }
@@ -746,14 +734,7 @@ static inline void smb_find_domain(const uint8_t* data, uint16_t size, const int
             byte_count--;
         }
         if (byte_count && smb->command != SERVICE_SMB_COMMAND_NEGOTIATE_PROTOCOL)
-        {
-#ifdef RNA_DEBUG_NETBIOS
-            _dpd.errMsg("Failed command %02X  %u  0x%08X:%u->0x%08X:%u",
-                smb->command, byte_count, pkt->src_ip.s_addr, pkt->src_port, pkt->dst_ip.s_addr,
-                pkt->dst_port);
-#endif
             return;
-        }
     }
     else
     {
@@ -777,25 +758,11 @@ static inline void smb_find_domain(const uint8_t* data, uint16_t size, const int
             }
         }
         if (byte_count && smb->command != SERVICE_SMB_COMMAND_NEGOTIATE_PROTOCOL)
-        {
-#ifdef RNA_DEBUG_NETBIOS
-            _dpd.errMsg("Failed command %02X  %u  0x%08X:%u->0x%08X:%u",
-                smb->command, byte_count, pkt->src_ip.s_addr, pkt->src_port, pkt->dst_ip.s_addr,
-                pkt->dst_port);
-#endif
             return;
-        }
-    }
-    if (pos)
-    {
-#ifdef RNA_DEBUG_NETBIOS
-        _dpd.debugMsg(DEBUG_LOG, "Found domain %s for command %02X 0x%08X:%u->0x%08X:%u",
-            domain, smb->command, pkt->src_ip.s_addr, pkt->src_port, pkt->dst_ip.s_addr,
-            pkt->dst_port);
-#endif
-        if (!asd->netbios_domain)
-            asd->netbios_domain = snort_strdup(domain);
     }
+
+    if ( pos && (!asd->netbios_domain) )
+        asd->netbios_domain = snort_strdup(domain);
 }
 
 NbssServiceDetector::NbssServiceDetector(ServiceDiscovery* sd)
@@ -804,7 +771,6 @@ NbssServiceDetector::NbssServiceDetector(ServiceDiscovery* sd)
     name = "nbss";
     proto = IpProtocol::TCP;
     detectorType = DETECTOR_TYPE_DECODER;
-    current_ref_count =  1;
 
     tcp_patterns =
     {
@@ -903,8 +869,7 @@ int NbssServiceDetector::validate(AppIdDiscoveryArgs& args)
                     if (nd->length <= tmp)
                     {
                         smb_find_domain(data + sizeof(NB_SMB_BANNER),
-                            nd->length - sizeof(NB_SMB_BANNER),
-                            dir, asd, pkt);
+                            nd->length - sizeof(NB_SMB_BANNER), dir, asd);
                     }
                 }
                 else if (tmp >= 4 && nd->length >= 4 &&
@@ -966,7 +931,7 @@ int NbssServiceDetector::validate(AppIdDiscoveryArgs& args)
                     }
                     if (nd->length <= tmp)
                     {
-                        smb_find_domain(data + sizeof(NB_SMB_BANNER), nd->length, dir, asd, pkt);
+                        smb_find_domain(data + sizeof(NB_SMB_BANNER), nd->length, dir, asd);
                     }
                 }
                 else if (tmp >= 4 && nd->length >= 4 &&
@@ -1060,7 +1025,6 @@ NbdgmServiceDetector::NbdgmServiceDetector(ServiceDiscovery* sd)
     name = "nbdgm";
     proto = IpProtocol::UDP;
     detectorType = DETECTOR_TYPE_DECODER;
-    current_ref_count =  1;
 
     appid_registry =
     {
diff --git a/src/network_inspectors/appid/service_plugins/service_nntp.cc b/src/network_inspectors/appid/service_plugins/service_nntp.cc
index c4d5084f1..223345533 100644
--- a/src/network_inspectors/appid/service_plugins/service_nntp.cc
+++ b/src/network_inspectors/appid/service_plugins/service_nntp.cc
@@ -70,7 +70,6 @@ NntpServiceDetector::NntpServiceDetector(ServiceDiscovery* sd)
     name = "nntp";
     proto = IpProtocol::TCP;
     detectorType = DETECTOR_TYPE_DECODER;
-    current_ref_count =  1;
 
     tcp_patterns =
     {
diff --git a/src/network_inspectors/appid/service_plugins/service_ntp.cc b/src/network_inspectors/appid/service_plugins/service_ntp.cc
index 1c44e7fa3..402bd1f36 100644
--- a/src/network_inspectors/appid/service_plugins/service_ntp.cc
+++ b/src/network_inspectors/appid/service_plugins/service_ntp.cc
@@ -65,7 +65,6 @@ NtpServiceDetector::NtpServiceDetector(ServiceDiscovery* sd)
     name = "ntp";
     proto = IpProtocol::TCP;
     detectorType = DETECTOR_TYPE_DECODER;
-    current_ref_count =  1;
 
     appid_registry =
     {
diff --git a/src/network_inspectors/appid/service_plugins/service_radius.cc b/src/network_inspectors/appid/service_plugins/service_radius.cc
index 6cdd4ca0b..6e2a68267 100644
--- a/src/network_inspectors/appid/service_plugins/service_radius.cc
+++ b/src/network_inspectors/appid/service_plugins/service_radius.cc
@@ -65,7 +65,6 @@ RadiusServiceDetector::RadiusServiceDetector(ServiceDiscovery* sd)
     name = "radius";
     proto = IpProtocol::UDP;
     detectorType = DETECTOR_TYPE_DECODER;
-    current_ref_count =  1;
 
     appid_registry =
     {
@@ -195,7 +194,6 @@ RadiusAcctServiceDetector::RadiusAcctServiceDetector(ServiceDiscovery* sd)
     name = "radiusacct";
     proto = IpProtocol::UDP;
     detectorType = DETECTOR_TYPE_DECODER;
-    current_ref_count =  1;
 
     appid_registry =
     {
diff --git a/src/network_inspectors/appid/service_plugins/service_rexec.cc b/src/network_inspectors/appid/service_plugins/service_rexec.cc
index 0809410ff..2e297ec92 100644
--- a/src/network_inspectors/appid/service_plugins/service_rexec.cc
+++ b/src/network_inspectors/appid/service_plugins/service_rexec.cc
@@ -59,7 +59,6 @@ RexecServiceDetector::RexecServiceDetector(ServiceDiscovery* sd)
     name = "rexec";
     proto = IpProtocol::TCP;
     detectorType = DETECTOR_TYPE_DECODER;
-    current_ref_count =  1;
 
     app_id = add_appid_protocol_reference("rexec");
 
@@ -160,14 +159,14 @@ int RexecServiceDetector::validate(AppIdDiscoveryArgs& args)
                 data_add(pf, tmp_rd, &rexec_free_state);
                 if (pf->add_flow_data_id((uint16_t)port, this))
                 {
-                    pf->rna_service_state = RNA_STATE_FINISHED;
+                    pf->service_disco_state = APPID_DISCO_STATE_FINISHED;
                     tmp_rd->state = REXEC_STATE_DONE;
                     tmp_rd->parent = nullptr;
                     return APPID_ENULL;
                 }
                 rd->child = tmp_rd;
                 rd->state = REXEC_STATE_SERVER_CONNECT;
-                pf->rna_service_state = RNA_STATE_STATEFUL;
+                pf->service_disco_state = APPID_DISCO_STATE_STATEFUL;
                 pf->scan_flags |= SCAN_HOST_PORT_FLAG;
                 PopulateExpectedFlow(asd, pf,
                     APPID_SESSION_CONTINUE |
@@ -176,7 +175,7 @@ int RexecServiceDetector::validate(AppIdDiscoveryArgs& args)
                     APPID_SESSION_SERVICE_DETECTED |
                     APPID_SESSION_NOT_A_SERVICE |
                     APPID_SESSION_PORT_SERVICE_DONE);
-                pf->rna_service_state = RNA_STATE_STATEFUL;
+                pf->service_disco_state = APPID_DISCO_STATE_STATEFUL;
             }
             else
                 rd->state = REXEC_STATE_USERNAME;
diff --git a/src/network_inspectors/appid/service_plugins/service_rfb.cc b/src/network_inspectors/appid/service_plugins/service_rfb.cc
index 0ff8cf832..af56d7f0a 100644
--- a/src/network_inspectors/appid/service_plugins/service_rfb.cc
+++ b/src/network_inspectors/appid/service_plugins/service_rfb.cc
@@ -37,7 +37,6 @@ RfbServiceDetector::RfbServiceDetector(ServiceDiscovery* sd)
     name = "rfb";
     proto = IpProtocol::TCP;
     detectorType = DETECTOR_TYPE_DECODER;
-    current_ref_count =  1;
 
     tcp_patterns =
     {
diff --git a/src/network_inspectors/appid/service_plugins/service_rlogin.cc b/src/network_inspectors/appid/service_plugins/service_rlogin.cc
index ac65b0f19..c5a1c8d93 100644
--- a/src/network_inspectors/appid/service_plugins/service_rlogin.cc
+++ b/src/network_inspectors/appid/service_plugins/service_rlogin.cc
@@ -51,7 +51,6 @@ RloginServiceDetector::RloginServiceDetector(ServiceDiscovery* sd)
     name = "rlogin";
     proto = IpProtocol::TCP;
     detectorType = DETECTOR_TYPE_DECODER;
-    current_ref_count =  1;
 
     appid_registry =
     {
diff --git a/src/network_inspectors/appid/service_plugins/service_rpc.cc b/src/network_inspectors/appid/service_plugins/service_rpc.cc
index e26f9854b..acdea71c5 100644
--- a/src/network_inspectors/appid/service_plugins/service_rpc.cc
+++ b/src/network_inspectors/appid/service_plugins/service_rpc.cc
@@ -39,7 +39,7 @@
 #include "log/messages.h"
 #include "protocols/packet.h"
 
-/*#define RNA_DEBUG_RPC   1 */
+/*#define APPID_DEBUG_RPC   1 */
 
 enum RPCState
 {
@@ -180,7 +180,6 @@ RpcServiceDetector::RpcServiceDetector(ServiceDiscovery* sd)
     name = "rpc";
     proto = IpProtocol::TCP;
     detectorType = DETECTOR_TYPE_DECODER;
-    current_ref_count =  1;
 
     struct rpcent* rpc;
     RPCProgram* prog;
@@ -410,7 +409,7 @@ int RpcServiceDetector::validate_packet(const uint8_t* data, uint16_t size, int
                         if (pf)
                         {
                             pf->add_flow_data_id((uint16_t)tmp, this);
-                            pf->rna_service_state = RNA_STATE_STATEFUL;
+                            pf->service_disco_state = APPID_DISCO_STATE_STATEFUL;
                             pf->set_session_flags(asd->get_session_flags(
                                 APPID_SESSION_RESPONDER_MONITORED |
                                 APPID_SESSION_INITIATOR_MONITORED |
@@ -479,14 +478,14 @@ int RpcServiceDetector::rpc_udp_validate(AppIdDiscoveryArgs& args)
         rd->xid = 0xFFFFFFFF;
     }
 
-#ifdef RNA_DEBUG_RPC
+#ifdef APPID_DEBUG_RPC
     fprintf(SF_DEBUG_FILE, "Begin %u -> %u %u %d state %d\n", pkt->src_port, pkt->dst_port,
         asd->proto, dir, rd->state);
 #endif
 
     rval = validate_packet(data, size, dir, asd, pkt, rd, &pname, &program);
 
-#ifdef RNA_DEBUG_RPC
+#ifdef APPID_DEBUG_RPC
     fprintf(SF_DEBUG_FILE, "End %u -> %u %u %d state %d rval %d\n", pkt->src_port, pkt->dst_port,
         asd->proto, dir, rd->state, rval);
 #endif
@@ -730,7 +729,7 @@ int RpcServiceDetector::rpc_tcp_validate(AppIdDiscoveryArgs& args)
                     {
                         if (rd->tcpsize[dir] & RPC_TCP_FRAG_MASK)
                         {
-#ifdef RNA_DEBUG_RPC
+#ifdef APPID_DEBUG_RPC
                             fprintf(SF_DEBUG_FILE, "V Begin %u -> %u %u %d state %d\n",
                                 pkt->src_port, pkt->dst_port, asd->proto, dir, rd->state);
 #endif
@@ -738,7 +737,7 @@ int RpcServiceDetector::rpc_tcp_validate(AppIdDiscoveryArgs& args)
                             ret = validate_packet(rd->tcpdata[dir], rd->tcppos[dir], dir, asd,
                                 pkt, rd, &pname, &program);
 
-#ifdef RNA_DEBUG_RPC
+#ifdef APPID_DEBUG_RPC
                             fprintf(SF_DEBUG_FILE, "V End %u -> %u %u %d state %d rval %d\n",
                                 pkt->src_port, pkt->dst_port, asd->proto, dir, rd->state, ret);
 #endif
@@ -807,7 +806,7 @@ int RpcServiceDetector::rpc_tcp_validate(AppIdDiscoveryArgs& args)
             {
                 if (rd->tcpsize[dir] & RPC_TCP_FRAG_MASK)
                 {
-#ifdef RNA_DEBUG_RPC
+#ifdef APPID_DEBUG_RPC
                     fprintf(SF_DEBUG_FILE, "P Begin %u -> %u %u %d state %d\n", pkt->src_port,
                         pkt->dst_port, asd->proto, dir, rd->state);
 #endif
@@ -815,7 +814,7 @@ int RpcServiceDetector::rpc_tcp_validate(AppIdDiscoveryArgs& args)
                     ret = validate_packet(rd->tcpdata[dir], rd->tcppos[dir], dir, asd, pkt,
                         rd, &pname, &program);
 
-#ifdef RNA_DEBUG_RPC
+#ifdef APPID_DEBUG_RPC
                     fprintf(SF_DEBUG_FILE, "P End %u -> %u %u %d state %d rval %d\n",
                         pkt->src_port, pkt->dst_port, asd->proto, dir, rd->state, ret);
 #endif
diff --git a/src/network_inspectors/appid/service_plugins/service_rshell.cc b/src/network_inspectors/appid/service_plugins/service_rshell.cc
index 879ef13ee..0ce287ed8 100644
--- a/src/network_inspectors/appid/service_plugins/service_rshell.cc
+++ b/src/network_inspectors/appid/service_plugins/service_rshell.cc
@@ -59,7 +59,6 @@ RshellServiceDetector::RshellServiceDetector(ServiceDiscovery* sd)
     name = "rshell";
     proto = IpProtocol::TCP;
     detectorType = DETECTOR_TYPE_DECODER;
-    current_ref_count =  1;
     app_id = add_appid_protocol_reference("rsh-error");
 
     appid_registry =
@@ -155,11 +154,11 @@ int RshellServiceDetector::validate(AppIdDiscoveryArgs& args)
                 APPID_EARLY_SESSION_FLAG_FW_RULE);
             if (pf)
             {
-                pf->rna_client_state = RNA_STATE_FINISHED;
+                pf->client_disco_state = APPID_DISCO_STATE_FINISHED;
                 data_add(pf, tmp_rd, &rshell_free_state);
                 if (pf->add_flow_data_id((uint16_t)port, this))
                 {
-                    pf->rna_service_state = RNA_STATE_FINISHED;
+                    pf->service_disco_state = APPID_DISCO_STATE_FINISHED;
                     tmp_rd->state = RSHELL_STATE_DONE;
                     tmp_rd->parent = nullptr;
                     return APPID_ENOMEM;
@@ -169,7 +168,7 @@ int RshellServiceDetector::validate(AppIdDiscoveryArgs& args)
                     APPID_SESSION_CONTINUE | APPID_SESSION_REXEC_STDERR | APPID_SESSION_NO_TPI |
                     APPID_SESSION_SERVICE_DETECTED | APPID_SESSION_NOT_A_SERVICE |
                     APPID_SESSION_PORT_SERVICE_DONE);
-                pf->rna_service_state = RNA_STATE_STATEFUL;
+                pf->service_disco_state = APPID_DISCO_STATE_STATEFUL;
             }
             else
             {
diff --git a/src/network_inspectors/appid/service_plugins/service_rsync.cc b/src/network_inspectors/appid/service_plugins/service_rsync.cc
index a09984a63..76bd6ed84 100644
--- a/src/network_inspectors/appid/service_plugins/service_rsync.cc
+++ b/src/network_inspectors/appid/service_plugins/service_rsync.cc
@@ -49,7 +49,6 @@ RsyncServiceDetector::RsyncServiceDetector(ServiceDiscovery* sd)
     name = "rsync";
     proto = IpProtocol::TCP;
     detectorType = DETECTOR_TYPE_DECODER;
-    current_ref_count =  1;
 
     tcp_patterns =
     {
diff --git a/src/network_inspectors/appid/service_plugins/service_rtmp.cc b/src/network_inspectors/appid/service_plugins/service_rtmp.cc
index aafa0bbae..7211bc4ac 100644
--- a/src/network_inspectors/appid/service_plugins/service_rtmp.cc
+++ b/src/network_inspectors/appid/service_plugins/service_rtmp.cc
@@ -79,7 +79,6 @@ RtmpServiceDetector::RtmpServiceDetector(ServiceDiscovery* sd)
     name = "rtmp";
     proto = IpProtocol::TCP;
     detectorType = DETECTOR_TYPE_DECODER;
-    current_ref_count =  1;
 
     appid_registry =
     {
diff --git a/src/network_inspectors/appid/service_plugins/service_snmp.cc b/src/network_inspectors/appid/service_plugins/service_snmp.cc
index 7cb812ea8..7218e59a7 100644
--- a/src/network_inspectors/appid/service_plugins/service_snmp.cc
+++ b/src/network_inspectors/appid/service_plugins/service_snmp.cc
@@ -95,7 +95,6 @@ SnmpServiceDetector::SnmpServiceDetector(ServiceDiscovery* sd)
     name = "snmp";
     proto = IpProtocol::UDP;
     detectorType = DETECTOR_TYPE_DECODER;
-    current_ref_count =  1;
 
     app_id = add_appid_protocol_reference("snmp");
 
@@ -500,7 +499,7 @@ int SnmpServiceDetector::validate(AppIdDiscoveryArgs& args)
                 return APPID_ENULL;
             }
             PopulateExpectedFlow(asd, pf, APPID_SESSION_EXPECTED_EVALUATE);
-            pf->rna_service_state = RNA_STATE_STATEFUL;
+            pf->service_disco_state = APPID_DISCO_STATE_STATEFUL;
             pf->scan_flags |= SCAN_HOST_PORT_FLAG;
             pf->common.initiator_ip = *sip;
         }
diff --git a/src/network_inspectors/appid/service_plugins/service_ssh.cc b/src/network_inspectors/appid/service_plugins/service_ssh.cc
index 609a91aa1..aaf98819b 100644
--- a/src/network_inspectors/appid/service_plugins/service_ssh.cc
+++ b/src/network_inspectors/appid/service_plugins/service_ssh.cc
@@ -136,7 +136,6 @@ SshServiceDetector::SshServiceDetector(ServiceDiscovery* sd)
     name = "ssh";
     proto = IpProtocol::TCP;
     detectorType = DETECTOR_TYPE_DECODER;
-    current_ref_count =  1;
 
     tcp_patterns =
     {
diff --git a/src/network_inspectors/appid/service_plugins/service_ssl.cc b/src/network_inspectors/appid/service_plugins/service_ssl.cc
index 54bf64bcb..effc37f01 100644
--- a/src/network_inspectors/appid/service_plugins/service_ssl.cc
+++ b/src/network_inspectors/appid/service_plugins/service_ssl.cc
@@ -282,7 +282,6 @@ SslServiceDetector::SslServiceDetector(ServiceDiscovery* sd)
     name = "ssl";
     proto = IpProtocol::TCP;
     detectorType = DETECTOR_TYPE_DECODER;
-    current_ref_count = 1;
 
     tcp_patterns =
     {
@@ -1154,7 +1153,7 @@ bool setSSLSquelch(Packet* p, int type, AppId appId)
         break;
     case 2:
         f->client_app_id = appId;
-        f->rna_client_state = RNA_STATE_FINISHED;
+        f->client_disco_state = APPID_DISCO_STATE_FINISHED;
         break;
     default:
         return false;
diff --git a/src/network_inspectors/appid/service_plugins/service_telnet.cc b/src/network_inspectors/appid/service_plugins/service_telnet.cc
index add9155fd..51c394373 100644
--- a/src/network_inspectors/appid/service_plugins/service_telnet.cc
+++ b/src/network_inspectors/appid/service_plugins/service_telnet.cc
@@ -78,7 +78,6 @@ TelnetServiceDetector::TelnetServiceDetector(ServiceDiscovery* sd)
     name = "telnet";
     proto = IpProtocol::TCP;
     detectorType = DETECTOR_TYPE_DECODER;
-    current_ref_count =  1;
 
     appid_registry =
     {
diff --git a/src/network_inspectors/appid/service_plugins/service_tftp.cc b/src/network_inspectors/appid/service_plugins/service_tftp.cc
index 8d65710c8..37955bbf1 100644
--- a/src/network_inspectors/appid/service_plugins/service_tftp.cc
+++ b/src/network_inspectors/appid/service_plugins/service_tftp.cc
@@ -71,7 +71,6 @@ TftpServiceDetector::TftpServiceDetector(ServiceDiscovery* sd)
     name = "tftp";
     proto = IpProtocol::UDP;
     detectorType = DETECTOR_TYPE_DECODER;
-    current_ref_count =  1;
 
     app_id = add_appid_protocol_reference("tftp");
 
@@ -206,7 +205,7 @@ int TftpServiceDetector::validate(AppIdDiscoveryArgs& args)
             }
             PopulateExpectedFlow(asd, pf, APPID_SESSION_EXPECTED_EVALUATE);
             pf->common.initiator_ip = *sip;
-            pf->rna_service_state = RNA_STATE_STATEFUL;
+            pf->service_disco_state = APPID_DISCO_STATE_STATEFUL;
             pf->scan_flags |= SCAN_HOST_PORT_FLAG;
         }
         else
diff --git a/src/network_inspectors/appid/service_plugins/service_timbuktu.cc b/src/network_inspectors/appid/service_plugins/service_timbuktu.cc
index b4beb46a3..4a269f1ab 100644
--- a/src/network_inspectors/appid/service_plugins/service_timbuktu.cc
+++ b/src/network_inspectors/appid/service_plugins/service_timbuktu.cc
@@ -64,7 +64,6 @@ TimbuktuServiceDetector::TimbuktuServiceDetector(ServiceDiscovery* sd)
     name = "timbuktu";
     proto = IpProtocol::TCP;
     detectorType = DETECTOR_TYPE_DECODER;
-    current_ref_count =  1;
 
     tcp_patterns =
     {
diff --git a/src/network_inspectors/appid/service_plugins/service_tns.cc b/src/network_inspectors/appid/service_plugins/service_tns.cc
index 2a78a90f1..fb23259fa 100644
--- a/src/network_inspectors/appid/service_plugins/service_tns.cc
+++ b/src/network_inspectors/appid/service_plugins/service_tns.cc
@@ -93,7 +93,6 @@ TnsServiceDetector::TnsServiceDetector(ServiceDiscovery* sd)
     name = "tns";
     proto = IpProtocol::TCP;
     detectorType = DETECTOR_TYPE_DECODER;
-    current_ref_count =  1;
 
     tcp_patterns =
     {
diff --git a/src/network_inspectors/appid/service_state.cc b/src/network_inspectors/appid/service_state.cc
index d1bb93c8f..e5704db78 100644
--- a/src/network_inspectors/appid/service_state.cc
+++ b/src/network_inspectors/appid/service_state.cc
@@ -28,11 +28,109 @@
 #include <map>
 
 #include "service_plugins/service_detector.h"
+#include "log/messages.h"
 #include "sfip/sf_ip.h"
+#include "time/packet_time.h"
 #include "utils/util.h"
-#include "log/messages.h"
 
-//#define DEBUG_SERVICE_STATE 1
+ServiceDiscoveryState::ServiceDiscoveryState()
+{
+    state = SERVICE_ID_STATE::SEARCHING_PORT_PATTERN;
+    last_detract.clear();
+    last_invalid_client.clear();
+    reset_time = 0;
+}
+
+ServiceDiscoveryState::~ServiceDiscoveryState()
+{
+    if ( brute_force_mgr )
+        delete brute_force_mgr;
+}
+
+void ServiceDiscoveryState::set_service_id_valid(ServiceDetector* sd)
+{
+    service = sd;
+    reset_time = 0;
+    if (state != SERVICE_ID_STATE::VALID)
+    {
+        state = SERVICE_ID_STATE::VALID;
+        valid_count = 0;
+    }
+
+    if(!valid_count)
+    {
+        detract_count = 0;
+        last_detract.clear();
+        invalid_client_count = 0;
+        last_invalid_client.clear();
+    }
+
+    if (valid_count < STATE_ID_MAX_VALID_COUNT)
+        valid_count++;
+}
+
+/* Handle some exception cases on failure:
+ *  - valid_count: If we have a detector that should be valid, but it keeps
+ *    failing, consider restarting the detector search.
+ *  - invalid_client_count: If our service detector search had trouble
+ *    simply because of unrecognized client data, then consider retrying
+ *    the search again. */
+void ServiceDiscoveryState::set_service_id_failed(AppIdSession* asd, const SfIp* client_ip)
+{
+    /* If we had a valid detector, check for too many fails.  If so, start
+     * search sequence again. */
+    if (state == SERVICE_ID_STATE::VALID)
+    {
+        /* Too many invalid clients?  If so, count it as an invalid detect. */
+        if (invalid_client_count >= STATE_ID_INVALID_CLIENT_THRESHOLD)
+        {
+            if (valid_count <= 1)
+            {
+                state = SERVICE_ID_STATE::SEARCHING_PORT_PATTERN;
+                invalid_client_count = 0;
+                last_invalid_client.clear();
+                valid_count = 0;
+                detract_count = 0;
+                last_detract.clear();
+            }
+            else
+            {
+                valid_count--;
+                last_invalid_client = *client_ip;
+                invalid_client_count = 0;
+            }
+        }
+        else if (invalid_client_count == 0)
+        {
+            // Just a plain old fail.  If too many of these happen, start search process over.
+            if (last_detract.fast_eq6(*client_ip))
+                detract_count++;
+            else
+                last_detract = *client_ip;
+
+            if (detract_count >= STATE_ID_NEEDED_DUPE_DETRACT_COUNT)
+            {
+                if (valid_count <= 1)
+                {
+                    state = SERVICE_ID_STATE::SEARCHING_PORT_PATTERN;
+                    invalid_client_count = 0;
+                    last_invalid_client.clear();
+                    valid_count = 0;
+                    detract_count = 0;
+                    last_detract.clear();
+                }
+                else
+                    valid_count--;
+            }
+        }
+    }
+    else if ( ( state == SERVICE_ID_STATE::SEARCHING_PORT_PATTERN ) &&
+              ( asd->service_search_state == SESSION_SERVICE_ID_STATE::PENDING ) &&
+              ( !asd->service_candidates.size() ) )
+    {
+        state = SEARCHING_BRUTE_FORCE;
+    }
+}
 
 class AppIdServiceStateKey
 {
@@ -76,10 +174,9 @@ public:
     char padding[3];
 };
 
-// FIXIT-L - no memcap on size of this table, do we need that?
 THREAD_LOCAL std::map<AppIdServiceStateKey, ServiceDiscoveryState*>* service_state_cache = nullptr;
 
-void AppIdServiceState::initialize(unsigned long)
+void AppIdServiceState::initialize()
 {
     service_state_cache = new std::map<AppIdServiceStateKey, ServiceDiscoveryState*>;
 }
@@ -98,7 +195,7 @@ void AppIdServiceState::clean(void)
 }
 
 ServiceDiscoveryState* AppIdServiceState::add(const SfIp* ip, IpProtocol proto, uint16_t port,
-    uint32_t level)
+    bool decrypted)
 {
     AppIdServiceStateKey ssk;
     ServiceDiscoveryState* ss = nullptr;
@@ -106,80 +203,48 @@ ServiceDiscoveryState* AppIdServiceState::add(const SfIp* ip, IpProtocol proto,
     ssk.ip.set(*ip);
     ssk.proto = proto;
     ssk.port = port;
-    ssk.level = level;
+    ssk.level = decrypted ? 1 : 0;
 
     std::map<AppIdServiceStateKey, ServiceDiscoveryState*>::iterator it;
     it = service_state_cache->find(ssk);
-    if ( it != service_state_cache->end())
-    {
-        ss = it->second;
-    }
-    else
+    if ( it == service_state_cache->end())
     {
         ss = new ServiceDiscoveryState;
         (*service_state_cache)[ssk] = ss;
     }
-
-#ifdef DEBUG_SERVICE_STATE
-    char ipstr[INET6_ADDRSTRLEN];
-
-    ipstr[0] = 0;
-    sfip_ntop(ip, ipstr, sizeof(ipstr));
-    DebugFormat(DEBUG_APPID, "ServiceState: Added to hash: %s:%u:%u:%u %p\n", ipstr,
-        (unsigned)proto,
-        (unsigned)port, level, (void*)ss);
-#endif
+    else
+        ss = it->second;
 
     return ss;
 }
 
 ServiceDiscoveryState* AppIdServiceState::get(const SfIp* ip, IpProtocol proto, uint16_t port,
-    uint32_t level)
+    bool decrypted)
 {
     AppIdServiceStateKey ssk;
     ServiceDiscoveryState* ss = nullptr;
-    char ipstr[INET6_ADDRSTRLEN];   // FIXIT-M ASAN reports mem leak on ServiceMatch* objects if
-                                    // this is not defined
-                                    //  which makes no sense, need to investigate further
 
     ssk.ip.set(*ip);
     ssk.proto = proto;
     ssk.port = port;
-    ssk.level = level;
+    ssk.level = decrypted ? 1 : 0;
 
     std::map<AppIdServiceStateKey, ServiceDiscoveryState*>::iterator it;
     it = service_state_cache->find(ssk);
     if ( it != service_state_cache->end())
-    {
         ss = it->second;
-        if (ss->service && !ss->service->ref_count)
-        {
-            ss->service = nullptr;
-            ss->state = SERVICE_ID_NEW;
-        }
-    }
-
-#ifdef DEBUG_SERVICE_STATE
-    ipstr[0] = 0;
-    sfip_ntop(ip, ipstr, sizeof(ipstr));
-    DebugFormat(DEBUG_APPID, "ServiceState: Read from hash: %s:%u:%u:%u %p %u %p\n", ipstr,
-        (unsigned)proto,
-        (unsigned)port, level, (void*)ss, ss ? ss->state : 0, ss ? (void*)ss->service : nullptr);
-#else
-    UNUSED(ipstr);
-#endif
 
     return ss;
 }
 
-void AppIdServiceState::remove(const SfIp* ip, IpProtocol proto, uint16_t port, uint32_t level)
+void AppIdServiceState::remove(const SfIp* ip, IpProtocol proto, uint16_t port, bool decrypted)
 {
     AppIdServiceStateKey ssk;
 
     ssk.ip.set(*ip);
     ssk.proto = proto;
     ssk.port = port;
-    ssk.level = level;
+    ssk.level = decrypted ? 1 : 0;
 
     std::map<AppIdServiceStateKey, ServiceDiscoveryState*>::iterator it;
     it = service_state_cache->find(ssk);
@@ -198,6 +263,22 @@ void AppIdServiceState::remove(const SfIp* ip, IpProtocol proto, uint16_t port,
     }
 }
 
+void AppIdServiceState::check_reset(AppIdSession* asd, const SfIp* ip, uint16_t port )
+{
+    ServiceDiscoveryState* sds = AppIdServiceState::get(ip, IpProtocol::TCP, port,
+        asd->is_decrypted());
+
+    if (sds)
+    {
+        if (!sds->reset_time)
+            sds->reset_time = packet_time();
+        else if ((packet_time() - sds->reset_time) >= 60)
+        {
+            AppIdServiceState::remove(ip, IpProtocol::TCP, port, asd->is_decrypted());
+            asd->set_session_flags(APPID_SESSION_SERVICE_DELETED);
+        }
+    }
+}
 void AppIdServiceState::dump_stats(void)
 {
     // FIXIT-L - do we need to keep ipv4 and ipv6 separate?
diff --git a/src/network_inspectors/appid/service_state.h b/src/network_inspectors/appid/service_state.h
index 0de276aef..38ba0b9db 100644
--- a/src/network_inspectors/appid/service_state.h
+++ b/src/network_inspectors/appid/service_state.h
@@ -22,6 +22,8 @@
 #ifndef SERVICE_STATE_H
 #define SERVICE_STATE_H
 
+#include <mutex>
+
 #include "sfip/sf_ip.h"
 #include "service_plugins/service_discovery.h"
 #include "protocols/protocol_ids.h"
@@ -29,88 +31,81 @@
 
 class ServiceDetector;
 
-enum class IpProtocol : uint8_t;
-
-// Service state stored in hosttracker for maintaining service matching states.
 enum SERVICE_ID_STATE
 {
-    SERVICE_ID_NEW = 0,     // service search starting
-    SERVICE_ID_VALID,       // service detected
-    SERVICE_ID_PORT,        // matched based on src/dest port of first packet
-    SERVICE_ID_PATTERN,     // match based on pattern in first response packet
-    SERVICE_ID_BRUTE_FORCE, // match based on round-robin through tcp/udp service lists
-                            // the lists are walked from first element to last. In a detector
-                            // declares a flow incompatible or the flow closes earlier than
-                            // expected by detector, then the next detector is tried. This can
-                            //  obviously delay detection under some scenarios.
-};
-
-enum DetectorType
-{
-    DETECTOR_TYPE_PASSIVE =  0,
-    DETECTOR_TYPE_DECODER =  0,
-    DETECTOR_TYPE_NETFLOW,
-    DETECTOR_TYPE_PORT,
-    DETECTOR_TYPE_DERIVED,
-    DETECTOR_TYPE_CONFLICT,
-    DETECTOR_TYPE_PATTERN,
-    DETECTOR_TYPE_NOT_SET
+    SEARCHING_PORT_PATTERN = 0,
+    SEARCHING_BRUTE_FORCE,
+    FAILED,
+    VALID
 };
 
-// Service state saved in hosttracker, for identifying a service across multiple flow instances.
-class ServiceDiscoveryState
+class AppIdDetectorList
 {
 public:
-    ServiceDiscoveryState()
+    AppIdDetectorList(IpProtocol proto)
     {
-        last_detract.clear();
-        last_invalid_client.clear();
-        reset_time = 0;
+        if (proto == IpProtocol::TCP)
+            detectors = &ServiceDiscovery::get_instance().tcp_detectors;
+        else
+            detectors = &ServiceDiscovery::get_instance().udp_detectors;
+        dit = detectors->begin();
     }
 
-    ~ServiceDiscoveryState()
+    ServiceDetector* next()
     {
-        if ( brute_force_mgr )
-            delete brute_force_mgr;
+        ServiceDetector* detector = nullptr;
+
+        if ( dit != detectors->end())
+            detector = (ServiceDetector*)(dit++)->second;
+        return detector;
     }
 
+    void reset()
+    {
+        dit = detectors->begin();
+    }
+
+private:
+    AppIdDetectors* detectors;
+    AppIdDetectorsIterator dit;
+};
+
+class ServiceDiscoveryState
+{
+public:
+    ServiceDiscoveryState();
+    ~ServiceDiscoveryState();
+    void set_service_id_valid(ServiceDetector* sd);
+    void set_service_id_failed(AppIdSession* asd, const SfIp* client_ip);
+
+    SERVICE_ID_STATE state;
     ServiceDetector* service = nullptr;
     AppIdDetectorList* brute_force_mgr = nullptr;
-
-    /**State of service identification.*/
-    SERVICE_ID_STATE state = SERVICE_ID_NEW;
     unsigned valid_count = 0;
     unsigned detract_count = 0;
     SfIp last_detract;
 
-    /**Number of consequetive flows that were declared incompatible by detectors. Incompatibility
-     * means client packet did not match.
-     */
+    // consecutive incompatible flows - incompatibile means client packet did not match.
     unsigned invalid_client_count = 0;
 
     /**IP address of client in last flow that was declared incompatible. If client IP address is
-     * different everytime, then consequetive incompatible status indicate that flow is not using
+     * different everytime, then consecutive incompatible status indicate that flow is not using
      * specific service.
      */
     SfIp last_invalid_client;
-
-    /** Count for number of unknown sessions saved
-     */
-    unsigned unknowns_logged = 0;
     time_t reset_time;
-
-    /** Is this entry currently being used in an active session? */
-    bool searching = false;
 };
 
 class AppIdServiceState
 {
 public:
-    static void initialize(unsigned long);
+    static void initialize();
     static void clean();
-    static ServiceDiscoveryState* add(const SfIp*, IpProtocol, uint16_t port, uint32_t level);
-    static ServiceDiscoveryState* get(const SfIp*, IpProtocol, uint16_t port, uint32_t level);
-    static void remove(const SfIp*, IpProtocol, uint16_t port, uint32_t level);
+    static ServiceDiscoveryState* add(const SfIp*, IpProtocol, uint16_t port, bool decrypted);
+    static ServiceDiscoveryState* get(const SfIp*, IpProtocol, uint16_t port, bool decrypted);
+    static void remove(const SfIp*, IpProtocol, uint16_t port, bool decrypted);
+    static void check_reset(AppIdSession* asd, const SfIp* ip, uint16_t port );
+
     static void dump_stats();
 };
 
diff --git a/src/network_inspectors/appid/thirdparty_appid_utils.cc b/src/network_inspectors/appid/thirdparty_appid_utils.cc
index 280ba1747..a5e9a5c09 100644
--- a/src/network_inspectors/appid/thirdparty_appid_utils.cc
+++ b/src/network_inspectors/appid/thirdparty_appid_utils.cc
@@ -936,10 +936,10 @@ bool do_third_party_discovery(AppIdSession* asd, IpProtocol protocol, const SfIp
                         APP_ID_HTTP
                         && !asd->get_session_flags(APPID_SESSION_APP_REINSPECT))
                     {
-                        asd->rna_client_state = RNA_STATE_FINISHED;
+                        asd->rna_client_state = APPID_STATE_FINISHED;
                         asd->set_session_flags(APPID_SESSION_CLIENT_DETECTED |
                             APPID_SESSION_SERVICE_DETECTED);
-                        asd->rna_service_state = RNA_STATE_FINISHED;
+                        asd->rna_service_state = APPID_STATE_FINISHED;
                         asd->clear_session_flags(APPID_SESSION_CONTINUE);
                         if (direction == APP_ID_FROM_INITIATOR)
                         {