From: Shravan Rangarajuvenkata (shrarang) <shrarang@cisco.com>
Date: Wed, 26 Aug 2020 03:11:01 +0000 (+0000)
Subject: Merge pull request #2380 in SNORT/snort3 from ~SHRARANG/snort3:appid_listener_more_lo... 
X-Git-Tag: 3.0.2-6~39
X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=d4cfb747ae066c1fd2bdcfaaed83f0ee497107d4;p=thirdparty%2Fsnort3.git

Merge pull request #2380 in SNORT/snort3 from ~SHRARANG/snort3:appid_listener_more_logging to master

Squashed commit of the following:

commit 86da97002e7b8c30e75cd99817ab5bab9cf1e19a
Author: Shravan Rangaraju <shrarang@cisco.com>
Date:   Wed Aug 5 11:14:54 2020 -0400

    appid: support json logging in appid_listener
---

diff --git a/src/dump_config/json_config_output.h b/src/dump_config/json_config_output.h
index 5bb41f8ce..617d48f56 100644
--- a/src/dump_config/json_config_output.h
+++ b/src/dump_config/json_config_output.h
@@ -35,7 +35,7 @@ private:
     void dump(const ConfigData&) override;
 
 private:
-    JsonStream json;
+    snort::JsonStream json;
 };
 
 class JsonTopConfigOutput : public ConfigOutput
@@ -47,7 +47,7 @@ private:
     void dump(const ConfigData&) override;
 
 private:
-    JsonStream json;
+    snort::JsonStream json;
 };
 
 #endif // JSON_CONFIG_OUTPUT_H
diff --git a/src/helpers/CMakeLists.txt b/src/helpers/CMakeLists.txt
index c51eb69e3..49e3f71c1 100644
--- a/src/helpers/CMakeLists.txt
+++ b/src/helpers/CMakeLists.txt
@@ -16,6 +16,7 @@ set (HELPERS_INCLUDES
     boyer_moore_search.h
     literal_search.h
     scratch_allocator.h
+    json_stream.h
 )
 
 add_library (helpers OBJECT
diff --git a/src/helpers/json_stream.cc b/src/helpers/json_stream.cc
index 545c37ebf..f3140e561 100644
--- a/src/helpers/json_stream.cc
+++ b/src/helpers/json_stream.cc
@@ -26,6 +26,8 @@
 #include <cassert>
 #include <iomanip>
 
+using namespace snort;
+
 void JsonStream::open(const char* key)
 {
     split();
@@ -96,6 +98,22 @@ void JsonStream::put(const char* key, long val)
     out << val;
 }
 
+void JsonStream::put(const char* key, const char* val)
+{
+    if (val and val[0] == '\0')
+        return;
+
+    split();
+
+    if ( key )
+        out << std::quoted(key) << ": ";
+
+    if (val)
+        out << std::quoted(val);
+    else
+        out << "null";
+}
+
 void JsonStream::put(const char* key, const std::string& val)
 {
     if ( val.empty() )
diff --git a/src/helpers/json_stream.h b/src/helpers/json_stream.h
index 211656035..7c4c1df67 100644
--- a/src/helpers/json_stream.h
+++ b/src/helpers/json_stream.h
@@ -23,8 +23,11 @@
 // Simple output stream for outputting JSON data.
 
 #include <iostream>
+#include "main/snort_types.h"
 
-class JsonStream
+namespace snort
+{
+class SO_PUBLIC JsonStream
 {
 public:
     JsonStream(std::ostream& o) : out(o) { }
@@ -38,6 +41,7 @@ public:
 
     void put(const char* key);    // null
     void put(const char* key, long val);
+    void put(const char* key, const char* val);
     void put(const char* key, const std::string& val);
     void put(const char* key, double val, int precision);
 
@@ -53,6 +57,6 @@ private:
     unsigned level = 0;
     unsigned level_array = 0;
 };
-
+}
 #endif
 
diff --git a/src/helpers/test/json_stream_test.cc b/src/helpers/test/json_stream_test.cc
index 06c8550ad..4e9236216 100644
--- a/src/helpers/test/json_stream_test.cc
+++ b/src/helpers/test/json_stream_test.cc
@@ -27,6 +27,8 @@
 
 #include "../json_stream.h"
 
+using namespace snort;
+
 TEST_CASE("basic", "[json_stream]")
 {
     std::ostringstream ss;
@@ -90,7 +92,7 @@ TEST_CASE("basic", "[json_stream]")
 
     SECTION("int")
     {
-        js.put("i", 0);
+        js.put("i", (long) 0);
         const char* x = R"-("i": 0)-";
         CHECK(ss.str() == x);
     }
diff --git a/src/network_inspectors/appid/appid_api.cc b/src/network_inspectors/appid/appid_api.cc
index c37ccb120..b06fdbc1f 100644
--- a/src/network_inspectors/appid/appid_api.cc
+++ b/src/network_inspectors/appid/appid_api.cc
@@ -25,6 +25,7 @@
 
 #include "appid_api.h"
 
+#include "detection/detection_engine.h"
 #include "framework/inspector.h"
 #include "managers/inspector_manager.h"
 #include "utils/util.h"
@@ -56,6 +57,25 @@ const char* AppIdApi::get_application_name(AppId app_id, OdpContext& odp_ctxt)
     return odp_ctxt.get_app_info_mgr().get_app_name(app_id);
 }
 
+const char* AppIdApi::get_application_name(AppId app_id, const Flow& flow)
+{
+    const char* app_name = nullptr;
+    AppIdSession* asd = get_appid_session(flow);
+    if (asd)
+    {
+        // Skip sessions using old odp context after odp reload
+        AppIdInspector* inspector = (AppIdInspector*) InspectorManager::get_inspector(MOD_NAME, true);
+        if (inspector and (&(inspector->get_ctxt().get_odp_ctxt()) != &(asd->get_odp_ctxt())))
+            return nullptr;
+
+        if (app_id == APP_ID_UNKNOWN)
+            return "unknown";
+        app_name = asd->get_odp_ctxt().get_app_info_mgr().get_app_name(app_id);
+    }
+
+    return app_name;
+}
+
 const char* AppIdApi::get_application_name(const Flow& flow, bool from_client)
 {
     const char* app_name = nullptr;
@@ -300,7 +320,9 @@ bool AppIdApi::ssl_app_group_id_lookup(Flow* flow, const char* server_name,
         asd->set_ss_application_ids(client_id, payload_id, change_bits);
         asd->set_tls_host(change_bits);
 
-        asd->publish_appid_event(change_bits, flow);
+        Packet* p = DetectionEngine::get_current_packet();
+        assert(p);
+        asd->publish_appid_event(change_bits, *p);
     }
     else
     {
diff --git a/src/network_inspectors/appid/appid_api.h b/src/network_inspectors/appid/appid_api.h
index ed4f60237..41fcfdcfa 100644
--- a/src/network_inspectors/appid/appid_api.h
+++ b/src/network_inspectors/appid/appid_api.h
@@ -54,6 +54,7 @@ public:
 
     AppIdSession* get_appid_session(const Flow& flow);
     const char* get_application_name(AppId app_id, OdpContext& odp_ctxt);
+    const char* get_application_name(AppId app_id, const Flow& flow);
     const char* get_application_name(const Flow& flow, bool from_client);
     AppId get_application_id(const char* appName, const AppIdContext& ctxt);
     uint32_t produce_ha_state(const Flow& flow, uint8_t* buf);
diff --git a/src/network_inspectors/appid/appid_discovery.cc b/src/network_inspectors/appid/appid_discovery.cc
index f623b9e6d..77a1ee7bd 100644
--- a/src/network_inspectors/appid/appid_discovery.cc
+++ b/src/network_inspectors/appid/appid_discovery.cc
@@ -448,7 +448,7 @@ bool AppIdDiscovery::do_pre_discovery(Packet* p, AppIdSession*& asd, AppIdInspec
         asd->set_ss_application_ids(asd->pick_service_app_id(), asd->pick_ss_client_app_id(),
             asd->pick_ss_payload_app_id(), asd->pick_ss_misc_app_id(),
             asd->pick_ss_referred_payload_app_id(), change_bits);
-        asd->publish_appid_event(change_bits, p->flow);
+        asd->publish_appid_event(change_bits, *p);
         asd->set_session_flags(APPID_SESSION_FUTURE_FLOW_IDED);
 
         if (appidDebug->is_active())
@@ -911,5 +911,5 @@ void AppIdDiscovery::do_post_discovery(Packet* p, AppIdSession& asd,
         asd.pick_ss_referred_payload_app_id(), change_bits);
     asd.set_tls_host(change_bits);
 
-    asd.publish_appid_event(change_bits, p->flow);
+    asd.publish_appid_event(change_bits, *p);
 }
diff --git a/src/network_inspectors/appid/appid_http_event_handler.cc b/src/network_inspectors/appid/appid_http_event_handler.cc
index 27d233976..06ee55248 100644
--- a/src/network_inspectors/appid/appid_http_event_handler.cc
+++ b/src/network_inspectors/appid/appid_http_event_handler.cc
@@ -29,6 +29,7 @@
 
 #include <cassert>
 
+#include "detection/detection_engine.h"
 #include "managers/inspector_manager.h"
 #include "app_info_table.h"
 #include "appid_debug.h"
@@ -193,7 +194,9 @@ void HttpEventHandler::handle(DataEvent& event, Flow* flow)
     else
         asd->set_application_ids_service(APP_ID_HTTP2, change_bits);
 
-    asd->publish_appid_event(change_bits, flow, http_event->get_is_http2(),
+    Packet* p = DetectionEngine::get_current_packet();
+    assert(p);
+    asd->publish_appid_event(change_bits, *p, http_event->get_is_http2(),
         asd->get_api().get_hsessions_size() - 1);
 }
 
diff --git a/src/network_inspectors/appid/appid_session.cc b/src/network_inspectors/appid/appid_session.cc
index 582169af7..76adeea2e 100644
--- a/src/network_inspectors/appid/appid_session.cc
+++ b/src/network_inspectors/appid/appid_session.cc
@@ -54,7 +54,6 @@
 using namespace snort;
 
 unsigned AppIdSession::inspector_id = 0;
-THREAD_LOCAL uint32_t AppIdSession::appid_flow_data_id = 0;
 std::mutex AppIdSession::inferred_svcs_lock;
 uint16_t AppIdSession::inferred_svcs_ver = 0;
 
@@ -100,7 +99,6 @@ AppIdSession::AppIdSession(IpProtocol proto, const SfIp* ip, uint16_t port,
         odp_ctxt(odp_ctxt), tp_appid_ctxt(inspector.get_ctxt().get_tp_appid_ctxt())
 {
     service_ip.clear();
-    session_id = ++appid_flow_data_id;
     initiator_port = port;
 
     appid_stats.total_sessions++;
@@ -1066,13 +1064,13 @@ void AppIdSession::set_tp_payload_app_id(Packet& p, AppidSessionDirection dir, A
     }
 }
 
-void AppIdSession::publish_appid_event(AppidChangeBits& change_bits, Flow* flow,
+void AppIdSession::publish_appid_event(AppidChangeBits& change_bits, const Packet& p,
     bool is_http2, uint32_t http2_stream_index)
 {
     if (!api.stored_in_stash and change_bits.any())
     {
-        assert(flow and flow->stash);
-        flow->stash->store(STASH_APPID_DATA, &api, false);
+        assert(p.flow and p.flow->stash);
+        p.flow->stash->store(STASH_APPID_DATA, &api, false);
         api.stored_in_stash = true;
     }
 
@@ -1085,8 +1083,8 @@ void AppIdSession::publish_appid_event(AppidChangeBits& change_bits, Flow* flow,
     if (change_bits.none())
         return;
 
-    AppidEvent app_event(change_bits, is_http2, http2_stream_index, api);
-    DataBus::publish(APPID_EVENT_ANY_CHANGE, app_event, flow);
+    AppidEvent app_event(change_bits, is_http2, http2_stream_index, api, p);
+    DataBus::publish(APPID_EVENT_ANY_CHANGE, app_event, p.flow);
     if (appidDebug->is_active())
     {
         std::string str;
diff --git a/src/network_inspectors/appid/appid_session.h b/src/network_inspectors/appid/appid_session.h
index a04db0c1f..98ab3ae1b 100644
--- a/src/network_inspectors/appid/appid_session.h
+++ b/src/network_inspectors/appid/appid_session.h
@@ -246,7 +246,6 @@ public:
     size_t size_of() override
     { return sizeof(*this); }
 
-    uint32_t session_id = 0;
     snort::Flow* flow = nullptr;
     AppIdConfig& config;
     std::unordered_map<unsigned, AppIdFlowData*> flow_data;
@@ -391,7 +390,7 @@ public:
         AppidChangeBits& change_bits);
     void set_tp_payload_app_id(snort::Packet& p, AppidSessionDirection dir, AppId app_id,
         AppidChangeBits& change_bits);
-    void publish_appid_event(AppidChangeBits&, snort::Flow*, bool is_http2 = false,
+    void publish_appid_event(AppidChangeBits&, const snort::Packet&, bool is_http2 = false,
         uint32_t http2_stream_index = 0);
 
     inline void set_tp_app_id(AppId app_id)
@@ -561,7 +560,6 @@ private:
     void reinit_session_data(AppidChangeBits& change_bits);
     void delete_session_data(bool free_api = true);
 
-    static THREAD_LOCAL uint32_t appid_flow_data_id;
     bool tp_app_id_deferred = false;
     bool tp_payload_app_id_deferred = false;
 
diff --git a/src/network_inspectors/appid/appid_session_api.cc b/src/network_inspectors/appid/appid_session_api.cc
index 26316f406..0644ead4c 100644
--- a/src/network_inspectors/appid/appid_session_api.cc
+++ b/src/network_inspectors/appid/appid_session_api.cc
@@ -33,6 +33,14 @@
 
 using namespace snort;
 
+THREAD_LOCAL uint32_t AppIdSessionApi::appid_flow_data_id = 0;
+
+AppIdSessionApi::AppIdSessionApi(const AppIdSession* asd, const SfIp& ip) :
+    StashGenericObject(STASH_GENERIC_OBJECT_APPID), asd(asd), initiator_ip(ip)
+{
+    session_id = std::to_string(get_instance_id()) + "." + std::to_string(++appid_flow_data_id);
+}
+
 AppId AppIdSessionApi::get_service_app_id() const
 {
     return application_ids[APP_PROTOID_SERVICE];
diff --git a/src/network_inspectors/appid/appid_session_api.h b/src/network_inspectors/appid/appid_session_api.h
index d5b0e0b3c..bb5ab6ecc 100644
--- a/src/network_inspectors/appid/appid_session_api.h
+++ b/src/network_inspectors/appid/appid_session_api.h
@@ -136,9 +136,13 @@ public:
         return hsessions.size();
     }
 
+    const std::string& get_session_id() const
+    {
+        return session_id;
+    }
+
 protected:
-    AppIdSessionApi(const AppIdSession* asd, const SfIp& ip) :
-        StashGenericObject(STASH_GENERIC_OBJECT_APPID), asd(asd), initiator_ip(ip) {}
+    AppIdSessionApi(const AppIdSession* asd, const SfIp& ip);
 
 private:
     const AppIdSession* asd = nullptr;
@@ -151,6 +155,7 @@ private:
     snort::SfIp initiator_ip;
     ServiceAppDescriptor service;
     char* tls_host = nullptr;
+    std::string session_id;
 
     // Following two fields are used only for non-http sessions. For HTTP traffic,
     // these fields are maintained inside AppIdHttpSession.
@@ -158,6 +163,8 @@ private:
     ClientAppDescriptor client;
     PayloadAppDescriptor payload;
 
+    static THREAD_LOCAL uint32_t appid_flow_data_id;
+
     void set_ss_application_ids(AppId service, AppId client, AppId payload, AppId misc,
         AppId referred, AppidChangeBits& change_bits);
     void set_ss_application_ids(AppId client, AppId payload, AppidChangeBits& change_bits);
diff --git a/src/network_inspectors/appid/detector_plugins/detector_sip.cc b/src/network_inspectors/appid/detector_plugins/detector_sip.cc
index 3b8daad8b..0c4713637 100644
--- a/src/network_inspectors/appid/detector_plugins/detector_sip.cc
+++ b/src/network_inspectors/appid/detector_plugins/detector_sip.cc
@@ -315,15 +315,16 @@ SipServiceDetector* SipEventHandler::service = nullptr;
 
 void SipEventHandler::handle(DataEvent& event, Flow* flow)
 {
-    SipEvent& sip_event = (SipEvent&)event;
-    AppIdSession* asd = nullptr;
+    if (!flow)
+        return;
 
-    if ( flow )
-        asd = appid_api.get_appid_session(*flow);
+    SipEvent& sip_event = (SipEvent&)event;
+    AppIdSession* asd = appid_api.get_appid_session(*flow);
 
+    const Packet* p = sip_event.get_packet();
+    assert(p);
     if ( !asd )
     {
-        const Packet* p = sip_event.get_packet();
         IpProtocol protocol = p->is_tcp() ? IpProtocol::TCP : IpProtocol::UDP;
         AppidSessionDirection direction = p->is_from_client() ? APP_ID_FROM_INITIATOR : APP_ID_FROM_RESPONDER;
         AppIdInspector* inspector = (AppIdInspector*) InspectorManager::get_inspector(MOD_NAME, true);
@@ -333,7 +334,7 @@ void SipEventHandler::handle(DataEvent& event, Flow* flow)
     AppidChangeBits change_bits;
     client_handler(sip_event, *asd, change_bits);
     service_handler(sip_event, *asd, change_bits);
-    asd->publish_appid_event(change_bits, flow);
+    asd->publish_appid_event(change_bits, *p);
 }
 
 void SipEventHandler::client_handler(SipEvent& sip_event, AppIdSession& asd,
diff --git a/src/network_inspectors/appid/detector_plugins/test/http_url_patterns_test.cc b/src/network_inspectors/appid/detector_plugins/test/http_url_patterns_test.cc
index c9b85e405..95e5023a4 100644
--- a/src/network_inspectors/appid/detector_plugins/test/http_url_patterns_test.cc
+++ b/src/network_inspectors/appid/detector_plugins/test/http_url_patterns_test.cc
@@ -56,6 +56,12 @@ static AppId service_id = APP_ID_NONE;
 static AppId client_id = APP_ID_NONE;
 static DetectorHTTPPattern mpattern;
 
+namespace snort
+{
+AppIdSessionApi::AppIdSessionApi(const AppIdSession*, const SfIp&) :
+    StashGenericObject(STASH_GENERIC_OBJECT_APPID) {}
+}
+
 void ApplicationDescriptor::set_id(const Packet&, AppIdSession&, AppidSessionDirection, AppId, AppidChangeBits&) { }
 AppIdDiscovery::AppIdDiscovery() { }
 AppIdDiscovery::~AppIdDiscovery() { }
diff --git a/src/network_inspectors/appid/lua_detector_flow_api.cc b/src/network_inspectors/appid/lua_detector_flow_api.cc
index 2292bc161..4bcb57753 100644
--- a/src/network_inspectors/appid/lua_detector_flow_api.cc
+++ b/src/network_inspectors/appid/lua_detector_flow_api.cc
@@ -355,8 +355,7 @@ static int get_detector_flow_key(lua_State* L)
     // Verify detector user data and that we are in packet context
     assert(pLuaData.ptr);
 
-    lua_pushlstring(L, (char*)&pLuaData->asd->session_id,
-        sizeof(pLuaData->asd->session_id));
+    lua_pushstring(L, pLuaData->asd->get_api().get_session_id().c_str());
 
     return 1;
 }
diff --git a/src/network_inspectors/appid/service_plugins/service_direct_connect.cc b/src/network_inspectors/appid/service_plugins/service_direct_connect.cc
index 306a04529..51a81a561 100644
--- a/src/network_inspectors/appid/service_plugins/service_direct_connect.cc
+++ b/src/network_inspectors/appid/service_plugins/service_direct_connect.cc
@@ -128,14 +128,12 @@ int DirectConnectServiceDetector::tcp_validate(const uint8_t* data, uint16_t siz
         {
             if (memcmp(data, PATTERN1, sizeof(PATTERN1)-1) == 0)
             {
-                printf("maybe first directconnect to hub  detected\n");
                 serviceData->state = CONN_STATE_1;
                 goto inprocess;
             }
 
             if (memcmp(data, PATTERN2, sizeof(PATTERN2)-1) == 0)
             {
-                printf("maybe first dc connect between peers  detected\n");
                 serviceData->state = CONN_STATE_2;
                 goto inprocess;
             }
@@ -154,7 +152,6 @@ int DirectConnectServiceDetector::tcp_validate(const uint8_t* data, uint16_t siz
         break;
 
     case CONN_STATE_1:
-        printf ("ValidateDirectConnectTcp(): state 1 size %d\n", size);
         if (size >= 11)
         {
             if (memcmp(data, PATTERN3, sizeof(PATTERN3)-1) == 0
@@ -162,7 +159,6 @@ int DirectConnectServiceDetector::tcp_validate(const uint8_t* data, uint16_t siz
                 || memcmp(data, PATTERN5, sizeof(PATTERN5)-1) == 0
                 || memcmp(data, PATTERN6, sizeof(PATTERN6)-1) == 0)
             {
-                printf("found directconnect HSUP ADBAS E in second packet\n");
                 goto success;
             }
         }
diff --git a/src/network_inspectors/appid/service_plugins/test/service_plugin_mock.h b/src/network_inspectors/appid/service_plugins/test/service_plugin_mock.h
index 9ea196348..51e235f32 100644
--- a/src/network_inspectors/appid/service_plugins/test/service_plugin_mock.h
+++ b/src/network_inspectors/appid/service_plugins/test/service_plugin_mock.h
@@ -75,6 +75,9 @@ void Module::show_interval_stats(std::vector<unsigned int, std::allocator<unsign
 void Module::show_stats(){}
 void Module::sum_stats(bool ){}
 void Module::reset_stats() {}
+
+AppIdSessionApi::AppIdSessionApi(const AppIdSession*, const SfIp&) :
+    StashGenericObject(STASH_GENERIC_OBJECT_APPID) {}
 }
 
 SslPatternMatchers::~SslPatternMatchers() { }
diff --git a/src/network_inspectors/appid/test/appid_api_test.cc b/src/network_inspectors/appid/test/appid_api_test.cc
index d10053ed3..f2936b3a0 100644
--- a/src/network_inspectors/appid/test/appid_api_test.cc
+++ b/src/network_inspectors/appid/test/appid_api_test.cc
@@ -54,6 +54,17 @@ namespace snort
 class Inspector* InspectorManager::get_inspector(const char*, bool, const SnortConfig*)
 { return &dummy_appid_inspector; }
 
+Packet::Packet(bool) { }
+Packet::~Packet() { }
+
+Packet* DetectionEngine::get_current_packet()
+{
+    static Packet p;
+    return &p;
+}
+
+AppIdSessionApi::AppIdSessionApi(const AppIdSession*, const SfIp&) :
+    StashGenericObject(STASH_GENERIC_OBJECT_APPID) {}
 }
 
 class DummyInspector : public snort::Inspector
@@ -71,10 +82,10 @@ void DataBus::publish(const char*, DataEvent& event, Flow*)
     mock().actualCall("publish");
 }
 
-void AppIdSession::publish_appid_event(AppidChangeBits& change_bits, Flow* flow, bool, uint32_t)
+void AppIdSession::publish_appid_event(AppidChangeBits& change_bits, const Packet& p, bool, uint32_t)
 {
-    AppidEvent app_event(change_bits, false, 0, this->get_api());
-    DataBus::publish(APPID_EVENT_ANY_CHANGE, app_event, flow);
+    AppidEvent app_event(change_bits, false, 0, this->get_api(), p);
+    DataBus::publish(APPID_EVENT_ANY_CHANGE, app_event, p.flow);
 }
 
 bool SslPatternMatchers::scan_hostname(const uint8_t* server_name, size_t, AppId& client_id, AppId& payload_id)
diff --git a/src/network_inspectors/appid/test/appid_debug_test.cc b/src/network_inspectors/appid/test/appid_debug_test.cc
index 5e6187fa4..5c23d2fc6 100644
--- a/src/network_inspectors/appid/test/appid_debug_test.cc
+++ b/src/network_inspectors/appid/test/appid_debug_test.cc
@@ -43,6 +43,8 @@ unsigned get_instance_id() { return 3; }
 
 FlowData::FlowData(unsigned, Inspector*) { }
 FlowData::~FlowData() = default;
+AppIdSessionApi::AppIdSessionApi(const AppIdSession* asd, const SfIp& ip) :
+    StashGenericObject(STASH_GENERIC_OBJECT_APPID), asd(asd), initiator_ip(ip) {}
 }
 
 void ApplicationDescriptor::set_id(const Packet&, AppIdSession&, AppidSessionDirection, AppId, AppidChangeBits&) { }
diff --git a/src/network_inspectors/appid/test/appid_detector_test.cc b/src/network_inspectors/appid/test/appid_detector_test.cc
index 175aac107..e38a03dd7 100644
--- a/src/network_inspectors/appid/test/appid_detector_test.cc
+++ b/src/network_inspectors/appid/test/appid_detector_test.cc
@@ -40,6 +40,8 @@ namespace snort
 {
 Inspector* InspectorManager::get_inspector(
     char const*, bool, const snort::SnortConfig*) { return nullptr; }
+AppIdSessionApi::AppIdSessionApi(const AppIdSession*, const SfIp&) :
+    StashGenericObject(STASH_GENERIC_OBJECT_APPID) {}
 }
 
 void ApplicationDescriptor::set_id(
diff --git a/src/network_inspectors/appid/test/appid_discovery_test.cc b/src/network_inspectors/appid/test/appid_discovery_test.cc
index bf0c3b760..709a22c3c 100644
--- a/src/network_inspectors/appid/test/appid_discovery_test.cc
+++ b/src/network_inspectors/appid/test/appid_discovery_test.cc
@@ -103,6 +103,8 @@ void IpApi::set(const SfIp& sip, const SfIp& dip)
 }
 } // namespace ip
 
+AppIdSessionApi::AppIdSessionApi(const AppIdSession*, const SfIp&) :
+    StashGenericObject(STASH_GENERIC_OBJECT_APPID) {}
 } // namespace snort
 
 // Stubs for publish
@@ -204,10 +206,10 @@ AppIdSession* AppIdSession::allocate_session(const Packet*, IpProtocol,
     return nullptr;
 }
 
-void AppIdSession::publish_appid_event(AppidChangeBits& change_bits, Flow* flow, bool, uint32_t)
+void AppIdSession::publish_appid_event(AppidChangeBits& change_bits, const Packet& p, bool, uint32_t)
 {
-    AppidEvent app_event(change_bits, false, 0, this->get_api());
-    DataBus::publish(APPID_EVENT_ANY_CHANGE, app_event, flow);
+    AppidEvent app_event(change_bits, false, 0, this->get_api(), p);
+    DataBus::publish(APPID_EVENT_ANY_CHANGE, app_event, p.flow);
 }
 
 void AppIdHttpSession::set_tun_dest(){}
diff --git a/src/network_inspectors/appid/test/appid_http_event_test.cc b/src/network_inspectors/appid/test/appid_http_event_test.cc
index 358327828..378a2dc1c 100644
--- a/src/network_inspectors/appid/test/appid_http_event_test.cc
+++ b/src/network_inspectors/appid/test/appid_http_event_test.cc
@@ -52,6 +52,18 @@ namespace snort
 AppIdApi appid_api;
 Inspector* InspectorManager::get_inspector(
     char const*, bool, const snort::SnortConfig*) { return nullptr; }
+
+Packet::Packet(bool) { }
+Packet::~Packet() { }
+
+Packet* DetectionEngine::get_current_packet()
+{
+    static Packet p;
+    return &p;
+}
+
+AppIdSessionApi::AppIdSessionApi(const AppIdSession*, const SfIp&) :
+    StashGenericObject(STASH_GENERIC_OBJECT_APPID) {}
 }
 
 const char* content_type = nullptr;
@@ -227,7 +239,7 @@ AppIdSession* AppIdApi::get_appid_session(const Flow&)
     return mock_session;
 }
 
-void AppIdSession::publish_appid_event(AppidChangeBits&, Flow*, bool, uint32_t) { }
+void AppIdSession::publish_appid_event(AppidChangeBits&, const Packet&, bool, uint32_t) { }
 
 TEST_GROUP(appid_http_event)
 {
diff --git a/src/network_inspectors/appid/test/appid_http_session_test.cc b/src/network_inspectors/appid/test/appid_http_session_test.cc
index c8e6c5163..a21073293 100644
--- a/src/network_inspectors/appid/test/appid_http_session_test.cc
+++ b/src/network_inspectors/appid/test/appid_http_session_test.cc
@@ -47,6 +47,12 @@
 #include <CppUTest/TestHarness.h>
 using namespace snort;
 
+namespace snort
+{
+AppIdSessionApi::AppIdSessionApi(const AppIdSession*, const SfIp&) :
+    StashGenericObject(STASH_GENERIC_OBJECT_APPID) {}
+}
+
 void ApplicationDescriptor::set_id(const Packet&, AppIdSession&, AppidSessionDirection, AppId, AppidChangeBits&) { }
 const char* AppInfoManager::get_app_name(AppId)
 {
diff --git a/src/network_inspectors/appid/test/appid_session_api_test.cc b/src/network_inspectors/appid/test/appid_session_api_test.cc
index 2045c5872..7d28ae086 100644
--- a/src/network_inspectors/appid/test/appid_session_api_test.cc
+++ b/src/network_inspectors/appid/test/appid_session_api_test.cc
@@ -37,6 +37,14 @@ AppIdSessionApi* appid_session_api = nullptr;
 static AppIdConfig config;
 static OdpContext odpctxt(config, nullptr);
 
+namespace snort
+{
+    unsigned get_instance_id()
+    {
+        return 0;
+    }
+}
+
 void AppIdSession::set_ss_application_ids(AppId service_id, AppId client_id, AppId payload_id,
     AppId misc_id, AppId referred_id, AppidChangeBits& change_bits)
 {
diff --git a/src/network_inspectors/appid/test/service_state_test.cc b/src/network_inspectors/appid/test/service_state_test.cc
index bf3143e12..7728fa263 100644
--- a/src/network_inspectors/appid/test/service_state_test.cc
+++ b/src/network_inspectors/appid/test/service_state_test.cc
@@ -52,6 +52,9 @@ char* snort_strdup(const char* str)
     return p;
 }
 time_t packet_time() { return std::time(0); }
+
+AppIdSessionApi::AppIdSessionApi(const AppIdSession*, const SfIp&) :
+    StashGenericObject(STASH_GENERIC_OBJECT_APPID) {}
 }
 
 // Stubs for AppInfoManager
diff --git a/src/pub_sub/appid_events.h b/src/pub_sub/appid_events.h
index 2e76377fa..c4f4a704d 100644
--- a/src/pub_sub/appid_events.h
+++ b/src/pub_sub/appid_events.h
@@ -108,8 +108,8 @@ class AppidEvent : public snort::DataEvent
 {
 public:
     AppidEvent(const AppidChangeBits& ac, bool is_http2, uint32_t http2_stream_index,
-        const snort::AppIdSessionApi& api) :
-        ac_bits(ac), is_http2(is_http2), http2_stream_index(http2_stream_index), api(api) {}
+        const snort::AppIdSessionApi& api, const snort::Packet& p) :
+        ac_bits(ac), is_http2(is_http2), http2_stream_index(http2_stream_index), api(api), p(p) {}
 
     const AppidChangeBits& get_change_bitset() const
     { return ac_bits; }
@@ -123,11 +123,15 @@ public:
     const snort::AppIdSessionApi& get_appid_session_api() const
     { return api; }
 
+    const snort::Packet* get_packet() override
+    { return &p; }
+
 private:
     const AppidChangeBits& ac_bits;
     bool is_http2;
     uint32_t http2_stream_index;
     const snort::AppIdSessionApi& api;
+    const snort::Packet& p;
 };
 
 #endif
diff --git a/src/utils/util.cc b/src/utils/util.cc
index d7e0f527f..2c4208038 100644
--- a/src/utils/util.cc
+++ b/src/utils/util.cc
@@ -86,17 +86,6 @@ void StoreSnortInfoStrings()
 #undef SNORT_VERSION_STRING
 #undef SNORT_VERSION_STRLEN
 
-/****************************************************************************
- *
- * Function: DisplayBanner()
- *
- * Purpose:  Show valuable proggie info
- *
- * Arguments: None.
- *
- * Returns: 0 all the time
- *
- ****************************************************************************/
 int DisplayBanner()
 {
     const char* ljv = LUAJIT_VERSION;
@@ -131,85 +120,7 @@ int DisplayBanner()
     return 0;
 }
 
-/****************************************************************************
- *
- * Function: ts_print(const struct, char *)
- *
- * Purpose: Generate a time stamp and stuff it in a buffer.  This one has
- *          millisecond precision.  Oh yeah, I ripped this code off from
- *          TCPdump, props to those guys.
- *
- * Arguments: timeval => clock struct coming out of libpcap
- *            timebuf => buffer to stuff timestamp into
- *
- * Returns: void function
- *
- ****************************************************************************/
-void ts_print(const struct timeval* tvp, char* timebuf)
-{
-    struct timeval tv;
-    struct timezone tz;
-
-    /* if null was passed, we use current time */
-    if (!tvp)
-    {
-        /* manual page (for linux) says tz is never used, so.. */
-        memset((char*)&tz, 0, sizeof(tz));
-        gettimeofday(&tv, &tz);
-        tvp = &tv;
-    }
-
-    const SnortConfig* sc = SnortConfig::get_conf();
-    int localzone = sc->thiszone;
-
-    /*
-    **  If we're doing UTC, then make sure that the timezone is correct.
-    */
-    if (sc->output_use_utc())
-        localzone = 0;
-
-    int s = (tvp->tv_sec + localzone) % SECONDS_PER_DAY;
-    time_t Time = (tvp->tv_sec + localzone) - s;
-
-    struct tm ttm;
-    struct tm* lt = gmtime_r(&Time, &ttm);
-
-    if ( !lt )
-    {
-        (void)SnortSnprintf(timebuf, TIMEBUF_SIZE, "%lu", tvp->tv_sec);
-
-    }
-    else if (sc->output_include_year())
-    {
-        int year = (lt->tm_year >= 100) ? (lt->tm_year - 100) : lt->tm_year;
-
-        (void)SnortSnprintf(timebuf, TIMEBUF_SIZE,
-            "%02d/%02d/%02d-%02d:%02d:%02d.%06u",
-            year, lt->tm_mon + 1, lt->tm_mday,
-            s / 3600, (s % 3600) / 60, s % 60,
-            (unsigned)tvp->tv_usec);
-    }
-    else
-    {
-        (void)SnortSnprintf(timebuf, TIMEBUF_SIZE,
-            "%02d/%02d-%02d:%02d:%02d.%06u", lt->tm_mon + 1,
-            lt->tm_mday, s / 3600, (s % 3600) / 60, s % 60,
-            (unsigned)tvp->tv_usec);
-    }
-}
-
-/****************************************************************************
- *
- * Function: gmt2local(time_t)
- *
- * Purpose: Figures out how to adjust the current clock reading based on the
- *          timezone you're in.  Ripped off from TCPdump.
- *
- * Arguments: time_t => offset from GMT
- *
- * Returns: offset seconds from GMT
- *
- ****************************************************************************/
+// get offset seconds from GMT
 int gmt2local(time_t t)
 {
     if (t == 0)
@@ -297,17 +208,6 @@ void CreatePidFile(pid_t pid)
         unlink(pid_lockfilename.c_str());
 }
 
-/****************************************************************************
- *
- * Function: ClosePidFile(char *)
- *
- * Purpose:  Releases lock on a PID file
- *
- * Arguments: None
- *
- * Returns: void function
- *
- ****************************************************************************/
 void ClosePidFile()
 {
     if (pid_file)
@@ -322,17 +222,7 @@ void ClosePidFile()
     }
 }
 
-/****************************************************************************
- *
- * Function: SetUidGid()
- *
- * Purpose:  Sets safe UserID and GroupID if needed
- *
- * Arguments: none
- *
- * Returns: void function
- *
- ****************************************************************************/
+// set safe UserID and GroupID, if needed
 bool SetUidGid(int user_id, int group_id)
 {
     // Were any changes requested?
@@ -362,18 +252,7 @@ bool SetUidGid(int user_id, int group_id)
     return true;
 }
 
-/****************************************************************************
- *
- * Function: InitGroups()
- *
- * Purpose:  Sets the groups of the process based on the UserID with the
- *           GroupID added
- *
- * Arguments: none
- *
- * Returns: void function
- *
- ****************************************************************************/
+// set the groups of the process based on the UserID with the GroupID added
 void InitGroups(int user_id, int group_id)
 {
     if ((user_id != -1) && (getuid() == 0))
@@ -442,17 +321,7 @@ void CleanupProtoNames()
     }
 }
 
-/****************************************************************************
- *
- * Function: read_infile(const char* key, const char* file)
- *
- * Purpose: Reads the BPF filters in from a file.  Ripped from tcpdump.
- *
- * Arguments: fname => the name of the file containing the BPF filters
- *
- * Returns: the processed BPF string
- *
- ****************************************************************************/
+// read the BPF filters in from a file, return the processed BPF string
 std::string read_infile(const char* key, const char* fname)
 {
     int fd = open(fname, O_RDONLY);
@@ -525,9 +394,7 @@ static char* GetAbsolutePath(const char* dir, PathBuf& buf)
     return buf;
 }
 
-/**
- * Chroot and adjust the log_dir reference
- */
+// Chroot and adjust the log_dir reference
 bool EnterChroot(std::string& root_dir, std::string& log_dir)
 {
     if (log_dir.empty())
@@ -641,6 +508,64 @@ char* snort_strdup(const char* str)
     return p;
 }
 
+void ts_print(const struct timeval* tvp, char* timebuf, bool yyyymmdd)
+{
+    struct timeval tv;
+    struct timezone tz;
+
+    // if null was passed, use current time
+    if (!tvp)
+    {
+        // manual page (for linux) says tz is never used, so..
+        memset((char*)&tz, 0, sizeof(tz));
+        gettimeofday(&tv, &tz);
+        tvp = &tv;
+    }
+
+    const SnortConfig* sc = SnortConfig::get_conf();
+    int localzone = sc->thiszone;
+
+    // If we're doing UTC, then make sure that the timezone is correct.
+    if (sc->output_use_utc())
+        localzone = 0;
+
+    int s = (tvp->tv_sec + localzone) % SECONDS_PER_DAY;
+    time_t Time = (tvp->tv_sec + localzone) - s;
+
+    struct tm ttm;
+    struct tm* lt = gmtime_r(&Time, &ttm);
+
+    if ( !lt )
+    {
+        (void)SnortSnprintf(timebuf, TIMEBUF_SIZE, "%lu", tvp->tv_sec);
+
+    }
+    else if (sc->output_include_year())
+    {
+        int year = (lt->tm_year >= 100) ? (lt->tm_year - 100) : lt->tm_year;
+
+        (void)SnortSnprintf(timebuf, TIMEBUF_SIZE,
+            "%02d/%02d/%02d-%02d:%02d:%02d.%06u",
+            year, lt->tm_mon + 1, lt->tm_mday,
+            s / 3600, (s % 3600) / 60, s % 60,
+            (unsigned)tvp->tv_usec);
+    }
+    else if (yyyymmdd)
+    {
+        (void)SnortSnprintf(timebuf, TIMEBUF_SIZE,
+            "%04d-%02d-%02d %02d:%02d:%02d.%06u",
+            lt->tm_year + 1900, lt->tm_mon + 1, lt->tm_mday,
+            s / 3600, (s % 3600) / 60, s % 60,
+            (unsigned)tvp->tv_usec);
+    }
+    else
+    {
+        (void)SnortSnprintf(timebuf, TIMEBUF_SIZE,
+            "%02d/%02d-%02d:%02d:%02d.%06u", lt->tm_mon + 1,
+            lt->tm_mday, s / 3600, (s % 3600) / 60, s % 60,
+            (unsigned)tvp->tv_usec);
+    }
+}
 }
 
 #ifdef UNIT_TEST
diff --git a/src/utils/util.h b/src/utils/util.h
index a0896c0aa..b96299674 100644
--- a/src/utils/util.h
+++ b/src/utils/util.h
@@ -36,7 +36,7 @@
 
 #include "main/snort_types.h"
 
-#define TIMEBUF_SIZE 26
+#define TIMEBUF_SIZE 27
 
 #define SECONDS_PER_DAY  86400  /* number of seconds in a day  */
 #define SECONDS_PER_HOUR  3600  /* number of seconds in a hour */
@@ -45,7 +45,6 @@
 void StoreSnortInfoStrings();
 int DisplayBanner();
 int gmt2local(time_t);
-void ts_print(const struct timeval*, char*);
 std::string read_infile(const char* key, const char* fname);
 void CleanupProtoNames();
 void CreatePidFile(pid_t);
@@ -107,6 +106,7 @@ SO_PUBLIC extern char** protocol_names;
 SO_PUBLIC const char* get_error(int errnum);
 SO_PUBLIC char* snort_strdup(const char*);
 SO_PUBLIC char* snort_strndup(const char*, size_t);
+SO_PUBLIC void ts_print(const struct timeval*, char*, bool yyyymmdd = false);
 }
 
 #endif