From: Russ Combs (rucombs) Date: Fri, 18 Aug 2017 14:53:04 +0000 (-0400) Subject: Merge pull request #994 in SNORT/snort3 from elk to master X-Git-Tag: 3.0.0-240~64 X-Git-Url: http://git.ipfire.org/gitweb.cgi?a=commitdiff_plain;h=d4a237e3777d2dbaf2da01d780e9ca85ce2ccc37;p=thirdparty%2Fsnort3.git Merge pull request #994 in SNORT/snort3 from elk to master Squashed commit of the following: commit 573659a4166f5e1c9583383d7bf0bddbee4472a5 Author: Russ Combs Date: Thu Aug 17 15:22:54 2017 -0400 csv: updates commit d566f6bb98497dd76baea1b88f451509a7291b96 Author: Russ Combs Date: Thu Aug 17 15:11:50 2017 -0400 b64: updates commit c25181487233e22511dcd7d5c2f06ba2ad6cf5a2 Author: Russ Combs Date: Wed Aug 16 13:45:31 2017 -0400 appid: convert appid_stats.log from u2 to csv commit c882db6d3c1901bb3f42e38f733cb70632f3139e Author: Russ Combs Date: Wed Aug 16 11:48:23 2017 -0400 configure: add --disable-stdlog for cases where logging alerts to file descriptor 3 is unhelpful commit 6d3c8c3d4c365e8d99866ada71ecc1b039e2c00b Author: Russ Combs Date: Wed Aug 16 11:22:48 2017 -0400 cleanup: remove rogue HAVE_CONFIG_H from includes commit d302999d9e784d6a6e2fe8e18514be33b2a3d470 Author: Russ Combs Date: Wed Aug 16 11:17:54 2017 -0400 appid: tweak help for instance_id commit 2b135326f84f4047aaef5336c7bc31948607d1d5 Author: Russ Combs Date: Mon Aug 14 12:56:28 2017 -0400 stream_tcp: ensure max pdu is flushed by default splitter commit 7f5eb5649c0f7012c518b5197c77c4b320407841 Author: Russ Combs Date: Sun Aug 13 20:39:22 2017 -0400 unified2: log buffers as cooked packets with legacy events commit 946b93bdb7aa35d23b259cb769e2eac940254ad7 Author: Russ Combs Date: Sun Aug 13 07:32:07 2017 -0400 build: fix unused parameter warning commit 4c7e0c61a43102c803c34ca22fe919e2998a941b Author: Russ Combs Date: Sat Aug 12 16:05:10 2017 -0400 conf: fix default classification capitalization commit a9ccc441b02898f6c9e6471404ced30a1f17f312 Author: Russ Combs Date: Sun Aug 13 07:28:57 2017 -0400 alert_csv: add vlan and mpls options commit 666b46fd3a976da03840aedf93f5522a639e0de9 Author: Russ Combs Date: Sat Aug 12 10:25:32 2017 -0400 alert_csv: add b64_data, rename dgm_len to pkt_len commit 64889ec87c4a8c953742a7a828d968334439ef48 Author: Russ Combs Date: Sat Aug 12 10:24:04 2017 -0400 loggers: add base64 encoder based on libb64 from devolve commit dfb8e204bcc8d6232b51fdd47ef91ac5d4609f8d Author: Russ Combs Date: Fri Aug 11 22:32:17 2017 -0400 alert_csv: add class, priority, and service options commit c39d12177bf9a4f304a2801dca1c4edb9b3b8f18 Author: Russ Combs Date: Fri Aug 11 22:32:01 2017 -0400 loggers: use standard year/mon/day format --- diff --git a/cmake/configure_options.cmake b/cmake/configure_options.cmake index 2348556fe..0513ca604 100644 --- a/cmake/configure_options.cmake +++ b/cmake/configure_options.cmake @@ -12,6 +12,7 @@ if ( NOT ENABLE_COREFILES ) endif ( NOT ENABLE_COREFILES ) set ( _LARGEFILE_SOURCE ${ENABLE_LARGE_PCAP} ) +set ( USE_STDLOG ${ENABLE_STDLOG} ) set ( USE_TSC_CLOCK ${ENABLE_TSC_CLOCK} ) if ( ENABLE_LARGE_PCAP ) diff --git a/cmake/create_options.cmake b/cmake/create_options.cmake index 062447b05..531bd8e3b 100644 --- a/cmake/create_options.cmake +++ b/cmake/create_options.cmake @@ -21,6 +21,7 @@ option ( ENABLE_PIGLET "enable piglet test harness" OFF ) option ( ENABLE_COREFILES "Prevent Snort from generating core files" ON ) option ( ENABLE_LARGE_PCAP "Enable support for pcaps larger than 2 GB" OFF ) +option ( ENABLE_STDLOG "Prefer file descriptor 3 over stdout for alerts" ON ) option ( ENABLE_TSC_CLOCK "Use timestamp counter register clock (x86 only)" OFF ) # documentation diff --git a/config.cmake.h.in b/config.cmake.h.in index 6e4e1af33..bf14693cb 100644 --- a/config.cmake.h.in +++ b/config.cmake.h.in @@ -88,7 +88,10 @@ #cmakedefine _LARGEFILE_SOURCE 1 #cmakedefine _FILE_OFFSET_BITS @_FILE_OFFSET_BITS@ -/* enable ha capable build */ +/* enable stdlog */ +#cmakedefine USE_STDLOG 1 + +/* enable tsc clock */ #cmakedefine USE_TSC_CLOCK 1 diff --git a/configure.ac b/configure.ac index a3f7179bc..2fbd32853 100644 --- a/configure.ac +++ b/configure.ac @@ -325,6 +325,14 @@ fi AM_CONDITIONAL(ENABLE_SHELL, test "x$enable_shell" = "xyes") +AC_ARG_ENABLE(stdlog, + AS_HELP_STRING([--disable-stdlog],[do not prefer file descriptor 3 over stdout for alerts]), + enable_stdlog="$enableval", enable_stdlog="yes") + +if test "x$enable_stdlog" = "xyes"; then + AC_DEFINE(USE_STDLOG, [1], [enable stdlog]) +fi + AC_ARG_ENABLE(tsc-clock, AS_HELP_STRING([--enable-tsc-clock],[use timestamp counter register clock (x86 only)]), enable_tsc_clock="$enableval", enable_tsc_clock="no") diff --git a/configure_cmake.sh b/configure_cmake.sh index 7fc410e27..a9d0cfb88 100755 --- a/configure_cmake.sh +++ b/configure_cmake.sh @@ -40,6 +40,7 @@ Optional Features: --disable-static-codecs do not include codecs in binary --enable-shell enable command line shell support --enable-large-pcap enable support for pcaps larger than 2 GB + --disable-stdlog do not prefer file descriptor 3 over stdout for alerts --enable-tsc-clock use timestamp counter register clock (x86 only) --enable-debug-msgs enable debug printing options (bugreports and developers only) @@ -224,6 +225,9 @@ while [ $# -ne 0 ]; do --enable-large-pcap) append_cache_entry ENABLE_LARGE_PCAP BOOL true ;; + --disable-stdlog) + append_cache_entry ENABLE_STDLOG BOOL false + ;; --enable-tsc-clock) append_cache_entry ENABLE_TSC_CLOCK BOOL true ;; diff --git a/lua/snort_defaults.lua b/lua/snort_defaults.lua index 3c3ad1f06..a302c70c1 100644 --- a/lua/snort_defaults.lua +++ b/lua/snort_defaults.lua @@ -470,7 +470,7 @@ default_classifications = text = 'Generic Protocol Command Decode' }, { name = 'web-application-activity', priority = 2, - text = 'access to a potentially vulnerable web application' }, + text = 'Access to a potentially vulnerable web application' }, { name = 'web-application-attack', priority = 1, text = 'Web Application Attack' }, diff --git a/src/detection/detect_trace.cc b/src/detection/detect_trace.cc index db345dfe5..ced91eaca 100644 --- a/src/detection/detect_trace.cc +++ b/src/detection/detect_trace.cc @@ -18,6 +18,10 @@ // detect_trace.cc author Maya Dagon +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + #include "detect_trace.h" #include diff --git a/src/detection/detect_trace.h b/src/detection/detect_trace.h index 1cb034d1b..9dea9b8a1 100644 --- a/src/detection/detect_trace.h +++ b/src/detection/detect_trace.h @@ -23,10 +23,6 @@ // Detection trace utility -#ifdef HAVE_CONFIG_H -#include "config.h" -#endif - #include "framework/cursor.h" #include "main/snort_types.h" diff --git a/src/helpers/CMakeLists.txt b/src/helpers/CMakeLists.txt index eadd658d5..629738224 100644 --- a/src/helpers/CMakeLists.txt +++ b/src/helpers/CMakeLists.txt @@ -1,4 +1,6 @@ add_library (helpers STATIC + base64_encoder.cc + base64_encoder.h chunk.cc chunk.h directory.cc diff --git a/src/helpers/Makefile.am b/src/helpers/Makefile.am index 6300b7229..af37e1f8c 100644 --- a/src/helpers/Makefile.am +++ b/src/helpers/Makefile.am @@ -1,9 +1,9 @@ noinst_LIBRARIES = libhelpers.a -x_includedir = $(pkgincludedir)/helpers - libhelpers_a_SOURCES = \ +base64_encoder.cc \ +base64_encoder.h \ chunk.cc \ chunk.h \ directory.cc \ diff --git a/src/helpers/base64_encoder.cc b/src/helpers/base64_encoder.cc new file mode 100644 index 000000000..74b8f8849 --- /dev/null +++ b/src/helpers/base64_encoder.cc @@ -0,0 +1,204 @@ +//-------------------------------------------------------------------------- +// Copyright (C) 2017-2017 Cisco and/or its affiliates. All rights reserved. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License Version 2 as published +// by the Free Software Foundation. You may not use, modify or distribute +// this program under any other version of the GNU General Public License. +// +// This program is distributed in the hope that it will be useful, but +// WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with this program; if not, write to the Free Software Foundation, Inc., +// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +//-------------------------------------------------------------------------- + +// base64_encoder.cc author Russ Combs + +// this is based on the excellent work by devolve found at +// https://sourceforge.net/projects/libb64/. + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "base64_encoder.h" + +#include + +#ifdef UNIT_TEST +#include +#include "catch/catch.hpp" +#endif + +void keep_base64_encoder() { } + +static inline char b64(uint8_t idx) +{ + static const char* encoding = + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + assert(idx < 64); + return encoding[idx]; +} + +unsigned Base64Encoder::encode( + const uint8_t* plain_text, unsigned length, char* buf) +{ + const uint8_t* data = plain_text; + const uint8_t* const data_end = plain_text + length; + char* p = buf; + uint8_t fragment; + + switch (step) + { + while (true) + { + case step_A: + if (data == data_end) + { + step = step_A; + return p - buf; + } + fragment = *data++; + state = (fragment & 0x0fc) >> 2; + *p++ = b64(state); + state = (fragment & 0x003) << 4; + case step_B: + if (data == data_end) + { + step = step_B; + return p - buf; + } + fragment = *data++; + state |= (fragment & 0x0f0) >> 4; + *p++ = b64(state); + state = (fragment & 0x00f) << 2; + case step_C: + if (data == data_end) + { + step = step_C; + return p - buf; + } + fragment = *data++; + state |= (fragment & 0x0c0) >> 6; + *p++ = b64(state); + state = (fragment & 0x03f) >> 0; + *p++ = b64(state); + } + } + /* control should not reach here */ + assert(false); + return p - buf; +} + +unsigned Base64Encoder::finish(char* buf) +{ + char* p = buf; + + switch (step) + { + case step_B: + *p++ = b64(state); + *p++ = '='; + *p++ = '='; + break; + case step_C: + *p++ = b64(state); + *p++ = '='; + break; + case step_A: + break; + } + return p - buf; +} + +//-------------------------------------------------------------------------- +// unit tests +// code string generated with: echo | base64 - +// which adds a \n to the input. +//-------------------------------------------------------------------------- + +#ifdef UNIT_TEST +TEST_CASE("b64 decode", "[Base64Encoder]") +{ + Base64Encoder b64; + + const char* text = "The quick brown segment jumped over the lazy dogs.\n"; + const char* code = "VGhlIHF1aWNrIGJyb3duIHNlZ21lbnQganVtcGVkIG92ZXIgdGhlIGxhenkgZG9ncy4K"; + + char buf[256]; + + SECTION("no decode") + { + CHECK(!b64.finish(buf)); + } + SECTION("null data") + { + CHECK(!b64.encode(nullptr, 0, buf)); + CHECK(!b64.finish(buf)); + } + SECTION("zero length data") + { + CHECK(!b64.encode((uint8_t*)"ignore", 0, buf)); + CHECK(!b64.finish(buf)); + } + SECTION("finish states") + { + const char* txt[] = { "test0\n", "test01\n", "test012\n" }; + const char* exp[] = { "dGVzdDAK", "dGVzdDAxCg==", "dGVzdDAxMgo=" }; + + const unsigned to_do = sizeof(txt)/sizeof(txt[0]); + + for ( unsigned i = 0; i < to_do; ++i ) + { + unsigned n = b64.encode((uint8_t*)txt[i], strlen(txt[i]), buf); + n += b64.finish(buf+n); + + REQUIRE(n < sizeof(buf)); + buf[n] = 0; + + CHECK(!strcmp(buf, exp[i])); + b64.reset(); + } + } + SECTION("one shot") + { + unsigned n = b64.encode((uint8_t*)text, strlen(text), buf); + n += b64.finish(buf+n); + + REQUIRE(n < sizeof(buf)); + buf[n] = 0; + + CHECK(!strcmp(buf, code)); + } + SECTION("slice and dice") + { + unsigned len = strlen(text); + + for ( unsigned chunk = 1; chunk < len; ++chunk ) + { + memset(buf, 0, sizeof(buf)); + unsigned offset = 0; + unsigned n = 0; + + while ( offset < len ) + { + unsigned k = (offset + chunk > len) ? len - offset : chunk; + n += b64.encode((uint8_t*)text+offset, k, buf+n); + offset += k; + } + n += b64.finish(buf+n); + + REQUIRE(n < sizeof(buf)); + buf[n] = 0; + + CHECK(!strcmp(buf, code)); + b64.reset(); + } + } +} +#endif + diff --git a/src/helpers/base64_encoder.h b/src/helpers/base64_encoder.h new file mode 100644 index 000000000..6c5a8d486 --- /dev/null +++ b/src/helpers/base64_encoder.h @@ -0,0 +1,54 @@ +//-------------------------------------------------------------------------- +// Copyright (C) 2017-2017 Cisco and/or its affiliates. All rights reserved. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License Version 2 as published +// by the Free Software Foundation. You may not use, modify or distribute +// this program under any other version of the GNU General Public License. +// +// This program is distributed in the hope that it will be useful, but +// WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with this program; if not, write to the Free Software Foundation, Inc., +// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +//-------------------------------------------------------------------------- + +// base64_encoder.h author Russ Combs + +#ifndef BASE64_ENCODER_H +#define BASE64_ENCODER_H + +// this is based on the excellent work by devolve found at +// https://sourceforge.net/projects/libb64/. + +// usage: instantiate, encode+, finish +// buf must hold 2*length_in + +#include +#include "main/snort_types.h" + +class SO_PUBLIC Base64Encoder +{ +public: + Base64Encoder() + { reset(); } + + unsigned encode(const uint8_t* plain_text, unsigned length, char* buf); + unsigned finish(char* buf); + + void reset() + { step = step_A, state = 0; } + +private: + enum Steps { step_A, step_B, step_C }; + Steps step; + uint8_t state; +}; + +void keep_base64_encoder(); + +#endif + diff --git a/src/log/text_log.cc b/src/log/text_log.cc index 6ea36c9b7..76dfda3c8 100644 --- a/src/log/text_log.cc +++ b/src/log/text_log.cc @@ -67,8 +67,12 @@ static FILE* TextLog_Open(const char* name) { if ( name && !strcasecmp(name, "stdout") ) { +#ifdef USE_STDLOG FILE* stdlog = fdopen(STDLOG_FILENO, "w"); return stdlog ? stdlog : stdout; +#else + return stdout; +#endif } return OpenAlertFile(name); diff --git a/src/log/unified2.h b/src/log/unified2.h index 1e6cb7f8e..3f10009f2 100644 --- a/src/log/unified2.h +++ b/src/log/unified2.h @@ -21,31 +21,29 @@ #ifndef UNIFIED2_H #define UNIFIED2_H -// Unified logging (events and packets) shared header. Fields marked -// deprecated are no longer generated but can still be dumped with -// u2spewfoo. +// Unified logging (events and packets) shared header. #include #include "protocols/protocol_ids.h" // OBSOLETE (no longer generated): -// #define UNIFIED2_EVENT 1 -// #define UNIFIED2_IDS_EVENT 7 -// #define UNIFIED2_IDS_EVENT_IPV6 72 -// #define UNIFIED2_IDS_EVENT_MPLS 99 +// #define UNIFIED2_EVENT 1 +// #define UNIFIED2_IDS_EVENT 7 +// #define UNIFIED2_IDS_EVENT_IPV6 72 +// #define UNIFIED2_IDS_EVENT_MPLS 99 // #define UNIFIED2_IDS_EVENT_IPV6_MPLS 100 // CURRENT -#define UNIFIED2_PACKET 2 -#define UNIFIED2_BUFFER 3 -#define UNIFIED2_IDS_EVENT_VLAN 104 // deprecated -#define UNIFIED2_IDS_EVENT_IPV6_VLAN 105 // deprecated +#define UNIFIED2_PACKET 2 +#define UNIFIED2_BUFFER 3 // !legacy_events +#define UNIFIED2_IDS_EVENT_VLAN 104 // legacy_events +#define UNIFIED2_IDS_EVENT_IPV6_VLAN 105 // legacy_events #define UNIFIED2_EXTRA_DATA 110 #define UNIFIED2_IDS_EVENT_APPSTAT 113 // FIXIT-L owned by appid (should have own # space) #define UNIFIED2_EVENT3 114 -#define MAX_EVENT_APPNAME_LEN 64 +#define MAX_EVENT_APPNAME_LEN 64 /* Data structure used for serialization of Unified2 Records */ struct Serial_Unified2_Header @@ -153,7 +151,7 @@ struct Serial_Unified2Packet uint32_t event_id; uint32_t event_second; uint32_t packet_second; - uint32_t packet_microsecond; + uint32_t packet_microsecond; uint32_t linktype; uint32_t packet_length; uint8_t packet_data[4]; diff --git a/src/loggers/CMakeLists.txt b/src/loggers/CMakeLists.txt index 195360c35..0e06e6b2e 100644 --- a/src/loggers/CMakeLists.txt +++ b/src/loggers/CMakeLists.txt @@ -14,6 +14,8 @@ set (PLUGIN_LIST alert_syslog.cc log_hext.cc log_pcap.cc + u2_packet.cc + u2_packet.h unified2.cc ) @@ -36,7 +38,7 @@ else (STATIC_LOGGERS) add_dynamic_module(alert_syslog loggers alert_syslog.cc) add_dynamic_module(log_hext loggers log_hext.cc) add_dynamic_module(log_pcap loggers log_pcap.cc) - add_dynamic_module(unified2 loggers unified2.cc) + add_dynamic_module(unified2 loggers unified2.cc u2_packet.cc u2_packet.h) endif (STATIC_LOGGERS) diff --git a/src/loggers/Makefile.am b/src/loggers/Makefile.am index ebe861fdd..020eb808a 100644 --- a/src/loggers/Makefile.am +++ b/src/loggers/Makefile.am @@ -14,6 +14,8 @@ alert_full.cc \ alert_syslog.cc \ log_hext.cc \ log_pcap.cc \ +u2_packet.cc \ +u2_packet.h \ unified2.cc if STATIC_LOGGERS @@ -55,6 +57,6 @@ log_pcap_la_SOURCES = log_pcap.cc ehlib_LTLIBRARIES += unified2.la unified2_la_CXXFLAGS = $(AM_CXXFLAGS) -DBUILDING_SO unified2_la_LDFLAGS = $(AM_LDFLAGS) -module -export-dynamic -avoid-version -shared -unified2_la_SOURCES = unified2.cc +unified2_la_SOURCES = unified2.cc u2_packet.cc u2_packet.h endif diff --git a/src/loggers/alert_csv.cc b/src/loggers/alert_csv.cc index c643b3453..4d99ad2bd 100644 --- a/src/loggers/alert_csv.cc +++ b/src/loggers/alert_csv.cc @@ -31,8 +31,10 @@ #include "detection/detection_engine.h" #include "detection/signature.h" #include "events/event.h" +#include "flow/flow_key.h" #include "framework/logger.h" #include "framework/module.h" +#include "helpers/base64_encoder.h" #include "log/log.h" #include "log/log_text.h" #include "log/text_log.h" @@ -42,6 +44,7 @@ #include "protocols/icmp4.h" #include "protocols/tcp.h" #include "protocols/udp.h" +#include "protocols/vlan.h" #include "utils/stats.h" using namespace std; @@ -69,6 +72,35 @@ static void ff_action(Args&) TextLog_Puts(csv_log, Active::get_action_string()); } +static void ff_class(Args& a) +{ + const char* cls = "none"; + if ( a.event.sig_info->class_type and a.event.sig_info->class_type->name ) + cls = a.event.sig_info->class_type->name; + TextLog_Puts(csv_log, cls); +} + +static void ff_b64_data(Args& a) +{ + const unsigned block_size = 2048; + char out[2*block_size]; + const uint8_t* in = a.pkt->data; + + unsigned nin = 0; + Base64Encoder b64; + + while ( nin < a.pkt->dsize ) + { + unsigned kin = min(a.pkt->dsize-nin, block_size); + unsigned kout = b64.encode(in+nin, kin, out); + TextLog_Write(csv_log, out, kout); + nin += kin; + } + + if ( unsigned kout = b64.finish(out) ) + TextLog_Write(csv_log, out, kout); +} + static void ff_dir(Args& a) { const char* dir; @@ -83,14 +115,6 @@ static void ff_dir(Args& a) TextLog_Puts(csv_log, dir); } -static void ff_dgm_len(Args& a) -{ - if (a.pkt->has_ip()) - TextLog_Print(csv_log, "%d", a.pkt->ptrs.ip_api.dgram_len()); - else - TextLog_Print(csv_log, "%d", a.pkt->dsize); -} - static void ff_dst_addr(Args& a) { if ( a.pkt->has_ip() or a.pkt->is_data() ) @@ -108,13 +132,13 @@ static void ff_dst_ap(Args& a) if ( a.pkt->proto_bits & (PROTO_BIT__TCP|PROTO_BIT__UDP) ) port = a.pkt->ptrs.dp; - TextLog_Print(csv_log, "%s:%d", addr, port); + TextLog_Print(csv_log, "%s:%u", addr, port); } static void ff_dst_port(Args& a) { if ( a.pkt->proto_bits & (PROTO_BIT__TCP|PROTO_BIT__UDP) ) - TextLog_Print(csv_log, "%d", a.pkt->ptrs.dp); + TextLog_Print(csv_log, "%u", a.pkt->ptrs.dp); } static void ff_eth_dst(Args& a) @@ -166,25 +190,25 @@ static void ff_gid(Args& a) static void ff_icmp_code(Args& a) { if (a.pkt->ptrs.icmph ) - TextLog_Print(csv_log, "%d", a.pkt->ptrs.icmph->code); + TextLog_Print(csv_log, "%u", a.pkt->ptrs.icmph->code); } static void ff_icmp_id(Args& a) { if (a.pkt->ptrs.icmph ) - TextLog_Print(csv_log, "%d", ntohs(a.pkt->ptrs.icmph->s_icmp_id)); + TextLog_Print(csv_log, "%u", ntohs(a.pkt->ptrs.icmph->s_icmp_id)); } static void ff_icmp_seq(Args& a) { if (a.pkt->ptrs.icmph ) - TextLog_Print(csv_log, "%d", ntohs(a.pkt->ptrs.icmph->s_icmp_seq)); + TextLog_Print(csv_log, "%u", ntohs(a.pkt->ptrs.icmph->s_icmp_seq)); } static void ff_icmp_type(Args& a) { if (a.pkt->ptrs.icmph ) - TextLog_Print(csv_log, "%d", a.pkt->ptrs.icmph->type); + TextLog_Print(csv_log, "%u", a.pkt->ptrs.icmph->type); } static void ff_iface(Args&) @@ -201,7 +225,7 @@ static void ff_ip_id(Args& a) static void ff_ip_len(Args& a) { if (a.pkt->has_ip()) - TextLog_Print(csv_log, "%d", a.pkt->ptrs.ip_api.pay_len()); + TextLog_Print(csv_log, "%u", a.pkt->ptrs.ip_api.pay_len()); } static void ff_msg(Args& a) @@ -209,16 +233,45 @@ static void ff_msg(Args& a) TextLog_Quote(csv_log, a.msg); } +static void ff_mpls(Args& a) +{ + uint32_t mpls; + + if (a.pkt->flow) + mpls = a.pkt->flow->key->mplsLabel; + + else if ( a.pkt->proto_bits & PROTO_BIT__MPLS ) + mpls = a.pkt->ptrs.mplsHdr.label; + + else + return; + + TextLog_Print(csv_log, "%u", ntohl(mpls)); +} + static void ff_pkt_gen(Args& a) { TextLog_Puts(csv_log, a.pkt->get_pseudo_type()); } +static void ff_pkt_len(Args& a) +{ + if (a.pkt->has_ip()) + TextLog_Print(csv_log, "%u", a.pkt->ptrs.ip_api.dgram_len()); + else + TextLog_Print(csv_log, "%u", a.pkt->dsize); +} + static void ff_pkt_num(Args&) { TextLog_Print(csv_log, STDu64, pc.total_from_daq); } +static void ff_priority(Args& a) +{ + TextLog_Print(csv_log, "%u", a.event.sig_info->priority); +} + static void ff_proto(Args& a) { TextLog_Puts(csv_log, a.pkt->get_type()); @@ -235,6 +288,14 @@ static void ff_rule(Args& a) a.event.sig_info->gid, a.event.sig_info->sid, a.event.sig_info->rev); } +static void ff_service(Args& a) +{ + const char* svc = "unknown"; + if ( a.pkt->flow and a.pkt->flow->service ) + svc = a.pkt->flow->service; + TextLog_Puts(csv_log, svc); +} + static void ff_sid(Args& a) { TextLog_Print(csv_log, "%u", a.event.sig_info->sid); @@ -257,13 +318,13 @@ static void ff_src_ap(Args& a) if ( a.pkt->proto_bits & (PROTO_BIT__TCP|PROTO_BIT__UDP) ) port = a.pkt->ptrs.sp; - TextLog_Print(csv_log, "%s:%d", addr, port); + TextLog_Print(csv_log, "%s:%u", addr, port); } static void ff_src_port(Args& a) { if ( a.pkt->proto_bits & (PROTO_BIT__TCP|PROTO_BIT__UDP) ) - TextLog_Print(csv_log, "%d", a.pkt->ptrs.sp); + TextLog_Print(csv_log, "%u", a.pkt->ptrs.sp); } static void ff_tcp_ack(Args& a) @@ -285,7 +346,7 @@ static void ff_tcp_flags(Args& a) static void ff_tcp_len(Args& a) { if (a.pkt->ptrs.tcph ) - TextLog_Print(csv_log, "%d", (a.pkt->ptrs.tcph->off())); + TextLog_Print(csv_log, "%u", (a.pkt->ptrs.tcph->off())); } static void ff_tcp_seq(Args& a) @@ -300,27 +361,43 @@ static void ff_tcp_win(Args& a) TextLog_Print(csv_log, "0x%X", ntohs(a.pkt->ptrs.tcph->th_win)); } +static void ff_timestamp(Args& a) +{ + LogTimeStamp(csv_log, a.pkt); +} + static void ff_tos(Args& a) { if (a.pkt->has_ip()) - TextLog_Print(csv_log, "%d", a.pkt->ptrs.ip_api.tos()); + TextLog_Print(csv_log, "%u", a.pkt->ptrs.ip_api.tos()); } static void ff_ttl(Args& a) { if (a.pkt->has_ip()) - TextLog_Print(csv_log, "%d",a.pkt->ptrs.ip_api.ttl()); + TextLog_Print(csv_log, "%u",a.pkt->ptrs.ip_api.ttl()); } -static void ff_timestamp(Args& a) +static void ff_udp_len(Args& a) { - LogTimeStamp(csv_log, a.pkt); + if (a.pkt->ptrs.udph ) + TextLog_Print(csv_log, "%u", ntohs(a.pkt->ptrs.udph->uh_len)); } -static void ff_udp_len(Args& a) +static void ff_vlan(Args& a) { - if (a.pkt->ptrs.udph ) - TextLog_Print(csv_log, "%d", ntohs(a.pkt->ptrs.udph->uh_len)); + uint16_t vid; + + if (a.pkt->flow) + vid = a.pkt->flow->key->vlan_tag; + + else if ( a.pkt->proto_bits & PROTO_BIT__VLAN ) + vid = layer::get_vlan_layer(a.pkt)->vid(); + + else + return; + + TextLog_Print(csv_log, "%hu", vid); } //------------------------------------------------------------------------- @@ -331,26 +408,26 @@ typedef void (*CsvFunc)(Args&); static const CsvFunc csv_func[] = { - ff_action, ff_dir, ff_dgm_len, ff_dst_addr, ff_dst_ap, ff_dst_port, - ff_eth_dst, ff_eth_len, ff_eth_src, ff_eth_type, ff_gid, - ff_icmp_code, ff_icmp_id, ff_icmp_seq, ff_icmp_type, ff_iface, - ff_ip_id, ff_ip_len, ff_msg, ff_pkt_gen, ff_pkt_num, ff_proto, - ff_rev, ff_rule, ff_sid, ff_src_addr, ff_src_ap, ff_src_port, - ff_tcp_ack, ff_tcp_flags, ff_tcp_len, ff_tcp_seq, ff_tcp_win, - ff_timestamp, ff_tos, ff_ttl, ff_udp_len + ff_action, ff_class, ff_b64_data, ff_dir, ff_dst_addr, ff_dst_ap, + ff_dst_port, ff_eth_dst, ff_eth_len, ff_eth_src, ff_eth_type, ff_gid, + ff_icmp_code, ff_icmp_id, ff_icmp_seq, ff_icmp_type, ff_iface, ff_ip_id, + ff_ip_len, ff_msg, ff_mpls, ff_pkt_gen, ff_pkt_len, ff_pkt_num, ff_priority, + ff_proto, ff_rev, ff_rule, ff_service, ff_sid, ff_src_addr, ff_src_ap, + ff_src_port, ff_tcp_ack, ff_tcp_flags, ff_tcp_len, ff_tcp_seq, + ff_tcp_win, ff_timestamp, ff_tos, ff_ttl, ff_udp_len, ff_vlan }; #define csv_range \ - "action | dir | dgm_len | dst_addr | dst_ap | dst_port | " \ - "eth_dst | eth_len | eth_src | eth_type | gid | " \ - "icmp_code | icmp_id | icmp_seq | icmp_type | iface | " \ - "ip_id | ip_len | msg | pkt_gen | pkt_num | proto | " \ - "rev | rule | sid | src_addr | src_ap | src_port | " \ - "tcp_ack | tcp_flags | tcp_len | tcp_seq | tcp_win | " \ - "timestamp | tos | ttl | udp_len" + "action | class | b64_data | dir | dst_addr | dst_ap | " \ + "dst_port | eth_dst | eth_len | eth_src | eth_type | gid | " \ + "icmp_code | icmp_id | icmp_seq | icmp_type | iface | ip_id | " \ + "ip_len | msg | mpls | pkt_gen | pkt_len | pkt_num | priority | " \ + "proto | rev | rule | service | sid | src_addr | src_ap | " \ + "src_port | tcp_ack | tcp_flags | tcp_len | tcp_seq | " \ + "tcp_win | timestamp | tos | ttl | udp_len | vlan" #define csv_deflt \ - "timestamp pkt_num proto pkt_gen dgm_len dir src_ap dst_ap rule action" + "timestamp pkt_num proto pkt_gen pkt_len dir src_ap dst_ap rule action" static const Parameter s_params[] = { diff --git a/src/loggers/u2_packet.cc b/src/loggers/u2_packet.cc new file mode 100644 index 000000000..54455ef9b --- /dev/null +++ b/src/loggers/u2_packet.cc @@ -0,0 +1,169 @@ +//-------------------------------------------------------------------------- +// Copyright (C) 2017-2017 Cisco and/or its affiliates. All rights reserved. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License Version 2 as published +// by the Free Software Foundation. You may not use, modify or distribute +// this program under any other version of the GNU General Public License. +// +// This program is distributed in the hope that it will be useful, but +// WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with this program; if not, write to the Free Software Foundation, Inc., +// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +//-------------------------------------------------------------------------- + +// u2_packet.cc author Russ Combs + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "u2_packet.h" + +#include + +#include "flow/flow.h" +#include "flow/flow_key.h" +#include "protocols/packet.h" +#include "protocols/protocol_ids.h" +#include "utils/util.h" + +static const uint8_t u2_ttl = 64; + +//-------------------------------------------------------------------------- +// public methods +//-------------------------------------------------------------------------- + +U2PseudoHeader::U2PseudoHeader(const Packet* p) +{ + assert(p->flow); + + if ( p->flow->key->version == 0x4 ) + { + cook_eth(p, u.v4.eth); + cook_ip4(p, u.v4.ip4); + cook_tcp(p, u.v4.tcp); + size = sizeof(u.v4) - offset; + } + else if ( p->flow->key->version == 0x6 ) + { + cook_eth(p, u.v6.eth); + cook_ip6(p, u.v6.ip6); + cook_tcp(p, u.v6.tcp); + size = sizeof(u.v6) - offset; + } + else size = 0; +} + +uint8_t* U2PseudoHeader::get_data() +{ return size ? u.buf + offset : nullptr; } + +uint16_t U2PseudoHeader::get_size() +{ return size; } + +uint16_t U2PseudoHeader::get_dsize() +{ return dsize; } + +//-------------------------------------------------------------------------- +// private methods +//-------------------------------------------------------------------------- + +void U2PseudoHeader::cook_eth(const Packet* p, eth::EtherHdr& h) +{ + const unsigned sz = sizeof(h.ether_src); + + const uint8_t src[sz] = { 0xFF, 0x01, 0x02, 0x0A, 0x0B, 0x0C }; + const uint8_t dst[sz] = { 0xFF, 0x02, 0x01, 0x0A, 0x0B, 0x0C }; + + for ( unsigned i = 0; i < sz; i++ ) + { + h.ether_src[i] = src[i]; + h.ether_dst[i] = dst[i]; + } + + if ( p->flow->key->version == 0x4 ) + h.ether_type = htons(to_utype(ProtocolId::ETHERTYPE_IPV4)); + else + h.ether_type = htons(to_utype(ProtocolId::ETHERTYPE_IPV6)); +} + +// there is no length restriction on reassembled PDU buffers so +// we must ensure that we don't produce a bogus datagram length. + +void U2PseudoHeader::cook_ip4(const Packet* p, ip::IP4Hdr& h) +{ + const uint16_t overhead = sizeof(h) + sizeof(tcp::TCPHdr); + const uint16_t max_data = IP_MAXPACKET - overhead; + dsize = p->dsize; + + if ( dsize > max_data ) + dsize = max_data; + + h.ip_verhl = 0x45; + h.ip_len = htons(overhead + dsize); + h.ip_proto = IpProtocol::TCP; + h.ip_ttl = u2_ttl; + + if (p->is_from_client()) + { + h.ip_src = p->flow->client_ip.get_ip4_value(); + h.ip_dst = p->flow->server_ip.get_ip4_value(); + } + else + { + h.ip_src = p->flow->server_ip.get_ip4_value(); + h.ip_dst = p->flow->client_ip.get_ip4_value(); + } + + h.ip_tos = 0; + h.ip_id = 0; + h.ip_off = 0; + h.ip_csum = 0; +} + +void U2PseudoHeader::cook_ip6(const Packet* p, ip::IP6Hdr& h) +{ + const uint16_t overhead = sizeof(tcp::TCPHdr); + const uint16_t max_data = IP_MAXPACKET - overhead; + dsize = p->dsize; + + if ( dsize > max_data ) + dsize = max_data; + + h.ip6_vtf = htonl(0x60 << 24); + h.ip6_payload_len = htons(overhead + dsize); + h.ip6_next = IpProtocol::TCP; + h.ip6_hoplim = u2_ttl; + + if (p->is_from_client()) + { + COPY4(h.ip6_src.u6_addr32, p->flow->client_ip.get_ip6_ptr()); + COPY4(h.ip6_dst.u6_addr32, p->flow->server_ip.get_ip6_ptr()); + } + else + { + COPY4(h.ip6_src.u6_addr32, p->flow->server_ip.get_ip6_ptr()); + COPY4(h.ip6_dst.u6_addr32, p->flow->client_ip.get_ip6_ptr()); + } +} + +void U2PseudoHeader::cook_tcp(const Packet* p, tcp::TCPHdr& h) +{ + h.th_sport = htons(p->ptrs.sp); + h.th_dport = htons(p->ptrs.dp); + + h.th_offx2 = 0x50; // these are required + h.th_flags = TH_ACK; + + h.th_seq = htonl(1); // just to make wireshark happy + h.th_ack = h.th_seq; + h.th_win = htons(8192); + + h.th_sum = 0; + h.th_urp = 0; +} + diff --git a/src/loggers/u2_packet.h b/src/loggers/u2_packet.h new file mode 100644 index 000000000..0d49b341a --- /dev/null +++ b/src/loggers/u2_packet.h @@ -0,0 +1,79 @@ +//-------------------------------------------------------------------------- +// Copyright (C) 2017-2017 Cisco and/or its affiliates. All rights reserved. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License Version 2 as published +// by the Free Software Foundation. You may not use, modify or distribute +// this program under any other version of the GNU General Public License. +// +// This program is distributed in the hope that it will be useful, but +// WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with this program; if not, write to the Free Software Foundation, Inc., +// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +//-------------------------------------------------------------------------- + +// u2_packet.h author Russ Combs + +#ifndef U2_PACKET_H +#define U2_PACKET_H + +// generate eth:ip:tcp headers for compatibility with legacy tool chains +// that must have packets + +#include "protocols/eth.h" +#include "protocols/ipv4.h" +#include "protocols/ipv6.h" +#include "protocols/tcp.h" + +struct Packet; + +class U2PseudoHeader +{ +public: + U2PseudoHeader(const Packet*); + + uint8_t* get_data(); + uint16_t get_size(); + uint16_t get_dsize(); + +private: + void cook_eth(const Packet*, eth::EtherHdr&); + void cook_ip4(const Packet*, ip::IP4Hdr&); + void cook_ip6(const Packet*, ip::IP6Hdr&); + void cook_tcp(const Packet*, tcp::TCPHdr&); + + static const unsigned offset = 2; + static const unsigned max_size = + offset + sizeof(eth::EtherHdr) + sizeof(ip::IP6Hdr) + sizeof(tcp::TCPHdr); + + uint16_t size; + uint16_t dsize; + + union + { + struct + { + uint8_t align[offset]; + eth::EtherHdr eth; + ip::IP4Hdr ip4; + tcp::TCPHdr tcp; + } v4; + + struct + { + uint8_t align[offset]; + eth::EtherHdr eth; + ip::IP6Hdr ip6; + tcp::TCPHdr tcp; + } v6; + + uint8_t buf[max_size]; + } u; +}; + +#endif + diff --git a/src/loggers/unified2.cc b/src/loggers/unified2.cc index bd3bf6b79..fc8e0fbf5 100644 --- a/src/loggers/unified2.cc +++ b/src/loggers/unified2.cc @@ -51,6 +51,8 @@ #include "utils/util.h" #include "utils/util_cstring.h" +#include "u2_packet.h" + using namespace std; #define S_NAME "unified2" @@ -99,16 +101,8 @@ static THREAD_LOCAL char* io_buffer = nullptr; /* -------------------- Local Functions -----------------------*/ -static void Unified2InitFile(Unified2Config*); -static inline void Unified2RotateFile(Unified2Config*); -static void _Unified2LogPacketAlert(Packet*, const char*, Unified2Config*, const Event*); static void Unified2Write(uint8_t*, uint32_t, Unified2Config*); -static void alert_event(Packet*, const char*, Unified2Config*, const Event*); - -static void AlertExtraData(Flow*, void* data, LogFunction* log_funcs, - uint32_t max_count, uint32_t xtradata_mask, uint32_t event_id, uint32_t event_second); - static void Unified2InitFile(Unified2Config* config) { assert(config); @@ -337,12 +331,15 @@ static void AlertExtraData( } static void _Unified2LogPacketAlert( - Packet* p, const char*, Unified2Config* config, const Event* event) + Packet* p, const char*, Unified2Config* config, const Event* event, + unsigned u2_type, U2PseudoHeader* u2h = nullptr) { Serial_Unified2_Header hdr; Serial_Unified2Packet logheader; + uint32_t pkt_length = 0; uint32_t write_len = sizeof(hdr) + sizeof(Serial_Unified2Packet) - 4; + unsigned u2h_len = u2h ? u2h->get_size() : 0; logheader.sensor_id = 0; logheader.linktype = u2.base_proto; @@ -365,8 +362,8 @@ static void _Unified2LogPacketAlert( logheader.packet_second = htonl((uint32_t)p->pkth->ts.tv_sec); logheader.packet_microsecond = htonl((uint32_t)p->pkth->ts.tv_usec); pkt_length = ( p->is_rebuilt() ) ? p->dsize : p->pkth->caplen; - logheader.packet_length = htonl(pkt_length); - write_len += pkt_length; + logheader.packet_length = htonl(pkt_length + u2h_len); + write_len += pkt_length + u2h_len; } else { @@ -378,17 +375,22 @@ static void _Unified2LogPacketAlert( if ( config->limit && (u2.current + write_len) > config->limit ) Unified2RotateFile(config); - hdr.length = htonl(sizeof(Serial_Unified2Packet) - 4 + pkt_length); - hdr.type = ( p and p->is_rebuilt() ) ? htonl(UNIFIED2_BUFFER) : htonl(UNIFIED2_PACKET); + hdr.length = htonl(sizeof(Serial_Unified2Packet) - 4 + pkt_length + u2h_len); + hdr.type = htonl(u2_type); memcpy_s(write_pkt_buffer, u2_buf_sz, &hdr, sizeof(hdr)); - size_t offset = sizeof(hdr); memcpy_s(write_pkt_buffer + offset, u2_buf_sz - offset, &logheader, sizeof(logheader) - 4); - offset += sizeof(logheader) - 4; + if ( u2h_len > 0 ) + { + assert(u2_buf_sz - offset > u2h_len); + memcpy_s(write_pkt_buffer + offset, u2_buf_sz - offset, u2h->get_data(), u2h_len); + offset += u2h_len; + } + if (pkt_length != 0) { if (pkt_length > u2_buf_sz - offset) @@ -976,21 +978,19 @@ void U2Logger::alert(Packet* p, const char* msg, const Event& event) void U2Logger::log(Packet* p, const char* msg, Event* event) { - if (p) + assert(p); + + // FIXIT-H convert to packet method if correct + if ( !p->is_cooked() or p->pseudo_type == PSEUDO_PKT_IP ) + _Unified2LogPacketAlert(p, msg, &config, event, UNIFIED2_PACKET); + + else if ( !config.legacy_events ) + _Unified2LogPacketAlert(p, msg, &config, event, UNIFIED2_BUFFER); + + else { - if ( (p->packet_flags & PKT_REBUILT_STREAM) and !p->is_data() ) - { - DebugMessage(DEBUG_LOG, - "[*] Reassembled packet, dumping stream packets\n"); - // FIXIT-H replace with reassembled stream data and - // optionally the first captured packet - //_Unified2LogStreamAlert(p, msg, &config, event); - } - else - { - DebugMessage(DEBUG_LOG, "[*] Logging unified 2 packets...\n"); - _Unified2LogPacketAlert(p, msg, &config, event); - } + U2PseudoHeader u2h(p); + _Unified2LogPacketAlert(p, msg, &config, event, UNIFIED2_PACKET, &u2h); } } diff --git a/src/main/snort.cc b/src/main/snort.cc index f102bd6c5..e849b4094 100644 --- a/src/main/snort.cc +++ b/src/main/snort.cc @@ -45,6 +45,7 @@ #include "flow/ha.h" #include "framework/endianness.h" #include "framework/mpse.h" +#include "helpers/base64_encoder.h" #include "helpers/process.h" #include "host_tracker/host_cache.h" #include "ips_options/ips_flowbits.h" @@ -514,6 +515,8 @@ void Snort::setup(int argc, char* argv[]) { set_main_thread(); + // must be done before any other files are opened because we + // will try to grab file descriptor 3 (if --enable-stdlog) OpenLogger(); init(argc, argv); @@ -532,10 +535,12 @@ void Snort::setup(int argc, char* argv[]) set_quick_exit(false); - keep_utf_lib(); - keep_kmap_lib(); + // FIXIT-L eliminate keep_*() required for dynamic-only linkage + keep_base64_encoder(); keep_decomp_lib(); keep_jsnorm_lib(); + keep_kmap_lib(); + keep_utf_lib(); TimeStart(); } diff --git a/src/network_inspectors/appid/appid_module.cc b/src/network_inspectors/appid/appid_module.cc index a166dd7e0..7229abee2 100644 --- a/src/network_inspectors/appid/appid_module.cc +++ b/src/network_inspectors/appid/appid_module.cc @@ -155,7 +155,7 @@ static const Parameter s_params[] = { "app_detector_dir", Parameter::PT_STRING, nullptr, nullptr, "directory to load appid detectors from" }, { "instance_id", Parameter::PT_INT, "0:", "0", - "instance id - need more details for what this is" }, + "instance id - ignored" }, { "debug", Parameter::PT_BOOL, nullptr, "false", "enable appid debug logging" }, { "dump_ports", Parameter::PT_BOOL, nullptr, "false", diff --git a/src/network_inspectors/appid/appid_stats.cc b/src/network_inspectors/appid/appid_stats.cc index befb44345..feb9947ee 100644 --- a/src/network_inspectors/appid/appid_stats.cc +++ b/src/network_inspectors/appid/appid_stats.cc @@ -25,13 +25,13 @@ #include "appid_stats.h" -#include +#include "log/text_log.h" +#include "log/unified2.h" +#include "time/packet_time.h" #include "appid_config.h" #include "app_info_table.h" #include "appid_session.h" -#include "log/messages.h" -#include "log/unified2.h" #define URLCATBUCKETS 100 #define URLREPBUCKETS 5 @@ -43,16 +43,7 @@ struct AppIdStatRecord uint32_t responderBytes; }; -#pragma pack(1) -struct AppIdStatOutputRecord -{ - char app_name[MAX_EVENT_APPNAME_LEN]; - uint32_t initiatorBytes; - uint32_t responderBytes; -}; -#pragma pack() - -static const char appid_stats_file_suffix[] = "appid_stats.log"; +static const char appid_stats_filename[] = "appid_stats.log"; static void delete_record(void* record) { @@ -71,12 +62,7 @@ StatsBucket* AppIdStatistics::get_stats_bucket(time_t startTime) StatsBucket* bucket = nullptr; if ( !currBuckets ) - { currBuckets = sflist_new(); -# ifdef DEBUG_STATS - fprintf(SF_DEBUG_FILE, "New Stats Bucket List\n"); -# endif - } if ( !currBuckets ) return nullptr; @@ -98,11 +84,6 @@ StatsBucket* AppIdStatistics::get_stats_bucket(time_t startTime) bucket->startTime = startTime; bucket->appsTree = fwAvlInit(); sflist_add_before(currBuckets, lNode, bucket); - -#ifdef DEBUG_STATS - fprintf(SF_DEBUG_FILE, "New Bucket Time: %u before %u\n", - bucket->startTime, lBucket->startTime); -#endif break; } } @@ -113,79 +94,37 @@ StatsBucket* AppIdStatistics::get_stats_bucket(time_t startTime) bucket->startTime = startTime; bucket->appsTree = fwAvlInit(); sflist_add_tail(currBuckets, bucket); - -#ifdef DEBUG_STATS - fprintf(SF_DEBUG_FILE, "New Bucket Time: %u at tail\n", bucket->startTime); -#endif } return bucket; } -FILE* AppIdStatistics::open_stats_log_file(const char* const filename, time_t tstamp) +void AppIdStatistics::open_stats_log_file() { - FILE* fp; - char output_fullpath[512]; - time_t curr_time; - - if (tstamp) - curr_time = tstamp; - else - curr_time = time(nullptr); - - snprintf(output_fullpath, sizeof(output_fullpath), "%s.%lu", filename, curr_time); - LogMessage("Opening %s for AppId statistics logging.\n", output_fullpath); - - if ((fp = fopen(output_fullpath, "w")) == nullptr) - { - ErrorMessage("Unable to open output file \"%s\": %s\n for AppId statistics logging.\n", - output_fullpath, strerror(errno)); - } - return fp; + log = TextLog_Init(appid_stats_filename, 4096, rollSize); } void AppIdStatistics::dump_statistics() { - struct StatsBucket* bucket = nullptr; - uint32_t* buffPtr; - time_t currTime = time(nullptr); + if ( !enabled ) + return; if ( !logBuckets ) return; + if ( !log ) + open_stats_log_file(); + + struct StatsBucket* bucket = nullptr; + while ((bucket = (struct StatsBucket*)sflist_remove_head(logBuckets)) != nullptr) { - uint8_t* buffer; - size_t buffSize; - Serial_Unified2_Header header; - if ( bucket->appRecordCnt ) { - buffSize = ( bucket->appRecordCnt * sizeof(struct AppIdStatOutputRecord) ) + - ( 4 * sizeof(uint32_t) ); - header.type = UNIFIED2_IDS_EVENT_APPSTAT; - header.length = buffSize - ( 2 * sizeof(uint32_t)); - buffer = (uint8_t*)snort_calloc(buffSize); -# ifdef DEBUG_STATS - fprintf(SF_DEBUG_FILE, "Write App Records %u Size: %zu\n", - bucket->appRecordCnt, buffSize); -# endif - } - else - buffer = nullptr; - - if ( buffer ) - { - buffPtr = (uint32_t*)buffer; - *buffPtr++ = htonl(header.type); - *buffPtr++ = htonl(header.length); - *buffPtr++ = htonl(bucket->startTime); - *buffPtr++ = htonl(bucket->appRecordCnt); + struct FwAvlNode* node; - struct FwAvlNode* node; for (node = fwAvlFirst(bucket->appsTree); node != nullptr; node = fwAvlNext(node)) { - struct AppIdStatOutputRecord* recBuffPtr; const char* app_name; bool cooked_client = false; AppId app_id; @@ -195,16 +134,15 @@ void AppIdStatistics::dump_statistics() record = (struct AppIdStatRecord*)node->data; app_id = record->app_id; - recBuffPtr = (struct AppIdStatOutputRecord*)buffPtr; - if ( app_id >= 2000000000 ) { cooked_client = true; app_id -= 2000000000; } - AppInfoTableEntry* entry = AppInfoManager::get_instance().get_app_info_entry( - app_id); + AppInfoTableEntry* entry + = AppInfoManager::get_instance().get_app_info_entry(app_id); + if ( entry ) { app_name = entry->app_name; @@ -230,46 +168,9 @@ void AppIdStatistics::dump_statistics() app_name = tmpBuff; } - memcpy(recBuffPtr->app_name, app_name, strlen(app_name)); - - /**buffPtr++ = htonl(record->app_id); */ - recBuffPtr->initiatorBytes = htonl(record->initiatorBytes); - recBuffPtr->responderBytes = htonl(record->responderBytes); - buffPtr += sizeof(*recBuffPtr)/sizeof(*buffPtr); + TextLog_Print(log, "%lu,%s,%u,%u\n", + packet_time(), app_name, record->initiatorBytes, record->responderBytes); } - - if ( appid_stats_filename ) - { - if ( !appfp ) - { - appfp = open_stats_log_file(appid_stats_filename, currTime); - appTime = currTime; - appSize = 0; - } - else if ( ( ( currTime - appTime ) > rollPeriod ) || - ( ( appSize + buffSize) > rollSize ) ) - { - fclose(appfp); - appfp = open_stats_log_file(appid_stats_filename, currTime); - appTime = currTime; - appSize = 0; - } - if ( appfp ) - { - if ( ( fwrite(buffer, buffSize, 1, appfp) == 1 ) && ( fflush(appfp) == 0 ) ) - { - appSize += buffSize; - } - else - { - ErrorMessage("AppID ailed to write to statistics file (%s): %s\n", - appid_stats_filename, strerror(errno)); - fclose(appfp); - appfp = nullptr; - } - } - } - snort_free(buffer); } fwAvlDeleteTree(bucket->appsTree, delete_record); snort_free(bucket); @@ -281,13 +182,11 @@ AppIdStatistics::AppIdStatistics(const AppIdModuleConfig& config) if ( config.stats_logging_enabled ) { enabled = true; - std::string stats_file; - appid_stats_filename = snort_strdup(get_instance_file(stats_file, - appid_stats_file_suffix)); rollPeriod = config.app_stats_rollover_time; rollSize = config.app_stats_rollover_size; bucketInterval = config.app_stats_period; + time_t now = get_time(); start_stats_period(now); } @@ -302,12 +201,8 @@ AppIdStatistics::~AppIdStatistics() end_stats_period(); dump_statistics(); - if ( appfp ) - { - fclose(appfp); - appfp = nullptr; - } - snort_free((void*)appid_stats_filename); + if ( log ) + TextLog_Term(log); if ( logBuckets ) snort_free(logBuckets); @@ -319,7 +214,6 @@ AppIdStatistics::~AppIdStatistics() fwAvlDeleteTree(bucket->appsTree, delete_record); snort_free(bucket); } - snort_free(currBuckets); } } diff --git a/src/network_inspectors/appid/appid_stats.h b/src/network_inspectors/appid/appid_stats.h index 6b28bd593..f3c141b45 100644 --- a/src/network_inspectors/appid/appid_stats.h +++ b/src/network_inspectors/appid/appid_stats.h @@ -69,21 +69,18 @@ private: void end_stats_period(void); StatsBucket* get_stats_bucket(time_t); - FILE* open_stats_log_file(const char* const filename, time_t); + void open_stats_log_file(); void dump_statistics(); bool enabled = false; SF_LIST* currBuckets = nullptr; SF_LIST* logBuckets = nullptr; - FILE* appfp = nullptr; - size_t appSize = 0; - time_t appTime = 0; + struct TextLog* log = nullptr; time_t bucketStart = 0; time_t bucketInterval = 0; time_t bucketEnd = 0; size_t rollSize = 0; time_t rollPeriod = 0; - const char* appid_stats_filename = nullptr; }; #endif diff --git a/src/network_inspectors/binder/binder.cc b/src/network_inspectors/binder/binder.cc index cf1b7c908..66e7091c8 100644 --- a/src/network_inspectors/binder/binder.cc +++ b/src/network_inspectors/binder/binder.cc @@ -460,7 +460,7 @@ bool Binder::configure(SnortConfig* sc) return true; } -void Binder::update(SnortConfig* sc, const char* name) +void Binder::update(SnortConfig*, const char* name) { vector::iterator it; for ( it = bindings.begin(); it != bindings.end(); ++it ) diff --git a/src/protocols/packet.h b/src/protocols/packet.h index e9c85331f..688823b50 100644 --- a/src/protocols/packet.h +++ b/src/protocols/packet.h @@ -166,7 +166,7 @@ struct SO_PUBLIC Packet { return ptrs.get_pkt_type() == PktType::ICMP; } inline bool is_data() const - { return (ptrs.get_pkt_type() == PktType::PDU)or (ptrs.get_pkt_type() == PktType::FILE); } + { return (ptrs.get_pkt_type() == PktType::PDU) or (ptrs.get_pkt_type() == PktType::FILE); } inline bool is_cooked() const { return packet_flags & PKT_PSEUDO; } diff --git a/src/stream/libtcp/tcp_stream_session.h b/src/stream/libtcp/tcp_stream_session.h index 7836209d7..72ffed8cd 100644 --- a/src/stream/libtcp/tcp_stream_session.h +++ b/src/stream/libtcp/tcp_stream_session.h @@ -22,10 +22,6 @@ #ifndef TCP_STREAM_SESSION_H_ #define TCP_STREAM_SESSION_H_ -#ifdef HAVE_CONFIG_H -#include "config.h" -#endif - #include "detection/detection_engine.h" #include "flow/session.h" #include "stream/libtcp/tcp_stream_tracker.h" diff --git a/src/stream/tcp/tcp_reassembler.cc b/src/stream/tcp/tcp_reassembler.cc index 137785a24..3d0b3b303 100644 --- a/src/stream/tcp/tcp_reassembler.cc +++ b/src/stream/tcp/tcp_reassembler.cc @@ -470,10 +470,13 @@ int TcpReassembler::flush_data_segments(Packet* p, uint32_t total, Packet* pdu) DebugFormat(DEBUG_STREAM_STATE, "Flushing %u bytes from %X\n", bytes_to_copy, tsn->seq); - if ( !tsn->next || ( bytes_to_copy < tsn->payload_size ) - || SEQ_EQ(tsn->seq + bytes_to_copy, to_seq) ) + if ( !tsn->next or (bytes_to_copy < tsn->payload_size) or + SEQ_EQ(tsn->seq + bytes_to_copy, to_seq) or + (bytes_flushed + tsn->payload_size + tsn->next->payload_size > + tracker->splitter->get_max_pdu()) ) + { flags |= PKT_PDU_TAIL; - + } const StreamBuffer sb = tracker->splitter->reassemble( session->flow, total, bytes_flushed, tsn->payload(), bytes_to_copy, flags, bytes_copied); @@ -525,7 +528,7 @@ int TcpReassembler::flush_data_segments(Packet* p, uint32_t total, Packet* pdu) if ( sb.data || !seglist.next ) break; - if ( bytes_flushed + seglist.next->payload_size >= tracker->splitter->get_max_pdu() ) + if ( bytes_flushed + seglist.next->payload_size > tracker->splitter->get_max_pdu() ) break; } diff --git a/src/utils/util.cc b/src/utils/util.cc index c3fd013c6..522512157 100644 --- a/src/utils/util.cc +++ b/src/utils/util.cc @@ -192,7 +192,7 @@ void ts_print(const struct timeval* tvp, char* timebuf) (void)SnortSnprintf(timebuf, TIMEBUF_SIZE, "%02d/%02d/%02d-%02d:%02d:%02d.%06u", - lt->tm_mon + 1, lt->tm_mday, year, + year, lt->tm_mon + 1, lt->tm_mday, s / 3600, (s % 3600) / 60, s % 60, (u_int)tvp->tv_usec); } diff --git a/tools/snort2lua/output_states/out_csv.cc b/tools/snort2lua/output_states/out_csv.cc index 928f7d666..f9976631f 100644 --- a/tools/snort2lua/output_states/out_csv.cc +++ b/tools/snort2lua/output_states/out_csv.cc @@ -165,8 +165,8 @@ bool AlertCsv::convert(std::istringstream& data_stream) } else if (!val.compare("dgmlen")) { - table_api.add_diff_option_comment("dgmlen", "dgm_len"); - tmpval = table_api.add_list("fields", "dgm_len"); + table_api.add_diff_option_comment("dgmlen", "pkt_len"); + tmpval = table_api.add_list("fields", "pkt_len"); } else if (!val.compare("id"))