Pull request #3728: build: generate and tag 3.1.51.0

Merge in SNORT/snort3 from ~PRBG/snort3:rel_build_3.1.51.0 to master

Squashed commit of the following:

commit 91cec43b99689a40963a1edbfd64f266851923f9
Author: Priyanka Gurudev <prbg@cisco.com>
Date:   Wed Jan 11 19:50:28 2023 -0500

    build: generate and tag 3.1.51.0

diff --git a/CMakeLists.txt b/CMakeLists.txt
index fb13cee7ca8e63d57a9136331e47391b6de80ab1..846efe100f50aedcc5009cedb013e5ebc9c143ae 100644 (file)
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -3,7 +3,7 @@ project (snort CXX C)
 
 set (VERSION_MAJOR 3)
 set (VERSION_MINOR 1)
-set (VERSION_PATCH 50)
+set (VERSION_PATCH 51)
 set (VERSION_SUBLEVEL 0)
 set (VERSION "${VERSION_MAJOR}.${VERSION_MINOR}.${VERSION_PATCH}.${VERSION_SUBLEVEL}")
 

diff --git a/ChangeLog.md b/ChangeLog.md
index 072f9f7f4b85cab09956a001432d98ac6e1d3fce..182171f64d11c3b729c780858766986942066c00 100644 (file)
--- a/ChangeLog.md
+++ b/ChangeLog.md
@@ -1,3 +1,26 @@
+2023-01-11: 3.1.51.0
+
+* appid: add support for cip service, client and payload detection
+* appid: do not create snmp future flow for udp reversed session
+* appid: use packet thread's odp context for future flow creation
+* build: error out if both jemalloc and tcmalloc are configured
+* build: exclude unused memory related sources
+* js_norm: add benchmark tests for PDF parser
+* js_norm: decode UTF-16BE to UTF-8 for JS in PDF
+* js_norm: delete unused method
+* js_norm: tune PDF parser performance
+* lua: add Adobe JavaScript related identifiers to snort_defaults
+* lua: fix typo in Sensitive Data classifications name
+* main: fix const issues causing compile warnings
+* memory: delete unnecessary includes
+* memory: incorporate overloads into profiler
+* memory: refactor jemalloc code and add relevant pegs
+* memory: rename manager to overloads to better indicate purpose
+* memory: update developer notes
+* memory: update stats regardless of state; add unit tests
+* memory: use the process total instead of per thread totals to enforce cap
+* watchdog: print thread id as well for better identification of unresponsive threads
+
 2022-12-19: 3.1.50.0
 
 * alert_fast: fix initialization of http_inspect cheat codes

diff --git a/doc/reference/snort_reference.text b/doc/reference/snort_reference.text
index 62932434ad9603ee97b1965fe5c306067a66c6af..5b962578b63a9fcfadedbcbc7cd44539cbd1c5f4 100644 (file)
--- a/doc/reference/snort_reference.text
+++ b/doc/reference/snort_reference.text
@@ -8,7 +8,7 @@ Snort 3 Reference Manual
 The Snort Team
 
 Revision History
-Revision 3.1.50.0 2022-12-19 15:10:28 EST TST
+Revision 3.1.51.0 2023-01-11 19:39:29 EST TST
 
 ---------------------------------------------------------------------
 
@@ -16,6 +16,7 @@ Table of Contents
 
 1. Help
 2. Basic Modules
+
     2.1. active
     2.2. alerts
     2.3. attribute_table
@@ -49,7 +50,9 @@ Table of Contents
     2.31. snort
     2.32. suppress
     2.33. trace
+
 3. Codec Modules
+
     3.1. arp
     3.2. auth
     3.3. ciscometadata
@@ -77,10 +80,14 @@ Table of Contents
     3.25. udp
     3.26. vlan
     3.27. wlan
+
 4. Connector Modules
+
     4.1. file_connector
     4.2. tcp_connector
+
 5. Inspector Modules
+
     5.1. appid
     5.2. appid_listener
     5.3. arp_spoof
@@ -136,10 +143,14 @@ Table of Contents
     5.53. stream_user
     5.54. telnet
     5.55. wizard
+
 6. IPS Action Modules
+
     6.1. react
     6.2. reject
+
 7. IPS Option Modules
+
     7.1. ack
     7.2. appids
     7.3. asn1
@@ -271,9 +282,11 @@ Table of Contents
     7.129. vba_data
     7.130. window
     7.131. wscale
+
 8. Search Engine Modules
 9. SO Rule Modules
 10. Logger Modules
+
     10.1. alert_csv
     10.2. alert_ex
     10.3. alert_fast
@@ -286,7 +299,9 @@ Table of Contents
     10.10. log_hext
     10.11. log_pcap
     10.12. unified2
+
 11. Appendix
+
     11.1. Build Options
     11.2. Environment Variables
     11.3. Command Line Options
@@ -1056,20 +1071,30 @@ Usage: global
 
 Configuration:
 
-  * int memory.cap = 0: set the per-packet-thread cap on memory
-    (bytes, 0 to disable) { 0:maxSZ }
+  * int memory.cap = 0: set the process cap on memory in bytes (0 to
+    disable) { 0:maxSZ }
+  * int memory.interval = 50: approximate ms between memory epochs {
+    1:max32 }
+  * int memory.prune_target = 1048576: bytes to prune per packet
+    thread prune cycle { 1:max32 }
   * int memory.threshold = 100: scale cap to account for heap
     overhead { 1:100 }
 
 Peg counts:
 
-  * memory.allocations: total number of allocations (now)
-  * memory.deallocations: total number of deallocations (now)
-  * memory.allocated: total amount of memory allocated (now)
-  * memory.deallocated: total amount of memory deallocated (now)
+  * memory.start_up_use: memory used before packet processing (now)
+  * memory.cur_in_use: current memory used (now)
+  * memory.max_in_use: maximum memory used (max)
+  * memory.epochs: number of memory updates (now)
+  * memory.allocated: total amount of memory allocated by packet
+    threads (now)
+  * memory.deallocated: total amount of memory deallocated by packet
+    threads (now)
+  * memory.reap_cycles: number of actionable over-limit conditions
+    (now)
   * memory.reap_attempts: attempts to reclaim memory (now)
   * memory.reap_failures: failures to reclaim memory (now)
-  * memory.max_in_use: maximum memory used (max)
+  * memory.pruned: total amount of memory pruned (now)
 
 
 2.20. network
@@ -1744,6 +1769,7 @@ Configuration:
   * int trace.modules.js_norm.proc: enable processing logging { 0:255
     }
   * int trace.modules.js_norm.dump: enable data logging { 0:255 }
+  * int trace.modules.memory.all: enable all trace options { 0:255 }
   * int trace.modules.snort.all: enable all trace options { 0:255 }
   * int trace.modules.snort.inspector_manager: enable inspector
     manager trace logging { 0:255 }
@@ -9895,8 +9921,12 @@ libraries see the Getting Started section of the manual.
     start search
   * implied md5.relative = false: offset from cursor instead of start
     of buffer
-  * int memory.cap = 0: set the per-packet-thread cap on memory
-    (bytes, 0 to disable) { 0:maxSZ }
+  * int memory.cap = 0: set the process cap on memory in bytes (0 to
+    disable) { 0:maxSZ }
+  * int memory.interval = 50: approximate ms between memory epochs {
+    1:max32 }
+  * int memory.prune_target = 1048576: bytes to prune per packet
+    thread prune cycle { 1:max32 }
   * int memory.threshold = 100: scale cap to account for heap
     overhead { 1:100 }
   * string metadata.*: comma-separated list of arbitrary name value
@@ -10912,6 +10942,7 @@ libraries see the Getting Started section of the manual.
   * int trace.modules.js_norm.dump: enable data logging { 0:255 }
   * int trace.modules.js_norm.proc: enable processing logging { 0:255
     }
+  * int trace.modules.memory.all: enable all trace options { 0:255 }
   * int trace.modules.snort.all: enable all trace options { 0:255 }
   * int trace.modules.snort.inspector_manager: enable inspector
     manager trace logging { 0:255 }
@@ -11604,13 +11635,19 @@ libraries see the Getting Started section of the manual.
   * latency.total_packets: total packets monitored (sum)
   * latency.total_rule_evals: total rule evals monitored (sum)
   * latency.total_usecs: total usecs elapsed (sum)
-  * memory.allocated: total amount of memory allocated (now)
-  * memory.allocations: total number of allocations (now)
-  * memory.deallocated: total amount of memory deallocated (now)
-  * memory.deallocations: total number of deallocations (now)
+  * memory.allocated: total amount of memory allocated by packet
+    threads (now)
+  * memory.cur_in_use: current memory used (now)
+  * memory.deallocated: total amount of memory deallocated by packet
+    threads (now)
+  * memory.epochs: number of memory updates (now)
   * memory.max_in_use: maximum memory used (max)
+  * memory.pruned: total amount of memory pruned (now)
   * memory.reap_attempts: attempts to reclaim memory (now)
+  * memory.reap_cycles: number of actionable over-limit conditions
+    (now)
   * memory.reap_failures: failures to reclaim memory (now)
+  * memory.start_up_use: memory used before packet processing (now)
   * mem_test.packets: total packets (sum)
   * mms.concurrent_sessions: total concurrent MMS sessions (now)
   * mms.frames: total MMS messages (sum)
@@ -12777,12 +12814,12 @@ session.
 The TCP packet is invalid because it doesn’t have a SYN, ACK, or RST
 flag set.
 
-116:424 (pbb) truncated ethernet header
+116:424 (eth) truncated ethernet header
 
 The packet length is less than the minimum ethernet header size (14
 bytes)
 
-116:424 (pbb) truncated ethernet header
+116:424 (eth) truncated ethernet header
 
 A truncated ethernet header was detected.
 

diff --git a/doc/upgrade/snort_upgrade.text b/doc/upgrade/snort_upgrade.text
index 03f3d748ad5ada9620765aad54be994d5cd6741a..709f6e1e058f35cdf7e975fb8e45057bec9f336f 100644 (file)
--- a/doc/upgrade/snort_upgrade.text
+++ b/doc/upgrade/snort_upgrade.text
@@ -8,19 +8,22 @@ Snort 3 Upgrade Manual
 The Snort Team
 
 Revision History
-Revision 3.1.50.0 2022-12-19 15:10:06 EST TST
+Revision 3.1.51.0 2023-01-11 19:40:33 EST TST
 
 ---------------------------------------------------------------------
 
 Table of Contents
 
 1. Overview
+
     1.1. Efficacy
     1.2. Performance
     1.3. Scalability
     1.4. Usability
     1.5. Extensibility
+
 2. Snort 3 vs Snort 2
+
     2.1. Features New to Snort 3
     2.2. Features Improved over Snort 2
     2.3. Build Options
@@ -30,10 +33,13 @@ Table of Contents
     2.7. Output
     2.8. Sensitive Data
     2.9. Features Not Yet Supported by Snort 3
+
 3. Snort2Lua
+
     3.1. Snort2Lua Command Line
     3.2. Known Problems
     3.3. Usage
+
 4. Configuration Changes
 
 
@@ -820,6 +826,7 @@ change -> config 'checksum_mode'  ==> 'network.checksum_eval'
 change -> config 'daq_dir'  ==> 'daq.module_dirs'
 change -> config 'detection_filter'  ==> 'alerts.detection_filter_memcap'
 change -> config 'enable_deep_teredo_inspection'  ==> 'udp.deep_teredo_inspection'
+change -> config 'enable_mpls_overlapping_ip'  ==> 'packets.mpls_agnostic'
 change -> config 'event_filter'  ==> 'alerts.event_filter_memcap'
 change -> config 'max_attribute_hosts'  ==> 'attribute_table.max_hosts'
 change -> config 'max_attribute_services_per_host'  ==> 'attribute_table.max_services_per_host'
@@ -859,17 +866,17 @@ change -> daq: 'config daq:' ==> 'name'
 change -> daq_mode: 'config daq_mode:' ==> 'mode'
 change -> daq_var: 'config daq_var:' ==> 'variables'
 change -> detection: 'ac' ==> 'ac_full'
-change -> detection: 'ac-banded' ==> 'ac_banded'
+change -> detection: 'ac-banded' ==> 'ac_full'
 change -> detection: 'ac-bnfa' ==> 'ac_bnfa'
 change -> detection: 'ac-bnfa-nq' ==> 'ac_bnfa'
 change -> detection: 'ac-bnfa-q' ==> 'ac_bnfa'
 change -> detection: 'ac-nq' ==> 'ac_full'
 change -> detection: 'ac-q' ==> 'ac_full'
-change -> detection: 'ac-sparsebands' ==> 'ac_sparse_bands'
+change -> detection: 'ac-sparsebands' ==> 'ac_full'
 change -> detection: 'ac-split' ==> 'ac_full'
 change -> detection: 'ac-split' ==> 'split_any_any'
-change -> detection: 'ac-std' ==> 'ac_std'
-change -> detection: 'acs' ==> 'ac_sparse'
+change -> detection: 'ac-std' ==> 'ac_full'
+change -> detection: 'acs' ==> 'ac_full'
 change -> detection: 'bleedover-port-limit' ==> 'bleedover_port_limit'
 change -> detection: 'debug-print-fast-pattern' ==> 'show_fast_patterns'
 change -> detection: 'intel-cpm' ==> 'hyperscan'
@@ -878,7 +885,6 @@ change -> detection: 'lowmem-q' ==> 'lowmem'
 change -> detection: 'max-pattern-len' ==> 'max_pattern_len'
 change -> detection: 'no_stream_inserts' ==> 'detect_raw_tcp'
 change -> detection: 'search-method' ==> 'search_method'
-change -> detection: 'search-optimize' ==> 'search_optimize'
 change -> detection: 'split-any-any' ==> 'split_any_any = true by default'
 change -> detection: 'split-any-any' ==> 'split_any_any'
 change -> dnp3: 'ports' ==> 'bindings'
@@ -956,6 +962,7 @@ change -> rate_filter: 'sig_id' ==> 'sid'
 change -> reputation: 'shared_mem' ==> 'list_dir'
 change -> sfportscan: 'proto' ==> 'protos'
 change -> sfportscan: 'scan_type' ==> 'scan_types'
+change -> sip: 'max_requestName_len' ==> 'max_request_name_len'
 change -> sip: 'ports' ==> 'bindings'
 change -> smtp: 'ports' ==> 'bindings'
 change -> ssh: 'server_ports' ==> 'bindings'
@@ -1021,6 +1028,7 @@ deleted -> config 'disable_decode_drops'
 deleted -> config 'disable_inline_init_failopen'
 deleted -> config 'disable_ipopt_alerts'
 deleted -> config 'disable_ipopt_drops'
+deleted -> config 'disable_replace'
 deleted -> config 'disable_tcpopt_alerts'
 deleted -> config 'disable_tcpopt_drops'
 deleted -> config 'disable_tcpopt_experimental_alerts'
@@ -1037,6 +1045,7 @@ deleted -> config 'enable_decode_oversized_alerts'
 deleted -> config 'enable_decode_oversized_drops'
 deleted -> config 'enable_gtp'
 deleted -> config 'enable_ipopt_drops'
+deleted -> config 'enable_mpls_multicast'
 deleted -> config 'enable_tcpopt_drops'
 deleted -> config 'enable_tcpopt_experimental_drops'
 deleted -> config 'enable_tcpopt_obsolete_drops'
@@ -1058,10 +1067,12 @@ deleted -> config 'sfalert_unified2'
 deleted -> config 'sflog_unified2'
 deleted -> config 'sidechannel'
 deleted -> config 'so_rule_memcap'
+deleted -> config 'stateful'
 deleted -> csv: '<filename> can no longer be specific'
 deleted -> csv: 'default'
 deleted -> csv: 'trheader'
 deleted -> detection: 'mwm'
+deleted -> detection: 'search-optimize is always true'
 deleted -> dnp3: 'disabled'
 deleted -> dnp3: 'memcap'
 deleted -> dns: 'enable_experimental_types'
@@ -1075,6 +1086,8 @@ deleted -> ftp_telnet_protocol: 'detect_anomalies'
 deleted -> full: '<filename> can no longer be specific'
 deleted -> http_inspect: 'detect_anomalous_servers'
 deleted -> http_inspect: 'disabled'
+deleted -> http_inspect: 'fast_blocking'
+deleted -> http_inspect: 'normalize_random_nulls_in_text'
 deleted -> http_inspect: 'proxy_alert'
 deleted -> http_inspect_server: 'allow_proxy_use'
 deleted -> http_inspect_server: 'enable_cookie'
@@ -1152,6 +1165,7 @@ deleted -> stream5_tcp: 'ignore_any_rules'
 deleted -> stream5_tcp: 'log_asymmetric_traffic'
 deleted -> stream5_tcp: 'policy noack'
 deleted -> stream5_tcp: 'policy unknown'
+deleted -> stream5_tcp: 'use_static_footprint_sizes'
 deleted -> stream5_udp: 'ignore_any_rules'
 deleted -> tcpdump: '<filename> can no longer be specific'
 deleted -> test: 'file'

diff --git a/doc/user/snort_user.text b/doc/user/snort_user.text
index d433afeb3211f530c0a22b95119c296aa2c26f24..1af4ddb62e7cf847696df42570937dcfcd345467 100644 (file)
--- a/doc/user/snort_user.text
+++ b/doc/user/snort_user.text
@@ -8,17 +8,20 @@ Snort 3 User Manual
 The Snort Team
 
 Revision History
-Revision 3.1.50.0 2022-12-19 15:10:08 EST TST
+Revision 3.1.51.0 2023-01-11 19:39:50 EST TST
 
 ---------------------------------------------------------------------
 
 Table of Contents
 
 1. Overview
+
     1.1. First Steps
     1.2. Configuration
     1.3. Output
+
 2. Concepts
+
     2.1. Terminology
     2.2. Modules
     2.3. Parameters
@@ -26,7 +29,9 @@ Table of Contents
     2.5. Operation
     2.6. Rules
     2.7. Pattern Matching
+
 3. Tutorial
+
     3.1. Dependencies
     3.2. Building
     3.3. Running
@@ -34,7 +39,9 @@ Table of Contents
     3.5. Common Errors
     3.6. Gotchas
     3.7. Known Issues
+
 4. Usage
+
     4.1. Help
     4.2. Sniffing and Logging
     4.3. Configuration
@@ -45,7 +52,9 @@ Table of Contents
     4.8. Logger Alternatives
     4.9. Shell
     4.10. Signals
+
 5. Features
+
     5.1. Active Response
     5.2. AppId
     5.3. Binder
@@ -68,7 +77,9 @@ Table of Contents
     5.20. Telnet
     5.21. Trace
     5.22. Wizard
+
 6. DAQ Configuration and Modules
+
     6.1. Building the DAQ Library and Its Bundled DAQ Modules
     6.2. Configuration
     6.3. Interaction With Multiple Packet Threads