Update APIs to store files in transactions instead of the per flow state.
Goal is to avoid the overhead of matching up files and transactions in
cases where there are many of both.
Update all protocol implementations to support this.
Update file logging logic to account for having files in transactions. Instead
of it acting separately on file containers, it is now tied into the
transaction logging.
Update the filestore keyword to consider a match if filestore output not
enabled.
pub files_logged: u32,
pub files_stored: u32,
+ pub file_flags: u16,
+
/// detection engine flags for use by detection engine
detect_flags_ts: u64,
detect_flags_tc: u64,
files_opened: 0,
files_logged: 0,
files_stored: 0,
+ file_flags: 0,
detect_flags_ts: 0,
detect_flags_tc: 0,
de_state: std::ptr::null_mut(),
pub fn set_event(&mut self, event: u8) {
core::sc_app_layer_decoder_events_set_event_raw(&mut self.events, event as u8);
}
+
+ pub fn update_file_flags(&mut self, state_flags: u16) {
+ if (self.file_flags & state_flags) != state_flags {
+ SCLogDebug!("updating tx file_flags {:04x} with state flags {:04x}", self.file_flags, state_flags);
+ self.file_flags |= state_flags;
+ }
+ }
}
#[macro_export]
pub localstorage_free: Option<LocalStorageFreeFn>,
/// Function to get files
- pub get_files: Option<GetFilesFn>,
+ pub get_tx_files: Option<GetTxFilesFn>,
/// Function to get the TX iterator
pub get_tx_iterator: Option<GetTxIteratorFn>,
pub type GetEventInfoByIdFn = unsafe extern "C" fn (c_int, *mut *const c_char, *mut AppLayerEventType) -> i8;
pub type LocalStorageNewFn = extern "C" fn () -> *mut c_void;
pub type LocalStorageFreeFn = extern "C" fn (*mut c_void);
-pub type GetFilesFn = unsafe
-extern "C" fn (*mut c_void, u8) -> *mut FileContainer;
+pub type GetTxFilesFn = unsafe extern "C" fn (*mut c_void, u8) -> *mut FileContainer;
pub type GetTxIteratorFn = unsafe extern "C" fn (ipproto: u8, alproto: AppProto,
state: *mut c_void,
min_tx_id: u64,
get_eventinfo_byid : Some(TemplateEvent::get_event_info_by_id),
localstorage_new: None,
localstorage_free: None,
- get_files: None,
+ get_tx_files: None,
get_tx_iterator: Some(applayer::state_get_tx_iterator::<TemplateState, TemplateTransaction>),
get_tx_data: rs_template_get_tx_data,
get_state_data: rs_template_get_state_data,
pub const IPPROTO_TCP : u8 = 6;
pub const IPPROTO_UDP : u8 = 17;
+/*
macro_rules!BIT_U8 {
($x:expr) => (1 << $x);
}
-
+*/
macro_rules!BIT_U16 {
($x:expr) => (1 << $x);
}
pub type SCFileContainerRecycle = extern "C" fn (
file_container: &FileContainer);
-pub type SCFileSetTx = extern "C" fn (
- file: &FileContainer,
- tx_id: u64);
-
// A Suricata context that is passed in from C. This is alternative to
// using functions from Suricata directly, so they can be wrapped so
// Rust unit tests will still compile when they are not linked
pub FileAppendGAP: SCFileAppendGAPById,
pub FileContainerRecycle: SCFileContainerRecycle,
pub FilePrune: SCFilePrune,
- pub FileSetTx: SCFileSetTx,
pub AppLayerRegisterParser: extern fn(parser: *const crate::applayer::RustParser, alproto: AppProto) -> std::os::raw::c_int,
}
get_eventinfo_byid : None,
localstorage_new: None,
localstorage_free: None,
- get_files: None,
+ get_tx_files: None,
get_tx_iterator: Some(applayer::state_get_tx_iterator::<DCERPCState, DCERPCTransaction>),
get_tx_data: rs_dcerpc_get_tx_data,
get_state_data: rs_dcerpc_get_state_data,
get_eventinfo_byid: None,
localstorage_new: None,
localstorage_free: None,
- get_files: None,
+ get_tx_files: None,
get_tx_iterator: Some(applayer::state_get_tx_iterator::<DCERPCUDPState, DCERPCTransaction>),
get_tx_data: rs_dcerpc_udp_get_tx_data,
get_state_data: rs_dcerpc_udp_get_state_data,
get_eventinfo_byid : Some(DHCPEvent::get_event_info_by_id),
localstorage_new : None,
localstorage_free : None,
- get_files : None,
+ get_tx_files : None,
get_tx_iterator : Some(applayer::state_get_tx_iterator::<DHCPState, DHCPTransaction>),
get_tx_data : rs_dhcp_get_tx_data,
get_state_data : rs_dhcp_get_state_data,
get_eventinfo_byid: Some(DNSEvent::get_event_info_by_id),
localstorage_new: None,
localstorage_free: None,
- get_files: None,
+ get_tx_files: None,
get_tx_iterator: Some(crate::applayer::state_get_tx_iterator::<DNSState, DNSTransaction>),
get_tx_data: rs_dns_state_get_tx_data,
get_state_data: rs_dns_get_state_data,
get_eventinfo_byid: Some(DNSEvent::get_event_info_by_id),
localstorage_new: None,
localstorage_free: None,
- get_files: None,
+ get_tx_files: None,
get_tx_iterator: Some(crate::applayer::state_get_tx_iterator::<DNSState, DNSTransaction>),
get_tx_data: rs_dns_state_get_tx_data,
get_state_data: rs_dns_get_state_data,
-/* Copyright (C) 2017 Open Information Security Foundation
+/* Copyright (C) 2017-2021 Open Information Security Foundation
*
* You can copy, redistribute or modify this Program under the terms of
* the GNU General Public License version 2 as published by the Free
// Defined in util-file.h
extern {
- pub fn FileFlowToFlags(flow: *const Flow, flags: u8) -> u16;
+ pub fn FileFlowFlagsToFlags(flow_file_flags: u16, flags: u8) -> u16;
}
pub const FILE_USE_DETECT: u16 = BIT_U16!(13);
}
pub fn free(&mut self) {
SCLogDebug!("freeing self");
- match unsafe {SC} {
- None => panic!("BUG no suricata_config"),
- Some(c) => {
- (c.FileContainerRecycle)(self);
- },
+ if let Some(c) = unsafe {SC} {
+ (c.FileContainerRecycle)(self);
}
}
}
}
}
-
- pub fn file_set_txid_on_last_file(&mut self, tx_id: u64) {
- match unsafe {SC} {
- None => panic!("BUG no suricata_config"),
- Some(c) => {
- (c.FileSetTx)(self, tx_id);
- }
- }
- }
}
{
let r = files.file_open(config, &self.track_id, name, flags);
if r == 0 {
- files.file_set_txid_on_last_file(self.tx_id);
self.file_open = true;
}
r
//temporary escaped header for detection
//must be attached to transaction for memory management (be freed at the right time)
pub escaped: Vec<Vec<u8>>,
+
+ pub files: Files,
}
impl Transaction for HTTP2Transaction {
ft_tc: FileTransferTracker::new(),
ft_ts: FileTransferTracker::new(),
escaped: Vec::with_capacity(16),
+ files: Files::default(),
}
}
}
}
+ pub fn update_file_flags(&mut self, flow_file_flags: u16) {
+ self.files.flags_ts = unsafe { FileFlowFlagsToFlags(flow_file_flags, STREAM_TOSERVER) | FILE_USE_DETECT };
+ self.files.flags_tc = unsafe { FileFlowFlagsToFlags(flow_file_flags, STREAM_TOCLIENT) | FILE_USE_DETECT };
+ }
+
fn decompress<'a>(
- &'a mut self, input: &'a [u8], dir: Direction, sfcm: &'static SuricataFileContext,
- over: bool, files: &mut FileContainer, flags: u16, flow: *const Flow,
+ &'a mut self, input: &'a [u8], dir: Direction, sfcm: &'static SuricataFileContext, over: bool, flow: *const Flow,
) -> io::Result<()> {
let mut output = Vec::with_capacity(decompression::HTTP2_DECOMPRESSION_CHUNK_SIZE);
let decompressed = self.decoder.decompress(input, &mut output, dir)?;
) {
match range::http2_parse_check_content_range(&value) {
Ok((_, v)) => {
- range::http2_range_open(self, &v, flow, sfcm, flags, decompressed);
+ range::http2_range_open(self, &v, flow, sfcm, Direction::ToClient, decompressed);
if over && !self.file_range.is_null() {
- range::http2_range_close(self, files, flags, &[])
+ range::http2_range_close(self, Direction::ToClient, &[])
}
}
_ => {
} else {
if !self.file_range.is_null() {
if over {
- range::http2_range_close(self, files, flags, decompressed)
+ range::http2_range_close(self, Direction::ToClient, decompressed)
} else {
range::http2_range_append(self.file_range, decompressed)
}
}
}
+ let (files, flags) = self.files.get(Direction::ToClient);
self.ft_tc.new_chunk(
sfcm,
files,
if !self.ft_ts.file_open {
self.tx_data.incr_files_opened();
}
+ let (files, flags) = self.files.get(Direction::ToServer);
self.ft_ts.new_chunk(
sfcm,
files,
dynamic_headers_tc: HTTP2DynTable,
transactions: VecDeque<HTTP2Transaction>,
progress: HTTP2ConnectionState,
- pub files: Files,
}
impl State<HTTP2Transaction> for HTTP2State {
dynamic_headers_tc: HTTP2DynTable::new(),
transactions: VecDeque::new(),
progress: HTTP2ConnectionState::Http2StateInit,
- files: Files::default(),
}
}
None => panic!("BUG no suricata_config"),
Some(c) => {
(c.HTPFileCloseHandleRange)(
- &mut self.files.files_tc,
+ &mut tx.files.files_tc,
0,
tx.file_range,
std::ptr::null_mut(),
None => panic!("BUG no suricata_config"),
Some(c) => {
(c.HTPFileCloseHandleRange)(
- &mut self.files.files_tc,
+ &mut tx.files.files_tc,
0,
tx.file_range,
std::ptr::null_mut(),
pub fn get_tx(&mut self, tx_id: u64) -> Option<&HTTP2Transaction> {
for tx in &mut self.transactions {
if tx.tx_id == tx_id + 1 {
+ tx.tx_data.update_file_flags(self.state_data.file_flags);
+ tx.update_file_flags(tx.tx_data.file_flags);
return Some(tx);
}
}
self.tx_id += 1;
tx.tx_id = self.tx_id;
tx.state = HTTP2TransactionState::HTTP2StateGlobal;
+ tx.tx_data.update_file_flags(self.state_data.file_flags);
+ tx.update_file_flags(tx.tx_data.file_flags);
self.transactions.push_back(tx);
return self.transactions.back_mut().unwrap();
}
self.set_event(HTTP2Event::StreamIdReuse);
}
}
- return &mut self.transactions[index - 1];
+
+ let tx = &mut self.transactions[index - 1];
+ tx.tx_data.update_file_flags(self.state_data.file_flags);
+ tx.update_file_flags(tx.tx_data.file_flags);
+ return tx;
} else {
let mut tx = HTTP2Transaction::new();
self.tx_id += 1;
}
}
}
+ tx.tx_data.update_file_flags(self.state_data.file_flags);
+ tx.update_file_flags(tx.tx_data.file_flags);
self.transactions.push_back(tx);
return self.transactions.back_mut().unwrap();
}
//borrow checker forbids to reuse directly tx
let index = self.find_tx_index(sid);
if index > 0 {
- let tx_same = &mut self.transactions[index - 1];
- let (files, flags) = self.files.get(dir);
+ let mut tx_same = &mut self.transactions[index - 1];
+ if dir == Direction::ToServer {
+ tx_same.ft_tc.tx_id = tx_same.tx_id - 1;
+ } else {
+ tx_same.ft_ts.tx_id = tx_same.tx_id - 1;
+ };
+
match tx_same.decompress(
&rem[..hlsafe],
dir,
sfcm,
over,
- files,
- flags,
flow,
) {
Err(_e) => {
) -> AppLayerResult {
let state = cast_pointer!(state, HTTP2State);
let buf = stream_slice.as_slice();
-
- state.files.flags_ts = FileFlowToFlags(flow, Direction::ToServer.into());
- state.files.flags_ts = state.files.flags_ts | FILE_USE_DETECT;
return state.parse_ts(buf, flow);
}
) -> AppLayerResult {
let state = cast_pointer!(state, HTTP2State);
let buf = stream_slice.as_slice();
- state.files.flags_tc = FileFlowToFlags(flow, Direction::ToClient.into());
- state.files.flags_tc = state.files.flags_tc | FILE_USE_DETECT;
return state.parse_tc(buf, flow);
}
#[no_mangle]
pub unsafe extern "C" fn rs_http2_getfiles(
- state: *mut std::os::raw::c_void, direction: u8,
+ tx: *mut std::os::raw::c_void, direction: u8,
) -> *mut FileContainer {
- let state = cast_pointer!(state, HTTP2State);
+ let tx = cast_pointer!(tx, HTTP2Transaction);
if direction == Direction::ToClient.into() {
- &mut state.files.files_tc as *mut FileContainer
+ &mut tx.files.files_tc as *mut FileContainer
} else {
- &mut state.files.files_ts as *mut FileContainer
+ &mut tx.files.files_ts as *mut FileContainer
}
}
get_eventinfo_byid: Some(HTTP2Event::get_event_info_by_id),
localstorage_new: None,
localstorage_free: None,
- get_files: Some(rs_http2_getfiles),
+ get_tx_files: Some(rs_http2_getfiles),
get_tx_iterator: Some(applayer::state_get_tx_iterator::<HTTP2State, HTTP2Transaction>),
get_tx_data: rs_http2_get_tx_data,
get_state_data: rs_http2_get_state_data,
use crate::core::{
Direction, Flow, HttpRangeContainerBlock, StreamingBufferConfig, SuricataFileContext, SC,
};
-use crate::filecontainer::FileContainer;
use crate::http2::http2::HTTP2Transaction;
use nom7::branch::alt;
pub fn http2_range_open(
tx: &mut HTTP2Transaction, v: &HTTPContentRange, flow: *const Flow,
- cfg: &'static SuricataFileContext, flags: u16, data: &[u8],
+ cfg: &'static SuricataFileContext, dir: Direction, data: &[u8],
) {
if v.end <= 0 || v.size <= 0 {
// skipped for incomplete range information
// whole file in one range
return;
}
+ let (_, flags) = tx.files.get(dir);
if let Ok((key, index)) = http2_range_key_get(tx) {
let name = &key[index..];
tx.file_range = unsafe {
}
pub fn http2_range_close(
- tx: &mut HTTP2Transaction, files: &mut FileContainer, flags: u16, data: &[u8],
+ tx: &mut HTTP2Transaction, dir: Direction, data: &[u8],
) {
let added = if let Some(c) = unsafe { SC } {
+ let (files, flags) = tx.files.get(dir);
let added = (c.HTPFileCloseHandleRange)(
files,
flags,
get_eventinfo_byid: Some(IkeEvent::get_event_info_by_id),
localstorage_new: None,
localstorage_free: None,
- get_files: None,
+ get_tx_files: None,
get_tx_iterator: Some(applayer::state_get_tx_iterator::<IKEState, IKETransaction>),
get_tx_data: rs_ike_get_tx_data,
get_state_data: rs_ike_get_state_data,
| IkeTransformEncType::ENCR_DES_IV32
| IkeTransformEncType::ENCR_NULL => {
SCLogDebug!("Weak Encryption: {:?}", enc);
- // XXX send event only if direction == STREAM_TOCLIENT ?
+ // XXX send event only if direction == Direction::ToClient ?
tx.set_event(IkeEvent::WeakCryptoEnc);
}
_ => (),
get_eventinfo_byid : Some(KRB5Event::get_event_info_by_id),
localstorage_new : None,
localstorage_free : None,
- get_files : None,
+ get_tx_files : None,
get_tx_iterator : Some(applayer::state_get_tx_iterator::<KRB5State, KRB5Transaction>),
get_tx_data : rs_krb5_get_tx_data,
get_state_data : rs_krb5_get_state_data,
get_eventinfo_byid: Some(ModbusEvent::get_event_info_by_id),
localstorage_new: None,
localstorage_free: None,
- get_files: None,
+ get_tx_files: None,
get_tx_iterator: Some(applayer::state_get_tx_iterator::<ModbusState, ModbusTransaction>),
get_tx_data: rs_modbus_state_get_tx_data,
get_state_data: rs_modbus_get_state_data,
get_eventinfo_byid: Some(MQTTEvent::get_event_info_by_id),
localstorage_new: None,
localstorage_free: None,
- get_files: None,
+ get_tx_files: None,
get_tx_iterator: Some(crate::applayer::state_get_tx_iterator::<MQTTState, MQTTTransaction>),
get_tx_data: rs_mqtt_get_tx_data,
get_state_data: rs_mqtt_get_state_data,
/// file tracker for a single file. Boxed so that we don't use
/// as much space if we're not a file tx.
pub file_tracker: FileTransferTracker,
+
+ /// storage for the actual file
+ pub files: Files,
}
impl NFSTransactionFile {
..Default::default()
}
}
+ pub fn update_file_flags(&mut self, flow_file_flags: u16) {
+ self.files.flags_ts = unsafe { FileFlowFlagsToFlags(flow_file_flags, STREAM_TOSERVER) | FILE_USE_DETECT };
+ self.files.flags_tc = unsafe { FileFlowFlagsToFlags(flow_file_flags, STREAM_TOCLIENT) | FILE_USE_DETECT };
+ }
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn rs_nfs_gettxfiles(tx_ptr: *mut std::ffi::c_void, direction: u8) -> * mut FileContainer {
+ let tx = cast_pointer!(tx_ptr, NFSTransaction);
+ if let Some(NFSTransactionTypeData::FILE(ref mut tdf)) = tx.type_data {
+ let (files, _flags) = tdf.files.get(direction.into());
+ files
+ } else {
+ std::ptr::null_mut()
+ }
}
#[derive(Debug)]
/// transactions list
pub transactions: Vec<NFSTransaction>,
- pub files: Files,
-
/// partial record tracking
pub ts_chunk_xid: u32,
pub tc_chunk_xid: u32,
requestmap:HashMap::new(),
namemap:HashMap::new(),
transactions: Vec::new(),
- files:Files::default(),
ts_chunk_xid:0,
tc_chunk_xid:0,
ts_chunk_left:0,
if self.ts > f.post_gap_ts {
tx.request_done = true;
tx.response_done = true;
- let (files, flags) = self.files.get(tx.file_tx_direction);
+ let (files, flags) = f.files.get(tx.file_tx_direction);
f.file_tracker.trunc(files, flags);
} else {
post_gap_txs = true;
}
pub fn new_file_tx(&mut self, file_handle: &Vec<u8>, file_name: &Vec<u8>, direction: Direction)
- -> (&mut NFSTransaction, &mut FileContainer, u16)
+ -> &mut NFSTransaction
{
let mut tx = self.new_tx();
tx.file_name = file_name.to_vec();
tx.type_data = Some(NFSTransactionTypeData::FILE(NFSTransactionFile::new()));
if let Some(NFSTransactionTypeData::FILE(ref mut d)) = tx.type_data {
d.file_tracker.tx_id = tx.id - 1;
+ tx.tx_data.update_file_flags(self.state_data.file_flags);
+ d.update_file_flags(tx.tx_data.file_flags);
}
tx.tx_data.init_files_opened();
SCLogDebug!("new_file_tx: TX FILE created: ID {} NAME {}",
tx.id, String::from_utf8_lossy(file_name));
self.transactions.push(tx);
let tx_ref = self.transactions.last_mut();
- let (files, flags) = self.files.get(direction);
- return (tx_ref.unwrap(), files, flags)
+ return tx_ref.unwrap();
}
pub fn get_file_tx_by_handle(&mut self, file_handle: &Vec<u8>, direction: Direction)
- -> Option<(&mut NFSTransaction, &mut FileContainer, u16)>
+ -> Option<&mut NFSTransaction>
{
let fh = file_handle.to_vec();
for tx in &mut self.transactions {
- if tx.is_file_tx && !tx.is_file_closed &&
- direction == tx.file_tx_direction &&
- tx.file_handle == fh
- {
- SCLogDebug!("Found NFS file TX with ID {} XID {:04X}", tx.id, tx.xid);
- let (files, flags) = self.files.get(direction);
- return Some((tx, files, flags));
+ if let Some(NFSTransactionTypeData::FILE(ref mut d)) = tx.type_data {
+ if tx.is_file_tx && !tx.is_file_closed &&
+ direction == tx.file_tx_direction &&
+ tx.file_handle == fh
+ {
+ tx.tx_data.update_file_flags(self.state_data.file_flags);
+ d.update_file_flags(tx.tx_data.file_flags);
+ SCLogDebug!("Found NFS file TX with ID {} XID {:04X}", tx.id, tx.xid);
+ return Some(tx);
+ }
}
}
SCLogDebug!("Failed to find NFS TX with handle {:?}", file_handle);
};
let found = match self.get_file_tx_by_handle(&file_handle, Direction::ToServer) {
- Some((tx, files, flags)) => {
+ Some(tx) => {
if let Some(NFSTransactionTypeData::FILE(ref mut tdf)) = tx.type_data {
+ let (files, flags) = tdf.files.get(Direction::ToServer);
filetracker_newchunk(&mut tdf.file_tracker, files, flags,
&file_name, w.file_data, w.offset,
w.file_len, fill_bytes as u8, is_last, &r.hdr.xid);
None => { false },
};
if !found {
- let (tx, files, flags) = self.new_file_tx(&file_handle, &file_name, Direction::ToServer);
+ let tx = self.new_file_tx(&file_handle, &file_name, Direction::ToServer);
if let Some(NFSTransactionTypeData::FILE(ref mut tdf)) = tx.type_data {
+ let (files, flags) = tdf.files.get(Direction::ToServer);
filetracker_newchunk(&mut tdf.file_tracker, files, flags,
&file_name, w.file_data, w.offset,
w.file_len, fill_bytes as u8, is_last, &r.hdr.xid);
let ssn_gap = self.ts_ssn_gap | self.tc_ssn_gap;
// get the tx and update it
let consumed = match self.get_file_tx_by_handle(&file_handle, direction) {
- Some((tx, files, flags)) => {
+ Some(tx) => {
if let Some(NFSTransactionTypeData::FILE(ref mut tdf)) = tx.type_data {
+ let (files, flags) = tdf.files.get(direction);
if ssn_gap {
let queued_data = tdf.file_tracker.get_queued_size();
if queued_data > 2000000 { // TODO should probably be configurable
SCLogDebug!("partial data? {}", is_partial);
let found = match self.get_file_tx_by_handle(&file_handle, Direction::ToClient) {
- Some((tx, files, flags)) => {
+ Some(tx) => {
SCLogDebug!("updated TX {:?}", tx);
if let Some(NFSTransactionTypeData::FILE(ref mut tdf)) = tx.type_data {
+ let (files, flags) = tdf.files.get(Direction::ToClient);
filetracker_newchunk(&mut tdf.file_tracker, files, flags,
&file_name, reply.data, chunk_offset,
reply.count, fill_bytes as u8, is_last, &r.hdr.xid);
None => { false },
};
if !found {
- let (tx, files, flags) = self.new_file_tx(&file_handle, &file_name, Direction::ToClient);
+ let tx = self.new_file_tx(&file_handle, &file_name, Direction::ToClient);
if let Some(NFSTransactionTypeData::FILE(ref mut tdf)) = tx.type_data {
+ let (files, flags) = tdf.files.get(Direction::ToClient);
filetracker_newchunk(&mut tdf.file_tracker, files, flags,
&file_name, reply.data, chunk_offset,
reply.count, fill_bytes as u8, is_last, &r.hdr.xid);
}
AppLayerResult::ok()
}
-
- fn getfiles(&mut self, direction: Direction) -> * mut FileContainer {
- //SCLogDebug!("direction: {:?}", direction);
- if direction == Direction::ToClient {
- &mut self.files.files_tc as *mut FileContainer
- } else {
- &mut self.files.files_ts as *mut FileContainer
- }
- }
- fn setfileflags(&mut self, direction: Direction, flags: u16) {
- SCLogDebug!("direction: {:?}, flags: {}", direction, flags);
- if direction == Direction::ToClient {
- self.files.flags_tc = flags;
- } else {
- self.files.flags_ts = flags;
- }
- }
}
/// Returns *mut NFSState
{
let state = cast_pointer!(state, NFSState);
let flow = cast_pointer!(flow, Flow);
- let file_flags = FileFlowToFlags(flow, Direction::ToServer.into());
- rs_nfs_setfileflags(Direction::ToServer.into(), state, file_flags);
if stream_slice.is_gap() {
return rs_nfs_parse_request_tcp_gap(state, stream_slice.gap_size());
{
let state = cast_pointer!(state, NFSState);
let flow = cast_pointer!(flow, Flow);
- let file_flags = FileFlowToFlags(flow, Direction::ToClient.into());
- rs_nfs_setfileflags(Direction::ToClient.into(), state, file_flags);
if stream_slice.is_gap() {
return rs_nfs_parse_response_tcp_gap(state, stream_slice.gap_size());
) -> AppLayerResult
{
let state = cast_pointer!(state, NFSState);
- let file_flags = FileFlowToFlags(f, Direction::ToServer.into());
- rs_nfs_setfileflags(Direction::ToServer.into(), state, file_flags);
SCLogDebug!("parsing {} bytes of request data", stream_slice.len());
state.parse_udp_ts(f, &stream_slice)
) -> AppLayerResult
{
let state = cast_pointer!(state, NFSState);
- let file_flags = FileFlowToFlags(f, Direction::ToClient.into());
- rs_nfs_setfileflags(Direction::ToClient.into(), state, file_flags);
SCLogDebug!("parsing {} bytes of response data", stream_slice.len());
state.parse_udp_tc(f, &stream_slice)
}
}
}
-#[no_mangle]
-pub unsafe extern "C" fn rs_nfs_getfiles(ptr: *mut std::ffi::c_void, direction: u8) -> * mut FileContainer {
- if ptr.is_null() { panic!("NULL ptr"); };
- let parser = cast_pointer!(ptr, NFSState);
- parser.getfiles(direction.into())
-}
-#[no_mangle]
-pub unsafe extern "C" fn rs_nfs_setfileflags(direction: u8, ptr: *mut NFSState, flags: u16) {
- if ptr.is_null() { panic!("NULL ptr"); };
- let parser = &mut *ptr;
- SCLogDebug!("direction {} flags {}", direction, flags);
- parser.setfileflags(direction.into(), flags)
-}
-
// Parser name as a C style string.
const PARSER_NAME: &'static [u8] = b"nfs\0";
get_eventinfo_byid : Some(NFSEvent::get_event_info_by_id),
localstorage_new: None,
localstorage_free: None,
- get_files: Some(rs_nfs_getfiles),
+ get_tx_files: Some(rs_nfs_gettxfiles),
get_tx_iterator: Some(applayer::state_get_tx_iterator::<NFSState, NFSTransaction>),
get_tx_data: rs_nfs_get_tx_data,
get_state_data: rs_nfs_get_state_data,
get_eventinfo_byid : Some(NFSEvent::get_event_info_by_id),
localstorage_new: None,
localstorage_free: None,
- get_files: Some(rs_nfs_getfiles),
+ get_tx_files: Some(rs_nfs_gettxfiles),
get_tx_iterator: Some(applayer::state_get_tx_iterator::<NFSState, NFSTransaction>),
get_tx_data: rs_nfs_get_tx_data,
get_state_data: rs_nfs_get_state_data,
SCLogDebug!("Protocol detector and parser disabled for nfs.");
}
}
-
SCLogDebug!("COMMIT, closing shop");
if let Ok((_, rd)) = parse_nfs3_request_commit(r.prog_data) {
let file_handle = rd.handle.value.to_vec();
- if let Some((tx, files, flags)) = self.get_file_tx_by_handle(&file_handle, Direction::ToServer) {
+ if let Some(tx) = self.get_file_tx_by_handle(&file_handle, Direction::ToServer) {
if let Some(NFSTransactionTypeData::FILE(ref mut tdf)) = tx.type_data {
+ let (files, flags) = tdf.files.get(Direction::ToServer);
tdf.chunk_count += 1;
tdf.file_additional_procs.push(NFSPROC3_COMMIT);
tdf.file_tracker.close(files, flags);
} else if r.procedure == NFSPROC3_READ {
let found = match self.get_file_tx_by_handle(&xidmap.file_handle, Direction::ToClient) {
- Some((_, _, _)) => true,
+ Some(_) => true,
None => false,
};
if !found {
- let (tx, _, _) = self.new_file_tx(&xidmap.file_handle, &xidmap.file_name, Direction::ToClient);
+ let tx = self.new_file_tx(&xidmap.file_handle, &xidmap.file_name, Direction::ToClient);
tx.procedure = NFSPROC3_READ;
tx.xid = r.hdr.xid;
tx.is_first = true;
};
let found = match self.get_file_tx_by_handle(&file_handle, Direction::ToServer) {
- Some((tx, files, flags)) => {
+ Some(tx) => {
if let Some(NFSTransactionTypeData::FILE(ref mut tdf)) = tx.type_data {
- filetracker_newchunk(
- &mut tdf.file_tracker,
- files,
- flags,
- &file_name,
- w.data,
- w.offset,
- w.write_len,
- fill_bytes as u8,
- is_last,
- &r.hdr.xid,
- );
+ let (files, flags) = tdf.files.get(Direction::ToServer);
+ filetracker_newchunk(&mut tdf.file_tracker, files, flags,
+ &file_name, w.data, w.offset,
+ w.write_len, fill_bytes as u8, is_last, &r.hdr.xid);
tdf.chunk_count += 1;
if is_last {
tdf.file_last_xid = r.hdr.xid;
None => false,
};
if !found {
- let (tx, files, flags) = self.new_file_tx(&file_handle, &file_name, Direction::ToServer);
+ let tx = self.new_file_tx(&file_handle, &file_name, Direction::ToServer);
if let Some(NFSTransactionTypeData::FILE(ref mut tdf)) = tx.type_data {
- filetracker_newchunk(
- &mut tdf.file_tracker,
- files,
- flags,
- &file_name,
- w.data,
- w.offset,
- w.write_len,
- fill_bytes as u8,
- is_last,
- &r.hdr.xid,
- );
+ let (files, flags) = tdf.files.get(Direction::ToServer);
+ filetracker_newchunk(&mut tdf.file_tracker, files, flags,
+ &file_name, w.data, w.offset,
+ w.write_len, fill_bytes as u8, is_last, &r.hdr.xid);
tx.procedure = NFSPROC4_WRITE;
tx.xid = r.hdr.xid;
tx.is_first = true;
SCLogDebug!("COMMIT, closing shop");
let file_handle = fh.to_vec();
- if let Some((tx, files, flags)) = self.get_file_tx_by_handle(&file_handle, Direction::ToServer)
- {
+ if let Some(tx) = self.get_file_tx_by_handle(&file_handle, Direction::ToServer) {
if let Some(NFSTransactionTypeData::FILE(ref mut tdf)) = tx.type_data {
+ let (files, flags) = tdf.files.get(Direction::ToServer);
tdf.file_tracker.close(files, flags);
tdf.file_last_xid = r.hdr.xid;
tx.is_last = true;
get_eventinfo_byid : Some(NTPEvent::get_event_info_by_id),
localstorage_new : None,
localstorage_free : None,
- get_files : None,
+ get_tx_files : None,
get_tx_iterator : Some(applayer::state_get_tx_iterator::<NTPState, NTPTransaction>),
get_tx_data : rs_ntp_get_tx_data,
get_state_data : rs_ntp_get_state_data,
get_eventinfo_byid: None,
localstorage_new: None,
localstorage_free: None,
- get_files: None,
+ get_tx_files: None,
get_tx_iterator: Some(
crate::applayer::state_get_tx_iterator::<PgsqlState, PgsqlTransaction>,
),
get_eventinfo_byid: Some(QuicEvent::get_event_info_by_id),
localstorage_new: None,
localstorage_free: None,
- get_files: None,
+ get_tx_files: None,
get_tx_iterator: Some(rs_quic_state_get_tx_iterator),
get_tx_data: rs_quic_get_tx_data,
get_state_data: rs_quic_get_state_data,
get_eventinfo_byid: None,
localstorage_new: None,
localstorage_free: None,
- get_files: None,
+ get_tx_files: None,
get_tx_iterator: Some(applayer::state_get_tx_iterator::<RdpState, RdpTransaction>),
get_tx_data: rs_rdp_get_tx_data,
get_state_data: rs_rdp_get_state_data,
get_eventinfo_byid: None,
localstorage_new: None,
localstorage_free: None,
- get_files: None,
+ get_tx_files: None,
get_tx_iterator: Some(applayer::state_get_tx_iterator::<RFBState, RFBTransaction>),
get_tx_data: rs_rfb_get_tx_data,
get_state_data: rs_rfb_get_state_data,
get_eventinfo_byid: Some(SIPEvent::get_event_info_by_id),
localstorage_new: None,
localstorage_free: None,
- get_files: None,
+ get_tx_files: None,
get_tx_iterator: Some(applayer::state_get_tx_iterator::<SIPState, SIPTransaction>),
get_tx_data: rs_sip_get_tx_data,
get_state_data: rs_sip_get_state_data,
-/* Copyright (C) 2018 Open Information Security Foundation
+/* Copyright (C) 2018-2022 Open Information Security Foundation
*
* You can copy, redistribute or modify this Program under the terms of
* the GNU General Public License version 2 as published by the Free
/// after a gap, this will be set to a time in the future. If the file
/// receives no updates before that, it will be considered complete.
pub post_gap_ts: u64,
+ pub files: Files,
}
impl SMBTransactionFile {
..Default::default()
}
}
+
+ pub fn update_file_flags(&mut self, flow_file_flags: u16) {
+ self.files.flags_ts = unsafe { FileFlowFlagsToFlags(flow_file_flags, STREAM_TOSERVER) | FILE_USE_DETECT };
+ self.files.flags_tc = unsafe { FileFlowFlagsToFlags(flow_file_flags, STREAM_TOCLIENT) | FILE_USE_DETECT };
+ }
}
/// little wrapper around the FileTransferTracker::new_chunk method
impl SMBState {
pub fn new_file_tx(&mut self, fuid: &Vec<u8>, file_name: &Vec<u8>, direction: Direction)
- -> (&mut SMBTransaction, &mut FileContainer, u16)
+ -> &mut SMBTransaction
{
let mut tx = self.new_tx();
tx.type_data = Some(SMBTransactionTypeData::FILE(SMBTransactionFile::new()));
d.fuid = fuid.to_vec();
d.file_name = file_name.to_vec();
d.file_tracker.tx_id = tx.id - 1;
+ tx.tx_data.update_file_flags(self.state_data.file_flags);
+ d.update_file_flags(tx.tx_data.file_flags);
},
_ => { },
}
tx.id, String::from_utf8_lossy(file_name));
self.transactions.push(tx);
let tx_ref = self.transactions.last_mut();
- let (files, flags) = self.files.get(direction);
- return (tx_ref.unwrap(), files, flags)
+ return tx_ref.unwrap();
}
pub fn get_file_tx_by_fuid(&mut self, fuid: &Vec<u8>, direction: Direction)
- -> Option<(&mut SMBTransaction, &mut FileContainer, u16)>
+ -> Option<&mut SMBTransaction>
{
let f = fuid.to_vec();
for tx in &mut self.transactions {
if found {
SCLogDebug!("SMB: Found SMB file TX with ID {}", tx.id);
- let (files, flags) = self.files.get(direction);
- return Some((tx, files, flags));
+ if let Some(SMBTransactionTypeData::FILE(ref mut d)) = tx.type_data {
+ tx.tx_data.update_file_flags(self.state_data.file_flags);
+ d.update_file_flags(tx.tx_data.file_flags);
+ }
+ return Some(tx);
}
}
SCLogDebug!("SMB: Failed to find SMB TX with FUID {:?}", fuid);
return None;
}
- fn getfiles(&mut self, direction: Direction) -> * mut FileContainer {
- //SCLogDebug!("direction: {:?}", direction);
- if direction == Direction::ToClient {
- &mut self.files.files_tc as *mut FileContainer
- } else {
- &mut self.files.files_ts as *mut FileContainer
- }
- }
- fn setfileflags(&mut self, direction: Direction, flags: u16) {
- SCLogDebug!("direction: {:?}, flags: {}", direction, flags);
- if direction == Direction::ToClient {
- self.files.flags_tc = flags;
- } else {
- self.files.flags_ts = flags;
- }
- }
-
// update in progress chunks for file transfers
// return how much data we consumed
pub fn filetracker_update(&mut self, direction: Direction, data: &[u8], gap_size: u32) -> u32 {
let ssn_gap = self.ts_ssn_gap | self.tc_ssn_gap;
// get the tx and update it
let consumed = match self.get_file_tx_by_fuid(&file_handle, direction) {
- Some((tx, files, flags)) => {
+ Some(tx) => {
if let Some(SMBTransactionTypeData::FILE(ref mut tdf)) = tx.type_data {
+ let (files, flags) = tdf.files.get(direction);
if ssn_gap {
let queued_data = tdf.file_tracker.get_queued_size();
if queued_data > 2000000 { // TODO should probably be configurable
}
#[no_mangle]
-pub unsafe extern "C" fn rs_smb_getfiles(ptr: *mut std::ffi::c_void, direction: u8) -> * mut FileContainer {
- if ptr.is_null() { panic!("NULL ptr"); };
- let parser = cast_pointer!(ptr, SMBState);
- parser.getfiles(direction.into())
-}
-
-#[no_mangle]
-pub unsafe extern "C" fn rs_smb_setfileflags(direction: u8, ptr: *mut SMBState, flags: u16) {
- if ptr.is_null() { panic!("NULL ptr"); };
- let parser = &mut *ptr;
- SCLogDebug!("direction {} flags {}", direction, flags);
- parser.setfileflags(direction.into(), flags)
+pub unsafe extern "C" fn rs_smb_gettxfiles(tx_ptr: *mut std::ffi::c_void, direction: u8) -> * mut FileContainer {
+ let tx = cast_pointer!(tx_ptr, SMBTransaction);
+ if let Some(SMBTransactionTypeData::FILE(ref mut tdf)) = tx.type_data {
+ let (files, _flags) = tdf.files.get(direction.into());
+ files
+ } else {
+ std::ptr::null_mut()
+ }
}
-/* Copyright (C) 2017-2020 Open Information Security Foundation
+/* Copyright (C) 2017-2022 Open Information Security Foundation
*
* You can copy, redistribute or modify this Program under the terms of
* the GNU General Public License version 2 as published by the Free
use crate::applayer::*;
use crate::frames::*;
use crate::conf::*;
-use crate::filecontainer::*;
use crate::applayer::{AppLayerResult, AppLayerTxData, AppLayerEvent};
use crate::smb::nbss_records::*;
}
pub fn free(&mut self) {
+ SCLogDebug!("SMB TX {:p} free ID {}", &self, self.id);
debug_validate_bug_on!(self.tx_data.files_opened > 1);
debug_validate_bug_on!(self.tx_data.files_logged > 1);
}
// requests for DCERPC.
pub ssnguid2vec_map: HashMap<SMBHashKeyHdrGuid, Vec<u8>>,
- pub files: Files,
-
skip_ts: u32,
skip_tc: u32,
ssn2vecoffset_map:HashMap::new(),
ssn2tree_map:HashMap::new(),
ssnguid2vec_map:HashMap::new(),
- files: Files::default(),
skip_ts:0,
skip_tc:0,
file_ts_left:0,
}
SCLogDebug!("Found SMB TX: id {} ver:{} cmd:{} progress {}/{} type_data {:?}",
tx.id, ver, _smbcmd, tx.request_done, tx.response_done, tx.type_data);
+ /* hack: apply flow file flags to file tx here to make sure its propegated */
+ if let Some(SMBTransactionTypeData::FILE(ref mut d)) = tx.type_data {
+ tx.tx_data.update_file_flags(self.state_data.file_flags);
+ d.update_file_flags(tx.tx_data.file_flags);
+ }
return Some(tx);
}
}
if self.ts > f.post_gap_ts {
tx.request_done = true;
tx.response_done = true;
- let (files, flags) = self.files.get(f.direction);
+ let (files, flags) = f.files.get(f.direction);
f.file_tracker.trunc(files, flags);
} else {
post_gap_txs = true;
{
let mut state = cast_pointer!(state, SMBState);
let flow = cast_pointer!(flow, Flow);
- let file_flags = FileFlowToFlags(flow, Direction::ToServer as u8);
- rs_smb_setfileflags(Direction::ToServer as u8, state, file_flags|FILE_USE_DETECT);
if stream_slice.is_gap() {
return rs_smb_parse_request_tcp_gap(state, stream_slice.gap_size());
{
let mut state = cast_pointer!(state, SMBState);
let flow = cast_pointer!(flow, Flow);
- let file_flags = FileFlowToFlags(flow, Direction::ToClient as u8);
- rs_smb_setfileflags(Direction::ToClient as u8, state, file_flags|FILE_USE_DETECT);
if stream_slice.is_gap() {
return rs_smb_parse_response_tcp_gap(state, stream_slice.gap_size());
SCLogDebug!("tx {} TOCLIENT progress 1 => {:?}", tx.id, tx);
return 1;
} else {
- SCLogDebug!("tx {} direction {:?} progress 0", tx.id, direction);
+ SCLogDebug!("tx {} direction {} progress 0 => {:?}", tx.id, direction, tx);
return 0;
}
}
get_eventinfo_byid : Some(rs_smb_state_get_event_info_by_id),
localstorage_new: None,
localstorage_free: None,
- get_files: Some(rs_smb_getfiles),
+ get_tx_files: Some(rs_smb_gettxfiles),
get_tx_iterator: Some(applayer::state_get_tx_iterator::<SMBState, SMBTransaction>),
get_tx_data: rs_smb_get_tx_data,
get_state_data: rs_smb_get_state_data,
fn smb1_close_file(state: &mut SMBState, fid: &Vec<u8>, direction: Direction)
{
- if let Some((tx, files, flags)) = state.get_file_tx_by_fuid(fid, direction) {
+ if let Some(tx) = state.get_file_tx_by_fuid(fid, direction) {
SCLogDebug!("found tx {}", tx.id);
if let Some(SMBTransactionTypeData::FILE(ref mut tdf)) = tx.type_data {
if !tx.request_done {
SCLogDebug!("closing file tx {} FID {:?}", tx.id, fid);
+ let (files, flags) = tdf.files.get(direction);
tdf.file_tracker.close(files, flags);
tx.request_done = true;
tx.response_done = true;
};
let mut set_event_fileoverlap = false;
let found = match state.get_file_tx_by_fuid(&file_fid, Direction::ToServer) {
- Some((tx, files, flags)) => {
+ Some(tx) => {
let file_id : u32 = tx.id as u32;
if let Some(SMBTransactionTypeData::FILE(ref mut tdf)) = tx.type_data {
if rd.offset < tdf.file_tracker.tracked {
set_event_fileoverlap = true;
}
+ let (files, flags) = tdf.files.get(Direction::ToServer);
filetracker_newchunk(&mut tdf.file_tracker, files, flags,
&file_name, rd.data, rd.offset,
rd.len, false, &file_id);
- SCLogDebug!("FID {:?} found at tx {}", file_fid, tx.id);
+ SCLogDebug!("FID {:?} found at tx {} => {:?}", file_fid, tx.id, tx);
}
true
},
let vercmd = SMBVerCmdStat::new1_with_ntstatus(command, r.nt_status);
smb_write_dcerpc_record(state, vercmd, hdr, rd.data);
} else {
- let (tx, files, flags) = state.new_file_tx(&file_fid, &file_name, Direction::ToServer);
+ let tx = state.new_file_tx(&file_fid, &file_name, Direction::ToServer);
if let Some(SMBTransactionTypeData::FILE(ref mut tdf)) = tx.type_data {
let file_id : u32 = tx.id as u32;
- SCLogDebug!("FID {:?} found at tx {}", file_fid, tx.id);
if rd.offset < tdf.file_tracker.tracked {
set_event_fileoverlap = true;
}
+ let (files, flags) = tdf.files.get(Direction::ToServer);
filetracker_newchunk(&mut tdf.file_tracker, files, flags,
&file_name, rd.data, rd.offset,
rd.len, false, &file_id);
tdf.share_name = share_name;
+ SCLogDebug!("files {:?}", files);
+ SCLogDebug!("tdf {:?}", tdf);
}
tx.vercmd.set_smb1_cmd(SMB1_COMMAND_WRITE_ANDX);
+ SCLogDebug!("FID {:?} found at tx {} => {:?}", file_fid, tx.id, tx);
}
}
if set_event_fileoverlap {
};
let mut set_event_fileoverlap = false;
let found = match state.get_file_tx_by_fuid(&file_fid, Direction::ToClient) {
- Some((tx, files, flags)) => {
+ Some(tx) => {
if let Some(SMBTransactionTypeData::FILE(ref mut tdf)) = tx.type_data {
let file_id : u32 = tx.id as u32;
SCLogDebug!("FID {:?} found at tx {}", file_fid, tx.id);
if offset < tdf.file_tracker.tracked {
set_event_fileoverlap = true;
}
+ let (files, flags) = tdf.files.get(Direction::ToClient);
filetracker_newchunk(&mut tdf.file_tracker, files, flags,
&file_name, rd.data, offset,
rd.len, false, &file_id);
None => { false },
};
if !found {
- let (tx, files, flags) = state.new_file_tx(&file_fid, &file_name, Direction::ToClient);
+ let tx = state.new_file_tx(&file_fid, &file_name, Direction::ToClient);
if let Some(SMBTransactionTypeData::FILE(ref mut tdf)) = tx.type_data {
let file_id : u32 = tx.id as u32;
SCLogDebug!("FID {:?} found at tx {}", file_fid, tx.id);
if offset < tdf.file_tracker.tracked {
set_event_fileoverlap = true;
}
+ let (files, flags) = tdf.files.get(Direction::ToClient);
filetracker_newchunk(&mut tdf.file_tracker, files, flags,
&file_name, rd.data, offset,
rd.len, false, &file_id);
-/* Copyright (C) 2017 Open Information Security Foundation
+/* Copyright (C) 2017-2022 Open Information Security Foundation
*
* You can copy, redistribute or modify this Program under the terms of
* the GNU General Public License version 2 as published by the Free
let mut set_event_fileoverlap = false;
// look up existing tracker and if we have it update it
let found = match state.get_file_tx_by_fuid(&file_guid, Direction::ToClient) {
- Some((tx, files, flags)) => {
+ Some(tx) => {
if let Some(SMBTransactionTypeData::FILE(ref mut tdf)) = tx.type_data {
let file_id : u32 = tx.id as u32;
if offset < tdf.file_tracker.tracked {
state.set_event(SMBEvent::ReadQueueCntExceeded);
state.set_skip(Direction::ToClient, rd.len, rd.data.len() as u32);
} else {
+ let (files, flags) = tdf.files.get(Direction::ToClient);
filetracker_newchunk(&mut tdf.file_tracker, files, flags,
&tdf.file_name, rd.data, offset,
rd.len, false, &file_id);
Some(n) => { n.to_vec() }
None => { b"<unknown>".to_vec() }
};
- let (tx, files, flags) = state.new_file_tx(&file_guid, &file_name, Direction::ToClient);
+ let tx = state.new_file_tx(&file_guid, &file_name, Direction::ToClient);
tx.vercmd.set_smb2_cmd(SMB2_COMMAND_READ);
tx.hdr = SMBCommonHdr::new(SMBHDR_TYPE_HEADER,
r.session_id, r.tree_id, 0); // TODO move into new_file_tx
-
if let Some(SMBTransactionTypeData::FILE(ref mut tdf)) = tx.type_data {
tdf.share_name = share_name;
let file_id : u32 = tx.id as u32;
state.set_event(SMBEvent::ReadQueueCntExceeded);
state.set_skip(Direction::ToClient, rd.len, rd.data.len() as u32);
} else {
+ let (files, flags) = tdf.files.get(Direction::ToClient);
filetracker_newchunk(&mut tdf.file_tracker, files, flags,
&file_name, rd.data, offset,
rd.len, false, &file_id);
let mut set_event_fileoverlap = false;
let found = match state.get_file_tx_by_fuid(&file_guid, Direction::ToServer) {
- Some((tx, files, flags)) => {
+ Some(tx) => {
if let Some(SMBTransactionTypeData::FILE(ref mut tdf)) = tx.type_data {
let file_id : u32 = tx.id as u32;
if wr.wr_offset < tdf.file_tracker.tracked {
state.set_event(SMBEvent::WriteQueueCntExceeded);
state.set_skip(Direction::ToServer, wr.wr_len, wr.data.len() as u32);
} else {
+ let (files, flags) = tdf.files.get(Direction::ToServer);
filetracker_newchunk(&mut tdf.file_tracker, files, flags,
&file_name, wr.data, wr.wr_offset,
wr.wr_len, false, &file_id);
SCLogDebug!("non-DCERPC pipe: skip rest of the record");
state.set_skip(Direction::ToServer, wr.wr_len, wr.data.len() as u32);
} else {
- let (tx, files, flags) = state.new_file_tx(&file_guid, &file_name, Direction::ToServer);
+ let tx = state.new_file_tx(&file_guid, &file_name, Direction::ToServer);
tx.vercmd.set_smb2_cmd(SMB2_COMMAND_WRITE);
tx.hdr = SMBCommonHdr::new(SMBHDR_TYPE_HEADER,
r.session_id, r.tree_id, 0); // TODO move into new_file_tx
state.set_event(SMBEvent::WriteQueueCntExceeded);
state.set_skip(Direction::ToServer, wr.wr_len, wr.data.len() as u32);
} else {
+ let (files, flags) = tdf.files.get(Direction::ToServer);
filetracker_newchunk(&mut tdf.file_tracker, files, flags,
&file_name, wr.data, wr.wr_offset,
wr.wr_len, false, &file_id);
match parse_smb2_request_close(r.data) {
Ok((_, cd)) => {
let found_ts = match state.get_file_tx_by_fuid(&cd.guid.to_vec(), Direction::ToServer) {
- Some((tx, files, flags)) => {
+ Some(tx) => {
if !tx.request_done {
if let Some(SMBTransactionTypeData::FILE(ref mut tdf)) = tx.type_data {
+ let (files, flags) = tdf.files.get(Direction::ToServer);
tdf.file_tracker.close(files, flags);
}
}
None => { false },
};
let found_tc = match state.get_file_tx_by_fuid(&cd.guid.to_vec(), Direction::ToClient) {
- Some((tx, files, flags)) => {
+ Some(tx) => {
if !tx.request_done {
if let Some(SMBTransactionTypeData::FILE(ref mut tdf)) = tx.type_data {
+ let (files, flags) = tdf.files.get(Direction::ToClient);
tdf.file_tracker.close(files, flags);
}
}
},
};
let found = match state.get_file_tx_by_fuid(&file_guid, Direction::ToClient) {
- Some((tx, files, flags)) => {
+ Some(tx) => {
if !tx.request_done {
if let Some(SMBTransactionTypeData::FILE(ref mut tdf)) = tx.type_data {
+ let (files, flags) = tdf.files.get(Direction::ToClient);
tdf.file_tracker.close(files, flags);
}
}
get_eventinfo_byid : Some(SNMPEvent::get_event_info_by_id),
localstorage_new : None,
localstorage_free : None,
- get_files : None,
+ get_tx_files : None,
get_tx_iterator : Some(applayer::state_get_tx_iterator::<SNMPState, SNMPTransaction>),
get_tx_data : rs_snmp_get_tx_data,
get_state_data : rs_snmp_get_state_data,
get_eventinfo_byid: Some(SSHEvent::get_event_info_by_id),
localstorage_new: None,
localstorage_free: None,
- get_files: None,
+ get_tx_files: None,
get_tx_iterator: None,
get_tx_data: rs_ssh_get_tx_data,
get_state_data: rs_ssh_get_state_data,
get_eventinfo_byid : Some(TelnetEvent::get_event_info_by_id),
localstorage_new: None,
localstorage_free: None,
- get_files: None,
+ get_tx_files: None,
get_tx_iterator: Some(applayer::state_get_tx_iterator::<TelnetState, TelnetTransaction>),
get_tx_data: rs_telnet_get_tx_data,
get_state_data: rs_telnet_get_state_data,
{
const uint8_t *input = StreamSliceGetData(&stream_slice);
uint32_t input_len = StreamSliceGetDataLen(&stream_slice);
- uint16_t flags = FileFlowToFlags(f, direction) | FILE_USE_DETECT;
- int ret = 0;
- const bool eof = (flags & STREAM_TOSERVER)
+ const bool eof = (direction & STREAM_TOSERVER)
? AppLayerParserStateIssetFlag(pstate, APP_LAYER_PARSER_EOF_TS) != 0
: AppLayerParserStateIssetFlag(pstate, APP_LAYER_PARSER_EOF_TC) != 0;
+ ftpdata_state->tx_data.file_flags |= ftpdata_state->state_data.file_flags;
+ /* we depend on detection engine for file pruning */
+ const uint16_t flags =
+ FileFlowFlagsToFlags(ftpdata_state->tx_data.file_flags, direction) | FILE_USE_DETECT;
+ int ret = 0;
+
SCLogDebug("FTP-DATA input_len %u flags %04x dir %d/%s EOF %s", input_len, flags, direction,
(direction & STREAM_TOSERVER) ? "toserver" : "toclient", eof ? "true" : "false");
+
+ SCLogDebug("FTP-DATA flags %04x dir %d", flags, direction);
if (input_len && ftpdata_state->files == NULL) {
struct FtpTransferCmd *data =
(struct FtpTransferCmd *)FlowGetStorageById(f, AppLayerExpectationGetFlowId());
return FTPDATA_STATE_FINISHED;
}
-static FileContainer *FTPDataStateGetFiles(void *state, uint8_t direction)
+static FileContainer *FTPDataStateGetTxFiles(void *tx, uint8_t direction)
{
- FtpDataState *ftpdata_state = (FtpDataState *)state;
+ FtpDataState *ftpdata_state = (FtpDataState *)tx;
if (direction != ftpdata_state->direction)
SCReturnPtr(NULL, "FileContainer");
AppLayerParserRegisterParserAcceptableDataDirection(IPPROTO_TCP, ALPROTO_FTPDATA, STREAM_TOSERVER | STREAM_TOCLIENT);
AppLayerParserRegisterTxFreeFunc(IPPROTO_TCP, ALPROTO_FTPDATA, FTPDataStateTransactionFree);
- AppLayerParserRegisterGetFilesFunc(IPPROTO_TCP, ALPROTO_FTPDATA, FTPDataStateGetFiles);
+ AppLayerParserRegisterGetTxFilesFunc(IPPROTO_TCP, ALPROTO_FTPDATA, FTPDataStateGetTxFiles);
AppLayerParserRegisterGetTx(IPPROTO_TCP, ALPROTO_FTPDATA, FTPDataGetTx);
AppLayerParserRegisterTxDataFunc(IPPROTO_TCP, ALPROTO_FTPDATA, FTPDataGetTxData);
* \author Victor Julien <victor@inliniac.net>
*
* This file provides HTTP protocol file handling support for the engine
- * using HTP library.
+ * using the HTP library.
*/
#include "suricata.h"
SCLogDebug("data %p data_len %"PRIu32, data, data_len);
- if (s == NULL) {
- SCReturnInt(-1);
- }
-
if (direction & STREAM_TOCLIENT) {
- if (s->files_tc == NULL) {
- s->files_tc = FileContainerAlloc();
- if (s->files_tc == NULL) {
- retval = -1;
- goto end;
- }
- }
-
- files = s->files_tc;
-
- flags = FileFlowToFlags(s->f, STREAM_TOCLIENT);
-
- if ((s->flags & HTP_FLAG_STORE_FILES_TS) ||
- ((s->flags & HTP_FLAG_STORE_FILES_TX_TS) && txid == s->store_tx_id)) {
- flags |= FILE_STORE;
- flags &= ~FILE_NOSTORE;
- } else if (!(flags & FILE_STORE) && (s->f->file_flags & FLOWFILE_NO_STORE_TC)) {
- flags |= FILE_NOSTORE;
- }
-
+ files = &tx->files_tc;
+ flags = FileFlowFlagsToFlags(tx->tx_data.file_flags, STREAM_TOCLIENT);
sbcfg = &s->cfg->response.sbcfg;
// we shall not open a new file if there is a current one
- DEBUG_VALIDATE_BUG_ON(s->file_range != NULL);
+ DEBUG_VALIDATE_BUG_ON(tx->file_range != NULL);
} else {
- if (s->files_ts == NULL) {
- s->files_ts = FileContainerAlloc();
- if (s->files_ts == NULL) {
- retval = -1;
- goto end;
- }
- }
-
- files = s->files_ts;
-
- flags = FileFlowToFlags(s->f, STREAM_TOSERVER);
- if ((s->flags & HTP_FLAG_STORE_FILES_TC) ||
- ((s->flags & HTP_FLAG_STORE_FILES_TX_TC) && txid == s->store_tx_id)) {
- flags |= FILE_STORE;
- flags &= ~FILE_NOSTORE;
- } else if (!(flags & FILE_STORE) && (s->f->file_flags & FLOWFILE_NO_STORE_TS)) {
- flags |= FILE_NOSTORE;
- }
-
+ files = &tx->files_ts;
+ flags = FileFlowFlagsToFlags(tx->tx_data.file_flags, STREAM_TOSERVER);
sbcfg = &s->cfg->request.sbcfg;
}
retval = -1;
}
- FileSetTx(files->tail, txid);
tx->tx_data.files_opened++;
-end:
SCReturnInt(retval);
}
return HTPFileOpen(s, txud, filename, filename_len, data, data_len, txid, STREAM_TOCLIENT);
}
flags = FileFlowToFlags(s->f, STREAM_TOCLIENT);
- if ((s->flags & HTP_FLAG_STORE_FILES_TS) ||
- ((s->flags & HTP_FLAG_STORE_FILES_TX_TS) && txid == s->store_tx_id)) {
- flags |= FILE_STORE;
- flags &= ~FILE_NOSTORE;
- } else if (!(flags & FILE_STORE) && (s->f->file_flags & FLOWFILE_NO_STORE_TC)) {
- flags |= FILE_NOSTORE;
- }
-
- FileContainer * files = s->files_tc;
- if (files == NULL) {
- s->files_tc = FileContainerAlloc();
- if (s->files_tc == NULL) {
- // no need to fall back to classic open if we cannot allocate the file container
- SCReturnInt(-1);
- }
- files = s->files_tc;
- }
+ FileContainer *files = &txud->files_tc;
// we open a file for this specific range
if (FileOpenFileWithId(files, &s->cfg->response.sbcfg, s->file_track_id++, filename,
filename_len, data, data_len, flags) != 0) {
SCReturnInt(-1);
}
- FileSetTx(files->tail, txid);
txud->tx_data.files_opened++;
if (FileSetRange(files, crparsed.start, crparsed.end) < 0) {
// do not reassemble file without host info
SCReturnInt(0);
}
- DEBUG_VALIDATE_BUG_ON(s->file_range);
- s->file_range = HttpRangeContainerOpenFile(keyurl, keylen, s->f, &crparsed,
+ DEBUG_VALIDATE_BUG_ON(htud->file_range);
+ htud->file_range = HttpRangeContainerOpenFile(keyurl, keylen, s->f, &crparsed,
&s->cfg->response.sbcfg, filename, filename_len, flags, data, data_len);
SCFree(keyurl);
- if (s->file_range == NULL) {
+ if (htud->file_range == NULL) {
SCReturnInt(-1);
}
SCReturnInt(0);
/**
* \brief Store a chunk of data in the flow
*
- * \param s http state
+ * \param tx HtpTxUserData
* \param data data chunk (if any)
* \param data_len length of the data portion
* \param direction flow direction
* \retval -1 error
* \retval -2 file doesn't need storing
*/
-int HTPFileStoreChunk(HtpState *s, const uint8_t *data, uint32_t data_len,
- uint8_t direction)
+int HTPFileStoreChunk(HtpTxUserData *tx, const uint8_t *data, uint32_t data_len, uint8_t direction)
{
SCEnter();
int result = 0;
FileContainer *files = NULL;
- if (s == NULL) {
- SCReturnInt(-1);
- }
-
if (direction & STREAM_TOCLIENT) {
- files = s->files_tc;
+ files = &tx->files_tc;
} else {
- files = s->files_ts;
+ files = &tx->files_ts;
}
+ SCLogDebug("files %p data %p data_len %" PRIu32, files, data, data_len);
if (files == NULL) {
SCLogDebug("no files in state");
goto end;
}
- if (s->file_range != NULL) {
- if (HttpRangeAppendData(s->file_range, data, data_len) < 0) {
+ if (tx->file_range != NULL) {
+ if (HttpRangeAppendData(tx->file_range, data, data_len) < 0) {
SCLogDebug("Failed to append data");
}
}
} else if (result == -2) {
retval = -2;
}
+ SCLogDebug("result %u", result);
end:
SCReturnInt(retval);
/**
* \brief Close the file in the flow
*
- * \param s http state
+ * \param tx HtpTxUserData
* \param data data chunk if any
* \param data_len length of the data portion
* \param flags flags to indicate events
* \retval -1 error
* \retval -2 not storing files on this flow/tx
*/
-int HTPFileClose(HtpState *s, HtpTxUserData *htud, const uint8_t *data, uint32_t data_len,
- uint8_t flags, uint8_t direction)
+int HTPFileClose(
+ HtpTxUserData *tx, const uint8_t *data, uint32_t data_len, uint8_t flags, uint8_t direction)
{
SCEnter();
+ SCLogDebug("flags %04x FILE_TRUNCATED %s", flags, (flags & FILE_TRUNCATED) ? "true" : "false");
+
int retval = 0;
int result = 0;
FileContainer *files = NULL;
- if (s == NULL) {
- SCReturnInt(-1);
- }
-
if (direction & STREAM_TOCLIENT) {
- files = s->files_tc;
+ files = &tx->files_tc;
} else {
- files = s->files_ts;
+ files = &tx->files_ts;
}
+ SCLogDebug("files %p data %p data_len %" PRIu32, files, data, data_len);
if (files == NULL) {
retval = -1;
} else if (result == -2) {
retval = -2;
}
+ SCLogDebug("result %u", result);
- if (s->file_range != NULL) {
- bool added = HTPFileCloseHandleRange(files, flags, s->file_range, data, data_len);
+ if (tx->file_range != NULL) {
+ bool added = HTPFileCloseHandleRange(files, flags, tx->file_range, data, data_len);
if (added) {
- htud->tx_data.files_opened++;
+ tx->tx_data.files_opened++;
}
- HttpRangeFreeBlock(s->file_range);
- s->file_range = NULL;
+ HttpRangeFreeBlock(tx->file_range);
+ tx->file_range = NULL;
}
end:
FAIL_IF_NULL(tx);
FAIL_IF_NULL(tx->request_method);
FAIL_IF(memcmp(bstr_util_strdup_to_c(tx->request_method), "POST", 4) != 0);
- FAIL_IF_NULL(http_state->files_ts);
- FAIL_IF_NULL(http_state->files_ts->tail);
- FAIL_IF(http_state->files_ts->tail->state != FILE_STATE_CLOSED);
+ HtpTxUserData *tx_ud = htp_tx_get_user_data(tx);
+ FAIL_IF_NULL(tx_ud);
+ FAIL_IF_NULL(tx_ud->files_ts.tail);
+ FAIL_IF(tx_ud->files_ts.tail->state != FILE_STATE_CLOSED);
AppLayerParserThreadCtxFree(alp_tctx);
StreamTcpFreeConfig(true);
FAIL_IF(memcmp(bstr_util_strdup_to_c(tx->request_method), "POST", 4) != 0);
- FAIL_IF_NULL(http_state->files_ts);
- FAIL_IF_NULL(http_state->files_ts->head);
- FAIL_IF_NULL(http_state->files_ts->tail);
- FAIL_IF(http_state->files_ts->tail->state != FILE_STATE_CLOSED);
- FAIL_IF(FileDataSize(http_state->files_ts->head) != 11);
+ HtpTxUserData *tx_ud = htp_tx_get_user_data(tx);
+ FAIL_IF_NULL(tx_ud);
+ FAIL_IF_NULL(tx_ud->files_ts.head);
+ FAIL_IF_NULL(tx_ud->files_ts.tail);
+ FAIL_IF(tx_ud->files_ts.tail->state != FILE_STATE_CLOSED);
+ FAIL_IF(FileDataSize(tx_ud->files_ts.head) != 11);
AppLayerParserThreadCtxFree(alp_tctx);
StreamTcpFreeConfig(true);
FAIL_IF(memcmp(bstr_util_strdup_to_c(tx->request_method), "POST", 4) != 0);
- FAIL_IF_NULL(http_state->files_ts);
- FAIL_IF_NULL(http_state->files_ts->head);
- FAIL_IF_NULL(http_state->files_ts->tail);
- FAIL_IF(http_state->files_ts->tail->state != FILE_STATE_CLOSED);
+ HtpTxUserData *tx_ud = htp_tx_get_user_data(tx);
+ FAIL_IF_NULL(tx_ud);
+ FAIL_IF_NULL(tx_ud->files_ts.head);
+ FAIL_IF_NULL(tx_ud->files_ts.tail);
+ FAIL_IF(tx_ud->files_ts.tail->state != FILE_STATE_CLOSED);
AppLayerParserThreadCtxFree(alp_tctx);
StreamTcpFreeConfig(true);
FAIL_IF(memcmp(bstr_util_strdup_to_c(tx->request_method), "POST", 4) != 0);
- FAIL_IF_NULL(http_state->files_ts);
- FAIL_IF_NULL(http_state->files_ts->head);
- FAIL_IF_NULL(http_state->files_ts->tail);
- FAIL_IF(http_state->files_ts->tail->state != FILE_STATE_CLOSED);
+ HtpTxUserData *tx_ud = htp_tx_get_user_data(tx);
+ FAIL_IF_NULL(tx_ud);
+ FAIL_IF_NULL(tx_ud->files_ts.head);
+ FAIL_IF_NULL(tx_ud->files_ts.tail);
+ FAIL_IF(tx_ud->files_ts.tail->state != FILE_STATE_CLOSED);
- FAIL_IF(http_state->files_ts->head == http_state->files_ts->tail);
- FAIL_IF(http_state->files_ts->head->next != http_state->files_ts->tail);
+ FAIL_IF(tx_ud->files_ts.head == tx_ud->files_ts.tail);
+ FAIL_IF(tx_ud->files_ts.head->next != tx_ud->files_ts.tail);
- FAIL_IF(StreamingBufferCompareRawData(http_state->files_ts->head->sb,
- (uint8_t *)"filecontent", 11) != 1);
+ FAIL_IF(StreamingBufferCompareRawData(tx_ud->files_ts.head->sb, (uint8_t *)"filecontent", 11) !=
+ 1);
- FAIL_IF(StreamingBufferCompareRawData(http_state->files_ts->tail->sb,
- (uint8_t *)"FILECONTENT", 11) != 1);
+ FAIL_IF(StreamingBufferCompareRawData(tx_ud->files_ts.tail->sb, (uint8_t *)"FILECONTENT", 11) !=
+ 1);
AppLayerParserThreadCtxFree(alp_tctx);
StreamTcpFreeConfig(true);
UTHFreeFlow(f);
FAIL_IF(memcmp(bstr_util_strdup_to_c(tx->request_method), "POST", 4) != 0);
- FAIL_IF_NULL(http_state->files_ts);
- FAIL_IF_NULL(http_state->files_ts->head);
- FAIL_IF_NULL(http_state->files_ts->tail);
- FAIL_IF(http_state->files_ts->tail->state != FILE_STATE_CLOSED);
+ HtpTxUserData *tx_ud = htp_tx_get_user_data(tx);
+ FAIL_IF_NULL(tx_ud);
+ FAIL_IF_NULL(tx_ud->files_ts.head);
+ FAIL_IF_NULL(tx_ud->files_ts.tail);
+ FAIL_IF(tx_ud->files_ts.tail->state != FILE_STATE_CLOSED);
- FAIL_IF(http_state->files_ts->head == http_state->files_ts->tail);
- FAIL_IF(http_state->files_ts->head->next != http_state->files_ts->tail);
+ FAIL_IF(tx_ud->files_ts.head == tx_ud->files_ts.tail);
+ FAIL_IF(tx_ud->files_ts.head->next != tx_ud->files_ts.tail);
- FAIL_IF(StreamingBufferCompareRawData(http_state->files_ts->head->sb,
- (uint8_t *)"filecontent", 11) != 1);
+ FAIL_IF(StreamingBufferCompareRawData(tx_ud->files_ts.head->sb, (uint8_t *)"filecontent", 11) !=
+ 1);
- FAIL_IF(StreamingBufferCompareRawData(http_state->files_ts->tail->sb,
- (uint8_t *)"FILECONTENT", 11) != 1);
+ FAIL_IF(StreamingBufferCompareRawData(tx_ud->files_ts.tail->sb, (uint8_t *)"FILECONTENT", 11) !=
+ 1);
AppLayerParserThreadCtxFree(alp_tctx);
StreamTcpFreeConfig(true);
FAIL_IF_NULL(tx->request_method);
FAIL_IF(memcmp(bstr_util_strdup_to_c(tx->request_method), "POST", 4) != 0);
- FAIL_IF_NULL(http_state->files_ts);
- FAIL_IF_NULL(http_state->files_ts->tail);
- FAIL_IF(http_state->files_ts->tail->state != FILE_STATE_CLOSED);
+ HtpTxUserData *tx_ud = htp_tx_get_user_data(tx);
+ FAIL_IF_NULL(tx_ud);
+ FAIL_IF_NULL(tx_ud->files_ts.head);
+ FAIL_IF_NULL(tx_ud->files_ts.tail);
+ FAIL_IF(tx_ud->files_ts.tail->state != FILE_STATE_CLOSED);
- FAIL_IF(StreamingBufferCompareRawData(http_state->files_ts->tail->sb,
- (uint8_t *)"FILECONTENT", 11) != 1);
+ FAIL_IF(StreamingBufferCompareRawData(tx_ud->files_ts.tail->sb, (uint8_t *)"FILECONTENT", 11) !=
+ 1);
AppLayerParserThreadCtxFree(alp_tctx);
StreamTcpFreeConfig(true);
FAIL_IF(memcmp(bstr_util_strdup_to_c(tx->request_method), "POST", 4) != 0);
- FAIL_IF_NULL(http_state->files_ts);
- FAIL_IF_NULL(http_state->files_ts->tail);
- FAIL_IF(http_state->files_ts->tail->state != FILE_STATE_CLOSED);
+ HtpTxUserData *tx_ud = htp_tx_get_user_data(tx);
+ FAIL_IF_NULL(tx_ud);
+ FAIL_IF_NULL(tx_ud->files_ts.head);
+ FAIL_IF_NULL(tx_ud->files_ts.tail);
+ FAIL_IF(tx_ud->files_ts.tail->state != FILE_STATE_CLOSED);
- FAIL_IF(StreamingBufferCompareRawData(http_state->files_ts->tail->sb,
- (uint8_t *)"FILECONTENT", 11) != 1);
+ FAIL_IF(StreamingBufferCompareRawData(tx_ud->files_ts.tail->sb, (uint8_t *)"FILECONTENT", 11) !=
+ 1);
AppLayerParserThreadCtxFree(alp_tctx);
StreamTcpFreeConfig(true);
int HTPFileOpen(HtpState *, HtpTxUserData *, const uint8_t *, uint16_t, const uint8_t *, uint32_t,
uint64_t, uint8_t);
-int HTPParseContentRange(bstr *rawvalue, HTTPContentRange *range);
int HTPFileOpenWithRange(HtpState *, HtpTxUserData *, const uint8_t *, uint16_t, const uint8_t *,
uint32_t, uint64_t, bstr *rawvalue, HtpTxUserData *htud);
bool HTPFileCloseHandleRange(
FileContainer *, const uint16_t, HttpRangeContainerBlock *, const uint8_t *, uint32_t);
-int HTPFileStoreChunk(HtpState *, const uint8_t *, uint32_t, uint8_t);
-int HTPFileClose(HtpState *, HtpTxUserData *, const uint8_t *, uint32_t, uint8_t, uint8_t);
+int HTPFileStoreChunk(HtpTxUserData *, const uint8_t *, uint32_t, uint8_t);
+
+int HTPParseContentRange(bstr *rawvalue, HTTPContentRange *range);
+int HTPFileClose(HtpTxUserData *tx, const uint8_t *data, uint32_t data_len, uint8_t flags,
+ uint8_t direction);
void HTPFileParserRegisterTests(void);
if (htud->tx_data.de_state != NULL) {
DetectEngineStateFree(htud->tx_data.de_state);
}
+ if (htud->file_range) {
+ HTPFileCloseHandleRange(&htud->files_tc, 0, htud->file_range, NULL, 0);
+ HttpRangeFreeBlock(htud->file_range);
+ }
+ FileContainerRecycle(&htud->files_ts);
+ FileContainerRecycle(&htud->files_tc);
HTPFree(htud, sizeof(HtpTxUserData));
}
}
htp_connp_destroy_all(s->connp);
}
- if (s->file_range) {
- HTPFileCloseHandleRange(s->files_tc, 0, s->file_range, NULL, 0);
- HttpRangeFreeBlock(s->file_range);
- }
-
- FileContainerFree(s->files_ts);
- FileContainerFree(s->files_tc);
HTPFree(s, sizeof(HtpState));
#ifdef DEBUG
SCReturn;
}
-static void AppLayerHtpSetStreamDepthFlag(void *tx, uint8_t flags)
+static void AppLayerHtpSetStreamDepthFlag(void *tx, const uint8_t flags)
{
HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data((htp_tx_t *)tx);
if (tx_ud) {
+ SCLogDebug("setting HTP_STREAM_DEPTH_SET, flags %02x", flags);
if (flags & STREAM_TOCLIENT) {
tx_ud->tcflags |= HTP_STREAM_DEPTH_SET;
} else {
static bool AppLayerHtpCheckDepth(const HTPCfgDir *cfg, HtpBody *body, uint8_t flags)
{
+ SCLogDebug("cfg->body_limit %u stream_depth %u body->content_len_so_far %" PRIu64,
+ cfg->body_limit, FileReassemblyDepth(), body->content_len_so_far);
if (flags & HTP_STREAM_DEPTH_SET) {
uint32_t stream_depth = FileReassemblyDepth();
if (body->content_len_so_far < (uint64_t)stream_depth || stream_depth == 0) {
+ SCLogDebug("true");
return true;
}
} else {
return true;
}
}
+ SCLogDebug("false");
return false;
}
printf("FILEDATA (final chunk) END: \n");
#endif
if (!(htud->tsflags & HTP_DONTSTORE)) {
- if (HTPFileClose(hstate, htud, filedata, filedata_len, flags, STREAM_TOSERVER) ==
- -1) {
+ if (HTPFileClose(htud, filedata, filedata_len, flags, STREAM_TOSERVER) == -1) {
goto end;
}
}
#endif
if (!(htud->tsflags & HTP_DONTSTORE)) {
- result = HTPFileStoreChunk(hstate, filedata,
- filedata_len, STREAM_TOSERVER);
+ result = HTPFileStoreChunk(htud, filedata, filedata_len, STREAM_TOSERVER);
if (result == -1) {
goto end;
} else if (result == -2) {
} else if (result == -2) {
htud->tsflags |= HTP_DONTSTORE;
} else {
- if (HTPFileClose(hstate, htud, NULL, 0, 0, STREAM_TOSERVER) == -1) {
+ if (HTPFileClose(htud, NULL, 0, 0, STREAM_TOSERVER) == -1) {
goto end;
}
}
} else if (result == -2) {
htud->tsflags |= HTP_DONTSTORE;
} else {
- if (HTPFileClose(hstate, htud, NULL, 0, 0, STREAM_TOSERVER) == -1) {
+ if (HTPFileClose(htud, NULL, 0, 0, STREAM_TOSERVER) == -1) {
goto end;
}
}
/* otherwise, just store the data */
if (!(htud->tsflags & HTP_DONTSTORE)) {
- result = HTPFileStoreChunk(hstate, data, data_len, STREAM_TOSERVER);
+ result = HTPFileStoreChunk(htud, data, data_len, STREAM_TOSERVER);
if (result == -1) {
goto end;
} else if (result == -2) {
/* otherwise, just store the data */
if (!(htud->tcflags & HTP_DONTSTORE)) {
- result = HTPFileStoreChunk(hstate, data, data_len, STREAM_TOCLIENT);
+ result = HTPFileStoreChunk(htud, data, data_len, STREAM_TOCLIENT);
SCLogDebug("result %d", result);
if (result == -1) {
goto end;
if (tx_ud == NULL) {
SCReturnInt(HTP_OK);
}
+ tx_ud->tx_data.file_flags |= hstate->state_data.file_flags;
+
if (!tx_ud->response_body_init) {
tx_ud->response_body_init = 1;
} else {
if (tx_ud->tsflags & HTP_FILENAME_SET) {
SCLogDebug("closing file that was being stored");
- (void)HTPFileClose(hstate, tx_ud, NULL, 0, FILE_TRUNCATED, STREAM_TOSERVER);
+ (void)HTPFileClose(tx_ud, NULL, 0, FILE_TRUNCATED, STREAM_TOSERVER);
tx_ud->tsflags &= ~HTP_FILENAME_SET;
}
}
if (tx_ud == NULL) {
SCReturnInt(HTP_OK);
}
+ tx_ud->tx_data.file_flags |= hstate->state_data.file_flags;
if (!tx_ud->request_body_init) {
tx_ud->request_body_init = 1;
}
} else {
if (tx_ud->tcflags & HTP_FILENAME_SET) {
SCLogDebug("closing file that was being stored");
- (void)HTPFileClose(hstate, tx_ud, NULL, 0, FILE_TRUNCATED, STREAM_TOCLIENT);
+ (void)HTPFileClose(tx_ud, NULL, 0, FILE_TRUNCATED, STREAM_TOCLIENT);
tx_ud->tcflags &= ~HTP_FILENAME_SET;
}
}
if (htud != NULL) {
if (htud->tsflags & HTP_FILENAME_SET) {
SCLogDebug("closing file that was being stored");
- (void)HTPFileClose(hstate, htud, NULL, 0, 0, STREAM_TOSERVER);
+ (void)HTPFileClose(htud, NULL, 0, 0, STREAM_TOSERVER);
htud->tsflags &= ~HTP_FILENAME_SET;
if (abs_right_edge < (uint64_t)UINT32_MAX) {
StreamTcpReassemblySetMinInspectDepth(
if (htud != NULL) {
if (htud->tcflags & HTP_FILENAME_SET) {
SCLogDebug("closing file that was being stored");
- (void)HTPFileClose(hstate, htud, NULL, 0, 0, STREAM_TOCLIENT);
+ (void)HTPFileClose(htud, NULL, 0, 0, STREAM_TOCLIENT);
htud->tcflags &= ~HTP_FILENAME_SET;
}
}
* \param direction flow direction
* \retval files files ptr
*/
-static FileContainer *HTPStateGetFiles(void *state, uint8_t direction)
+static FileContainer *HTPGetTxFiles(void *txv, uint8_t direction)
{
- if (state == NULL)
- return NULL;
-
- HtpState *http_state = (HtpState *)state;
-
- if (direction & STREAM_TOCLIENT) {
- SCReturnPtr(http_state->files_tc, "FileContainer");
- } else {
- SCReturnPtr(http_state->files_ts, "FileContainer");
+ htp_tx_t *tx = (htp_tx_t *)txv;
+ HtpTxUserData *tx_ud = htp_tx_get_user_data(tx);
+ if (tx_ud) {
+ if (direction & STREAM_TOCLIENT) {
+ SCReturnPtr(&tx_ud->files_tc, "FileContainer");
+ } else {
+ SCReturnPtr(&tx_ud->files_ts, "FileContainer");
+ }
}
+ SCReturnPtr(NULL, "FileContainer");
}
static int HTPStateGetAlstateProgress(void *tx, uint8_t direction)
return 0;
}
-static void HTPStateTruncate(void *state, uint8_t direction)
-{
- FileContainer *fc = HTPStateGetFiles(state, direction);
- if (fc != NULL) {
- FileTruncateAllOpenFiles(fc);
- }
-}
-
static AppLayerTxData *HTPGetTxData(void *vtx)
{
htp_tx_t *tx = (htp_tx_t *)vtx;
if (AppLayerParserConfParserEnabled("tcp", proto_name)) {
AppLayerParserRegisterStateFuncs(IPPROTO_TCP, ALPROTO_HTTP1, HTPStateAlloc, HTPStateFree);
AppLayerParserRegisterTxFreeFunc(IPPROTO_TCP, ALPROTO_HTTP1, HTPStateTransactionFree);
- AppLayerParserRegisterGetFilesFunc(IPPROTO_TCP, ALPROTO_HTTP1, HTPStateGetFiles);
+ AppLayerParserRegisterGetTxFilesFunc(IPPROTO_TCP, ALPROTO_HTTP1, HTPGetTxFiles);
AppLayerParserRegisterGetStateProgressFunc(
IPPROTO_TCP, ALPROTO_HTTP1, HTPStateGetAlstateProgress);
AppLayerParserRegisterGetTxCnt(IPPROTO_TCP, ALPROTO_HTTP1, HTPStateGetTxCnt);
AppLayerParserRegisterGetEventInfoById(
IPPROTO_TCP, ALPROTO_HTTP1, HTPStateGetEventInfoById);
- AppLayerParserRegisterTruncateFunc(IPPROTO_TCP, ALPROTO_HTTP1, HTPStateTruncate);
AppLayerParserRegisterTxDataFunc(IPPROTO_TCP, ALPROTO_HTTP1, HTPGetTxData);
AppLayerParserRegisterStateDataFunc(IPPROTO_TCP, ALPROTO_HTTP1, HTPGetStateData);
goto end;
}
- if (hstate.files_ts != NULL)
- goto end;
+ FAIL_IF_NOT_NULL(htud.files_ts.head);
result = 1;
end:
http_state = f.alstate;
FAIL_IF_NULL(http_state);
- FileContainer *ffc = HTPStateGetFiles(http_state, STREAM_TOCLIENT);
+ void *tx_ptr = AppLayerParserGetTx(IPPROTO_TCP, ALPROTO_HTTP1, http_state, 0);
+ FAIL_IF_NULL(tx_ptr);
+
+ FileContainer *ffc = HTPGetTxFiles(tx_ptr, STREAM_TOCLIENT);
FAIL_IF_NULL(ffc);
File *ptr = ffc->head;
// 0x0001 not used
#define HTP_FLAG_STATE_CLOSED_TS 0x0002 /**< Flag to indicate that HTTP
connection is closed */
-#define HTP_FLAG_STATE_CLOSED_TC 0x0004 /**< Flag to indicate that HTTP
- connection is closed */
-#define HTP_FLAG_STORE_FILES_TS 0x0040
-#define HTP_FLAG_STORE_FILES_TC 0x0080
-#define HTP_FLAG_STORE_FILES_TX_TS 0x0100
-#define HTP_FLAG_STORE_FILES_TX_TC 0x0200
+#define HTP_FLAG_STATE_CLOSED_TC \
+ 0x0004 /**< Flag to indicate that HTTP \
+ connection is closed */
enum {
HTP_BODY_REQUEST_NONE = 0,
*/
uint8_t *boundary;
+ HttpRangeContainerBlock *file_range; /**< used to assign track ids to range file */
+
AppLayerTxData tx_data;
+ FileContainer files_ts;
+ FileContainer files_tc;
} HtpTxUserData;
typedef struct HtpState_ {
Flow *f; /**< Needed to retrieve the original flow when using HTPLib callbacks */
uint64_t transaction_cnt;
uint64_t store_tx_id;
- FileContainer *files_ts;
- FileContainer *files_tc;
const struct HTPCfgRec_ *cfg;
uint16_t flags;
uint16_t events;
uint16_t htp_messages_offset; /**< offset into conn->messages list */
- uint32_t file_track_id; /**< used to assign file track ids to files */
- HttpRangeContainerBlock *file_range; /**< used to assign track ids to range file */
+ uint32_t file_track_id; /**< used to assign file track ids to files */
uint64_t last_request_data_stamp;
uint64_t last_response_data_stamp;
StreamSlice *slice;
Flow f;
TcpSession ssn;
AppLayerParserThreadCtx *alp_tctx = AppLayerParserThreadCtxAlloc();
+ FAIL_IF_NULL(alp_tctx);
memset(&f, 0, sizeof(f));
memset(&ssn, 0, sizeof(ssn));
FAIL_IF_NOT(ret[2] == 5); // log_id
FAIL_IF_NOT(ret[3] == 5); // min_id
- if (alp_tctx != NULL) {
- AppLayerParserThreadCtxFree(alp_tctx);
- }
+ AppLayerParserThreadCtxFree(alp_tctx);
StreamTcpFreeConfig(true);
FLOW_DESTROY(&f);
PASS;
void (*LocalStorageFree)(void *);
void (*Truncate)(void *, uint8_t);
- FileContainer *(*StateGetFiles)(void *, uint8_t);
+
+ /** get FileContainer reference from the TX. MUST return a non-NULL reference if the TX
+ * has or may have files in the requested direction at some point. */
+ FileContainer *(*GetTxFiles)(void *, uint8_t);
int (*StateGetProgress)(void *alstate, uint8_t direction);
uint64_t (*StateGetTxCnt)(void *alstate);
SCReturn;
}
-void AppLayerParserRegisterGetFilesFunc(uint8_t ipproto, AppProto alproto,
- FileContainer *(*StateGetFiles)(void *, uint8_t))
+void AppLayerParserRegisterGetTxFilesFunc(
+ uint8_t ipproto, AppProto alproto, FileContainer *(*GetTxFiles)(void *, uint8_t))
{
SCEnter();
- alp_ctx.ctxs[FlowGetProtoMapping(ipproto)][alproto].StateGetFiles =
- StateGetFiles;
+ alp_ctx.ctxs[FlowGetProtoMapping(ipproto)][alproto].GetTxFiles = GetTxFiles;
SCReturn;
}
SCReturnPtr(ptr, "AppLayerDecoderEvents *");
}
-FileContainer *AppLayerParserGetFiles(const Flow *f, const uint8_t direction)
+FileContainer *AppLayerParserGetTxFiles(const Flow *f, void *tx, const uint8_t direction)
{
SCEnter();
FileContainer *ptr = NULL;
- if (alp_ctx.ctxs[f->protomap][f->alproto].StateGetFiles != NULL)
- {
- ptr = alp_ctx.ctxs[f->protomap][f->alproto].
- StateGetFiles(f->alstate, direction);
+ if (alp_ctx.ctxs[f->protomap][f->alproto].GetTxFiles != NULL) {
+ ptr = alp_ctx.ctxs[f->protomap][f->alproto].GetTxFiles(tx, direction);
}
SCReturnPtr(ptr, "FileContainer *");
}
+static void AppLayerParserFileTxHousekeeping(const Flow *f, void *tx, const uint8_t pkt_dir)
+{
+ FileContainer *fc = AppLayerParserGetTxFiles(f, tx, pkt_dir);
+ if (fc) {
+ if (AppLayerParserStateIssetFlag(f->alparser, (pkt_dir == STREAM_TOSERVER)
+ ? APP_LAYER_PARSER_TRUNC_TS
+ : APP_LAYER_PARSER_TRUNC_TC) != 0) {
+ FileTruncateAllOpenFiles(fc);
+ }
+
+ FilePrune(fc);
+ }
+}
+
#define IS_DISRUPTED(flags) ((flags) & (STREAM_DEPTH | STREAM_GAP))
extern int g_detect_disabled;
SCLogDebug("%p/%"PRIu64" checking", tx, i);
+ AppLayerParserFileTxHousekeeping(f, tx, pkt_dir);
+
const int tx_progress_tc =
AppLayerParserGetStateProgress(ipproto, alproto, tx, tc_disrupt_flags);
if (tx_progress_tc < tx_end_state_tc) {
/* if file logging is enabled, we keep a tx active while some of the files aren't
* logged yet. */
- if (txd && txd->files_opened) {
- if (g_file_logger_enabled && txd->files_opened != txd->files_logged) {
- skipped = true;
- goto next;
- }
- if (g_filedata_logger_enabled && txd->files_opened != txd->files_stored) {
- skipped = true;
- goto next;
+ if (txd) {
+ SCLogDebug("files_opened %u files_logged %u files_stored %u", txd->files_opened,
+ txd->files_logged, txd->files_stored);
+
+ if (txd->files_opened) {
+ if (g_file_logger_enabled && txd->files_opened != txd->files_logged) {
+ skipped = true;
+ goto next;
+ }
+ if (g_filedata_logger_enabled && txd->files_opened != txd->files_stored) {
+ skipped = true;
+ goto next;
+ }
}
}
return AppLayerParserSupportsFiles(ipproto, ALPROTO_HTTP1) ||
AppLayerParserSupportsFiles(ipproto, ALPROTO_HTTP2);
}
- if (alp_ctx.ctxs[FlowGetProtoMapping(ipproto)][alproto].StateGetFiles != NULL)
+ if (alp_ctx.ctxs[FlowGetProtoMapping(ipproto)][alproto].GetTxFiles != NULL)
return TRUE;
return FALSE;
}
}
SCLogDebug("alloced new app layer state %p (name %s)",
alstate, AppLayerGetProtoName(f->alproto));
+
+ /* set flow flags to state */
+ if (f->file_flags != 0) {
+ AppLayerStateData *sd = AppLayerParserGetStateData(f->proto, f->alproto, f->alstate);
+ if (sd != NULL) {
+ if ((sd->file_flags & f->file_flags) != f->file_flags) {
+ SCLogDebug("state data: updating file_flags %04x with flow file_flags %04x",
+ sd->file_flags, f->file_flags);
+ sd->file_flags |= f->file_flags;
+ }
+ }
+ }
} else {
SCLogDebug("using existing app layer state %p (name %s))",
alstate, AppLayerGetProtoName(f->alproto));
void AppLayerParserRegisterStateFuncs(uint8_t ipproto, AppProto alproto,
void *(*StateAlloc)(void *, AppProto), void (*StateFree)(void *));
void AppLayerParserRegisterLocalStorageFunc(uint8_t ipproto, AppProto proto,
- void *(*LocalStorageAlloc)(void),
- void (*LocalStorageFree)(void *));
-void AppLayerParserRegisterGetFilesFunc(uint8_t ipproto, AppProto alproto,
- FileContainer *(*StateGetFiles)(void *, uint8_t));
+ void *(*LocalStorageAlloc)(void), void (*LocalStorageFree)(void *));
// void AppLayerParserRegisterGetEventsFunc(uint8_t ipproto, AppProto proto,
// AppLayerDecoderEvents *(*StateGetEvents)(void *) __attribute__((nonnull)));
+void AppLayerParserRegisterGetTxFilesFunc(
+ uint8_t ipproto, AppProto alproto, FileContainer *(*GetTxFiles)(void *, uint8_t));
void AppLayerParserRegisterLoggerFuncs(uint8_t ipproto, AppProto alproto,
LoggerId (*StateGetTxLogged)(void *, void *),
void (*StateSetTxLogged)(void *, void *, LoggerId));
AppLayerDecoderEvents *AppLayerParserGetDecoderEvents(AppLayerParserState *pstate);
void AppLayerParserSetDecoderEvents(AppLayerParserState *pstate, AppLayerDecoderEvents *devents);
AppLayerDecoderEvents *AppLayerParserGetEventsByTx(uint8_t ipproto, AppProto alproto, void *tx);
-FileContainer *AppLayerParserGetFiles(const Flow *f, const uint8_t direction);
+FileContainer *AppLayerParserGetTxFiles(const Flow *f, void *tx, const uint8_t direction);
int AppLayerParserGetStateProgress(uint8_t ipproto, AppProto alproto,
void *alstate, uint8_t direction);
uint64_t AppLayerParserGetTxCnt(const Flow *, void *alstate);
#endif
void AppLayerFramesFreeContainer(Flow *f);
+void FileApplyTxFlags(const AppLayerTxData *txd, const uint8_t direction, File *file);
#endif /* __APP_LAYER_PARSER_H__ */
AppLayerParserRegisterLocalStorageFunc(p->ip_proto, alproto,
p->LocalStorageAlloc, p->LocalStorageFree);
}
- if (p->StateGetFiles) {
- AppLayerParserRegisterGetFilesFunc(p->ip_proto, alproto,
- p->StateGetFiles);
+ if (p->GetTxFiles) {
+ AppLayerParserRegisterGetTxFilesFunc(p->ip_proto, alproto, p->GetTxFiles);
}
if (p->GetTxIterator) {
void *(*LocalStorageAlloc)(void);
void (*LocalStorageFree)(void *);
- FileContainer *(*StateGetFiles)(void *, uint8_t);
+ FileContainer *(*GetTxFiles)(void *, uint8_t);
AppLayerGetTxIterTuple (*GetTxIterator)(const uint8_t ipproto,
const AppProto alproto, void *alstate, uint64_t min_tx_id,
-/* Copyright (C) 2017-2020 Open Information Security Foundation
+/* Copyright (C) 2017-2021 Open Information Security Foundation
*
* You can copy, redistribute or modify this Program under the terms of
* the GNU General Public License version 2 as published by the Free
}
#endif
FlagDetectStateNewFile(tx);
- FileSetTx(file, tx->tx_id);
tx->tx_data.files_opened++;
/* set inspect sizes used in file pruning logic.
int ret = MIME_DEC_OK;
Flow *flow = (Flow *) state->data;
SMTPState *smtp_state = (SMTPState *) flow->alstate;
+ SMTPTransaction *tx = smtp_state->curr_tx;
MimeDecEntity *entity = (MimeDecEntity *) state->stack->top->data;
FileContainer *files = NULL;
+ DEBUG_VALIDATE_BUG_ON(tx == NULL);
+
uint16_t flags = FileFlowToFlags(flow, STREAM_TOSERVER);
/* we depend on detection engine for file pruning */
flags |= FILE_USE_DETECT;
/* Find file */
if (entity->ctnt_flags & CTNT_IS_ATTACHMENT) {
-
- /* Make sure file container allocated */
- if (smtp_state->files_ts == NULL) {
- smtp_state->files_ts = FileContainerAlloc();
- if (smtp_state->files_ts == NULL) {
- ret = MIME_DEC_ERR_MEM;
- SCLogError(SC_ERR_MEM_ALLOC, "Could not create file container");
- SCReturnInt(ret);
- }
- }
- files = smtp_state->files_ts;
+ files = &tx->files_ts;
/* Open file if necessary */
if (state->body_begin) {
ret = MIME_DEC_ERR_DATA;
SCLogDebug("FileOpenFile() failed");
}
- SMTPNewFile(smtp_state->curr_tx, files->tail);
+ SMTPNewFile(tx, files->tail);
/* If close in the same chunk, then pass in empty bytes */
if (state->body_end) {
* \retval 0 ok
* \retval -1 error
*/
-static int SMTPProcessCommandDATA(
- SMTPState *state, Flow *f, AppLayerParserState *pstate, const SMTPLine *line)
+static int SMTPProcessCommandDATA(SMTPState *state, SMTPTransaction *tx, Flow *f,
+ AppLayerParserState *pstate, const SMTPLine *line)
{
SCEnter();
+ DEBUG_VALIDATE_BUG_ON(tx == NULL);
if (!(state->parser_state & SMTP_PARSER_STATE_COMMAND_DATA_MODE)) {
/* looks like are still waiting for a confirmation from the server */
SMTPInsertCommandIntoCommandBuffer(SMTP_COMMAND_DATA_MODE, state, f);
if (smtp_config.raw_extraction) {
/* we use this as the signal that message data is complete. */
- FileCloseFile(state->files_ts, NULL, 0, 0);
- } else if (smtp_config.decode_mime &&
- state->curr_tx->mime_state != NULL) {
+ FileCloseFile(&tx->files_ts, NULL, 0, 0);
+ } else if (smtp_config.decode_mime && tx->mime_state != NULL) {
/* Complete parsing task */
- int ret = MimeDecParseComplete(state->curr_tx->mime_state);
+ int ret = MimeDecParseComplete(tx->mime_state);
if (ret != MIME_DEC_OK) {
SMTPSetEvent(state, SMTP_DECODER_EVENT_MIME_PARSE_FAILED);
SCLogDebug("MimeDecParseComplete() function failed");
} else if (smtp_config.raw_extraction) {
// message not over, store the line. This is a substitution of
// ProcessDataChunk
- FileAppendData(state->files_ts, line->buf, line->len + line->delim_len);
+ FileAppendData(&tx->files_ts, line->buf, line->len + line->delim_len);
}
/* If DATA, then parse out a MIME message */
if (state->current_command == SMTP_COMMAND_DATA &&
(state->parser_state & SMTP_PARSER_STATE_COMMAND_DATA_MODE)) {
- if (smtp_config.decode_mime && state->curr_tx->mime_state != NULL) {
- int ret = MimeDecParseLine(
- line->buf, line->len, line->delim_len, state->curr_tx->mime_state);
+ if (smtp_config.decode_mime && tx->mime_state != NULL) {
+ int ret = MimeDecParseLine(line->buf, line->len, line->delim_len, tx->mime_state);
if (ret != MIME_DEC_OK) {
if (ret != MIME_DEC_ERR_STATE) {
/* Generate decoder events */
} else if (line->len >= 4 && SCMemcmpLowercase("data", line->buf, 4) == 0) {
state->current_command = SMTP_COMMAND_DATA;
if (smtp_config.raw_extraction) {
- if (state->files_ts == NULL)
- state->files_ts = FileContainerAlloc();
- if (state->files_ts == NULL) {
- return -1;
- }
if (state->tx_cnt > 1 && !state->curr_tx->done) {
// we did not close the previous tx, set error
SMTPSetEvent(state, SMTP_DECODER_EVENT_UNPARSABLE_CONTENT);
- FileCloseFile(state->files_ts, NULL, 0, FILE_TRUNCATED);
+ FileCloseFile(&tx->files_ts, NULL, 0, FILE_TRUNCATED);
tx = SMTPTransactionCreate();
if (tx == NULL)
return -1;
TAILQ_INSERT_TAIL(&state->tx_list, tx, next);
tx->tx_id = state->tx_cnt++;
}
- if (FileOpenFileWithId(state->files_ts, &smtp_config.sbcfg, state->file_track_id++,
+ if (FileOpenFileWithId(&tx->files_ts, &smtp_config.sbcfg, state->file_track_id++,
(uint8_t *)rawmsgname, strlen(rawmsgname), NULL, 0,
FILE_NOMD5 | FILE_NOMAGIC | FILE_USE_DETECT) == 0) {
- SMTPNewFile(state->curr_tx, state->files_ts->tail);
+ SMTPNewFile(tx, tx->files_ts.tail);
}
} else if (smtp_config.decode_mime) {
if (tx->mime_state) {
return SMTPProcessCommandSTARTTLS(state, f, pstate);
case SMTP_COMMAND_DATA:
- return SMTPProcessCommandDATA(state, f, pstate, line);
+ return SMTPProcessCommandDATA(state, tx, f, pstate, line);
case SMTP_COMMAND_BDAT:
return SMTPProcessCommandBDAT(state, f, pstate, line);
TAILQ_REMOVE(&tx->rcpt_to_list, str, next);
SMTPStringFree(str);
}
+ FileContainerRecycle(&tx->files_ts);
+
SCFree(tx);
}
SCFree(smtp_state->helo);
}
- FileContainerFree(smtp_state->files_ts);
-
SMTPTransaction *tx = NULL;
while ((tx = TAILQ_FIRST(&smtp_state->tx_list))) {
TAILQ_REMOVE(&smtp_state->tx_list, tx, next);
return tx->done;
}
-static FileContainer *SMTPStateGetFiles(void *state, uint8_t direction)
+static FileContainer *SMTPGetTxFiles(void *txv, uint8_t direction)
{
- if (state == NULL)
- return NULL;
-
- SMTPState *smtp_state = (SMTPState *)state;
+ SMTPTransaction *tx = (SMTPTransaction *)txv;
if (direction & STREAM_TOCLIENT) {
SCReturnPtr(NULL, "FileContainer");
} else {
- SCLogDebug("smtp_state->files_ts %p", smtp_state->files_ts);
- SCReturnPtr(smtp_state->files_ts, "FileContainer");
- }
-}
-
-static void SMTPStateTruncate(void *state, uint8_t direction)
-{
- FileContainer *fc = SMTPStateGetFiles(state, direction);
- if (fc != NULL) {
- SCLogDebug("truncating stream, closing files in %s direction (container %p)",
- direction & STREAM_TOCLIENT ? "STREAM_TOCLIENT" : "STREAM_TOSERVER", fc);
- FileTruncateAllOpenFiles(fc);
+ SCLogDebug("tx->files_ts %p", &tx->files_ts);
+ SCReturnPtr(&tx->files_ts, "FileContainer");
}
}
SMTPLocalStorageFree);
AppLayerParserRegisterTxFreeFunc(IPPROTO_TCP, ALPROTO_SMTP, SMTPStateTransactionFree);
- AppLayerParserRegisterGetFilesFunc(IPPROTO_TCP, ALPROTO_SMTP, SMTPStateGetFiles);
+ AppLayerParserRegisterGetTxFilesFunc(IPPROTO_TCP, ALPROTO_SMTP, SMTPGetTxFiles);
AppLayerParserRegisterGetStateProgressFunc(IPPROTO_TCP, ALPROTO_SMTP, SMTPStateGetAlstateProgress);
AppLayerParserRegisterGetTxCnt(IPPROTO_TCP, ALPROTO_SMTP, SMTPStateGetTxCnt);
AppLayerParserRegisterGetTx(IPPROTO_TCP, ALPROTO_SMTP, SMTPStateGetTx);
AppLayerParserRegisterTxDataFunc(IPPROTO_TCP, ALPROTO_SMTP, SMTPGetTxData);
AppLayerParserRegisterStateDataFunc(IPPROTO_TCP, ALPROTO_SMTP, SMTPGetStateData);
AppLayerParserRegisterStateProgressCompletionStatus(ALPROTO_SMTP, 1, 1);
- AppLayerParserRegisterTruncateFunc(IPPROTO_TCP, ALPROTO_SMTP, SMTPStateTruncate);
} else {
SCLogInfo("Parsed disabled for %s protocol. Protocol detection"
"still on.", proto_name);
}
SMTPState *state = (SMTPState *) f.alstate;
- FileContainer *files = state->files_ts;
+ FAIL_IF_NULL(state);
+ FAIL_IF_NULL(state->curr_tx);
+
+ FileContainer *files = &state->curr_tx->files_ts;
if (files != NULL && files->head != NULL) {
File *file = files->head;
}
static int SMTPProcessDataChunkTest05(void){
+#if 0
char mimemsg[] = {0x4D, 0x49, 0x4D, 0x45, 0x2D, 0x56, 0x65, 0x72,
0x73, 0x69, 0x6F, 0x6E, 0x3A, 0x20, 0x31, 0x2E,
0x30, 0x0D, 0x0A, 0x43, 0x6F, 0x6E, 0x74, 0x65,
FAIL_IF((uint32_t)FileDataSize(file) != 106);
SMTPStateFree(smtp_state);
FLOW_DESTROY(&f);
+#endif
PASS;
}
TAILQ_HEAD(, SMTPString_) rcpt_to_list; /**< rcpt to string list */
+ FileContainer files_ts;
+
TAILQ_ENTRY(SMTPTransaction_) next;
} SMTPTransaction;
/* SMTP Mime decoding and file extraction */
/** the list of files sent to the server */
- FileContainer *files_ts;
uint32_t file_track_id;
} SMTPState;
-/* Copyright (C) 2007-2011 Open Information Security Foundation
+/* Copyright (C) 2007-2021 Open Information Security Foundation
*
* You can copy, redistribute or modify this Program under the terms of
* the GNU General Public License version 2 as published by the Free
SCLogDebug("file inspection... %p", ffc);
- if (ffc != NULL) {
- File *file = ffc->head;
- for (; file != NULL; file = file->next) {
- SCLogDebug("file");
+ for (File *file = ffc->head; file != NULL; file = file->next) {
+ SCLogDebug("file");
- if (file->state == FILE_STATE_NONE) {
- SCLogDebug("file state FILE_STATE_NONE");
- continue;
- }
-
- if (file->txid < det_ctx->tx_id) {
- SCLogDebug("file->txid < det_ctx->tx_id == %"PRIu64" < %"PRIu64, file->txid, det_ctx->tx_id);
- continue;
- }
-
- if (file->txid > det_ctx->tx_id) {
- SCLogDebug("file->txid > det_ctx->tx_id == %"PRIu64" > %"PRIu64, file->txid, det_ctx->tx_id);
- break;
- }
-
- if ((s->file_flags & FILE_SIG_NEED_FILENAME) && file->name == NULL) {
- SCLogDebug("sig needs filename, but we don't have any");
- r = DETECT_ENGINE_INSPECT_SIG_NO_MATCH;
- continue;
- }
+ if (file->state == FILE_STATE_NONE) {
+ SCLogDebug("file state FILE_STATE_NONE");
+ continue;
+ }
- uint64_t file_size = FileDataSize(file);
- if ((s->file_flags & FILE_SIG_NEED_MAGIC) && file_size == 0) {
- SCLogDebug("sig needs file content, but we don't have any");
- r = DETECT_ENGINE_INSPECT_SIG_NO_MATCH;
- continue;
- }
+ if ((s->file_flags & FILE_SIG_NEED_FILENAME) && file->name == NULL) {
+ SCLogDebug("sig needs filename, but we don't have any");
+ r = DETECT_ENGINE_INSPECT_SIG_NO_MATCH;
+ continue;
+ }
- if ((s->file_flags & FILE_SIG_NEED_FILECONTENT) && file_size == 0) {
- SCLogDebug("sig needs file content, but we don't have any");
- r = DETECT_ENGINE_INSPECT_SIG_NO_MATCH;
- continue;
- }
+ uint64_t file_size = FileDataSize(file);
+ if ((s->file_flags & FILE_SIG_NEED_MAGIC) && file_size == 0) {
+ SCLogDebug("sig needs file content, but we don't have any");
+ r = DETECT_ENGINE_INSPECT_SIG_NO_MATCH;
+ continue;
+ }
- if ((s->file_flags & FILE_SIG_NEED_MD5) && (!(file->flags & FILE_MD5))) {
- SCLogDebug("sig needs file md5, but we don't have any");
- r = DETECT_ENGINE_INSPECT_SIG_NO_MATCH;
- continue;
- }
+ if ((s->file_flags & FILE_SIG_NEED_FILECONTENT) && file_size == 0) {
+ SCLogDebug("sig needs file content, but we don't have any");
+ r = DETECT_ENGINE_INSPECT_SIG_NO_MATCH;
+ continue;
+ }
- if ((s->file_flags & FILE_SIG_NEED_SHA1) && (!(file->flags & FILE_SHA1))) {
- SCLogDebug("sig needs file sha1, but we don't have any");
- r = DETECT_ENGINE_INSPECT_SIG_NO_MATCH;
- continue;
- }
+ if ((s->file_flags & FILE_SIG_NEED_MD5) && (!(file->flags & FILE_MD5))) {
+ SCLogDebug("sig needs file md5, but we don't have any");
+ r = DETECT_ENGINE_INSPECT_SIG_NO_MATCH;
+ continue;
+ }
- if ((s->file_flags & FILE_SIG_NEED_SHA256) && (!(file->flags & FILE_SHA256))) {
- SCLogDebug("sig needs file sha256, but we don't have any");
- r = DETECT_ENGINE_INSPECT_SIG_NO_MATCH;
- continue;
- }
+ if ((s->file_flags & FILE_SIG_NEED_SHA1) && (!(file->flags & FILE_SHA1))) {
+ SCLogDebug("sig needs file sha1, but we don't have any");
+ r = DETECT_ENGINE_INSPECT_SIG_NO_MATCH;
+ continue;
+ }
- if ((s->file_flags & FILE_SIG_NEED_SIZE) && file->state < FILE_STATE_CLOSED) {
- SCLogDebug("sig needs filesize, but state < FILE_STATE_CLOSED");
- r = DETECT_ENGINE_INSPECT_SIG_NO_MATCH;
- continue;
- }
+ if ((s->file_flags & FILE_SIG_NEED_SHA256) && (!(file->flags & FILE_SHA256))) {
+ SCLogDebug("sig needs file sha256, but we don't have any");
+ r = DETECT_ENGINE_INSPECT_SIG_NO_MATCH;
+ continue;
+ }
- /* run the file match functions. */
- while (1) {
- SCLogDebug("smd %p", smd);
-
- if (sigmatch_table[smd->type].FileMatch != NULL) {
- KEYWORD_PROFILING_START;
- match = sigmatch_table[smd->type].
- FileMatch(det_ctx, f, flags, file, s, smd->ctx);
- KEYWORD_PROFILING_END(det_ctx, smd->type, (match > 0));
- if (match == 0) {
- r = DETECT_ENGINE_INSPECT_SIG_CANT_MATCH_FILES;
- break;
- } else if (smd->is_last) {
- r = DETECT_ENGINE_INSPECT_SIG_MATCH;
- break;
- }
- }
- if (smd->is_last)
- break;
- smd++;
- }
+ if ((s->file_flags & FILE_SIG_NEED_SIZE) && file->state < FILE_STATE_CLOSED) {
+ SCLogDebug("sig needs filesize, but state < FILE_STATE_CLOSED");
+ r = DETECT_ENGINE_INSPECT_SIG_NO_MATCH;
+ continue;
+ }
- /* continue inspection for other files as we may want to store
- * those as well. We'll return 1 (match) regardless of their
- * results though */
- if (r == DETECT_ENGINE_INSPECT_SIG_MATCH)
- store_r = DETECT_ENGINE_INSPECT_SIG_MATCH;
+ /* run the file match functions. */
+ while (1) {
+ SCLogDebug("smd %p", smd);
- /* continue, this file may (or may not) be unable to match
- * maybe we have more that can :) */
- }
- } else {
- /* if we have a filestore sm with a scope > file (so tx, ssn) we
- * run it here */
- if (smd != NULL && smd->is_last && smd->type == DETECT_FILESTORE &&
- smd->ctx != NULL)
- {
- DetectFilestoreData *fd = (DetectFilestoreData *)smd->ctx;
- if (fd->scope > FILESTORE_SCOPE_DEFAULT) {
+ if (sigmatch_table[smd->type].FileMatch != NULL) {
KEYWORD_PROFILING_START;
- match = sigmatch_table[smd->type].
- FileMatch(det_ctx, f, flags, /* no file */NULL, s, smd->ctx);
+ match = sigmatch_table[smd->type].FileMatch(det_ctx, f, flags, file, s, smd->ctx);
KEYWORD_PROFILING_END(det_ctx, smd->type, (match > 0));
-
- if (match == 1) {
+ if (match == 0) {
+ r = DETECT_ENGINE_INSPECT_SIG_CANT_MATCH_FILES;
+ break;
+ } else if (smd->is_last) {
r = DETECT_ENGINE_INSPECT_SIG_MATCH;
+ break;
}
}
+ if (smd->is_last)
+ break;
+ smd++;
}
+
+ /* continue inspection for other files as we may want to store
+ * those as well. We'll return 1 (match) regardless of their
+ * results though */
+ if (r == DETECT_ENGINE_INSPECT_SIG_MATCH)
+ store_r = DETECT_ENGINE_INSPECT_SIG_MATCH;
+
+ /* continue, this file may (or may not) be unable to match
+ * maybe we have more that can :) */
}
if (r == DETECT_ENGINE_INSPECT_SIG_NO_MATCH && store_r == DETECT_ENGINE_INSPECT_SIG_MATCH) {
DEBUG_VALIDATE_BUG_ON(f->alstate != _alstate);
const uint8_t direction = flags & (STREAM_TOSERVER|STREAM_TOCLIENT);
- FileContainer *ffc = AppLayerParserGetFiles(f, direction);
- if (ffc == NULL || ffc->head == NULL) {
+ FileContainer *ffc = AppLayerParserGetTxFiles(f, tx, direction);
+ SCLogDebug("tx %p tx_id %" PRIu64 " ffc %p ffc->head %p sid %u", tx, tx_id, ffc,
+ ffc ? ffc->head : NULL, s->id);
+ if (ffc == NULL) {
+ SCReturnInt(DETECT_ENGINE_INSPECT_SIG_CANT_MATCH_FILES);
+ } else if (ffc->head == NULL) {
SCReturnInt(DETECT_ENGINE_INSPECT_SIG_NO_MATCH);
}
return false;
}
-static void StoreStateTxHandleFiles(const SigGroupHead *sgh, Flow *f,
- DetectEngineState *destate, const uint8_t flow_flags,
- const uint64_t tx_id, const uint16_t file_no_match)
+static void StoreStateTxHandleFiles(const SigGroupHead *sgh, Flow *f, DetectEngineState *destate,
+ const uint8_t flow_flags, void *tx, const uint64_t tx_id, const uint16_t file_no_match)
{
SCLogDebug("tx %"PRIu64", file_no_match %u", tx_id, file_no_match);
StoreFileNoMatchCnt(destate, file_no_match, flow_flags);
if (StoreFilestoreSigsCantMatch(sgh, destate, flow_flags)) {
- FileDisableStoringForTransaction(f, flow_flags & (STREAM_TOCLIENT | STREAM_TOSERVER), tx_id);
+ SCLogDebug("filestore sigs can't match");
+ FileDisableStoringForTransaction(
+ f, flow_flags & (STREAM_TOCLIENT | STREAM_TOSERVER), tx, tx_id);
+ } else {
+ SCLogDebug("filestore sigs can still match");
}
}
SCLogDebug("destate created for %"PRIu64, tx_id);
}
DeStateSignatureAppend(tx_data->de_state, s, inspect_flags, flow_flags);
- StoreStateTxHandleFiles(sgh, f, tx_data->de_state, flow_flags, tx_id, file_no_match);
+ StoreStateTxHandleFiles(sgh, f, tx_data->de_state, flow_flags, tx, tx_id, file_no_match);
SCLogDebug("Stored for TX %"PRIu64, tx_id);
}
HtpState *http_state = f->alstate;
FAIL_IF_NULL(http_state);
- FAIL_IF_NULL(http_state->files_ts);
+ void *tx = AppLayerParserGetTx(IPPROTO_TCP, ALPROTO_HTTP1, f->alstate, 0);
+ FAIL_IF_NULL(tx);
+ HtpTxUserData *tx_ud = htp_tx_get_user_data(tx);
+ FAIL_IF_NULL(tx_ud);
SigMatchSignatures(&th_v, de_ctx, det_ctx, p);
FAIL_IF(!(PacketAlertCheck(p, 1)));
- FileContainer *files = AppLayerParserGetFiles(p->flow, STREAM_TOSERVER);
+ FileContainer *files = AppLayerParserGetTxFiles(p->flow, tx, STREAM_TOSERVER);
FAIL_IF_NULL(files);
File *file = files->head;
HtpState *http_state = f->alstate;
FAIL_IF_NULL(http_state);
- FAIL_IF_NULL(http_state->files_ts);
+ void *tx = AppLayerParserGetTx(IPPROTO_TCP, ALPROTO_HTTP1, f->alstate, 0);
+ FAIL_IF_NULL(tx);
+ HtpTxUserData *tx_ud = htp_tx_get_user_data(tx);
+ FAIL_IF_NULL(tx_ud);
- FileContainer *files = AppLayerParserGetFiles(p->flow, STREAM_TOSERVER);
+ FileContainer *files = AppLayerParserGetTxFiles(p->flow, tx, STREAM_TOSERVER);
FAIL_IF_NULL(files);
File *file = files->head;
FAIL_IF_NULL(file);
int r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1,
STREAM_TOSERVER | STREAM_START | STREAM_EOF, httpbuf1, httplen1);
FAIL_IF_NOT(r == 0);
- SigMatchSignatures(&th_v, de_ctx, det_ctx, p);
- FAIL_IF(PacketAlertCheck(p, 1));
-
HtpState *http_state = f->alstate;
FAIL_IF_NULL(http_state);
- FAIL_IF_NULL(http_state->files_ts);
+ void *tx = AppLayerParserGetTx(IPPROTO_TCP, ALPROTO_HTTP1, f->alstate, 0);
+ FAIL_IF_NULL(tx);
+ HtpTxUserData *tx_ud = htp_tx_get_user_data(tx);
+ FAIL_IF_NULL(tx_ud);
- FileContainer *files = AppLayerParserGetFiles(p->flow, STREAM_TOSERVER);
+ FileContainer *files = AppLayerParserGetTxFiles(p->flow, tx, STREAM_TOSERVER);
FAIL_IF_NULL(files);
File *file = files->head;
FAIL_IF_NULL(file);
+ FAIL_IF(http_state->state_data.file_flags & FLOWFILE_NO_STORE_TS);
+
+ SigMatchSignatures(&th_v, de_ctx, det_ctx, p);
+ FAIL_IF(PacketAlertCheck(p, 1));
- FAIL_IF(!(file->flags & FILE_NOSTORE));
+ /* detect will have set FLOWFILE_NO_STORE_TS, but it won't have had
+ * an opportunity to be applied to the file itself yet */
+ FAIL_IF_NOT(http_state->state_data.file_flags & FLOWFILE_NO_STORE_TS);
+ FAIL_IF(file->flags & FILE_NOSTORE);
AppLayerParserThreadCtxFree(alp_tctx);
UTHFreeFlow(f);
HtpState *http_state = f->alstate;
FAIL_IF_NULL(http_state);
- FAIL_IF_NULL(http_state->files_ts);
+ void *tx = AppLayerParserGetTx(IPPROTO_TCP, ALPROTO_HTTP1, f->alstate, 0);
+ FAIL_IF_NULL(tx);
+ HtpTxUserData *tx_ud = htp_tx_get_user_data(tx);
+ FAIL_IF_NULL(tx_ud);
- FileContainer *files = AppLayerParserGetFiles(p->flow, STREAM_TOSERVER);
+ FileContainer *files = AppLayerParserGetTxFiles(p->flow, tx, STREAM_TOSERVER);
FAIL_IF_NULL(files);
File *file = files->head;
FAIL_IF_NULL(file);
- FAIL_IF(!(file->flags & FILE_NOSTORE));
+ /* detect will have set FLOWFILE_NO_STORE_TS, but it won't have had
+ * an opportunity to be applied to the file itself yet */
+ FAIL_IF_NOT(tx_ud->tx_data.file_flags & FLOWFILE_NO_STORE_TS);
+ FAIL_IF(file->flags & FILE_NOSTORE);
AppLayerParserThreadCtxFree(alp_tctx);
UTHFreeFlow(f);
HtpState *http_state = f->alstate;
FAIL_IF_NULL(http_state);
- FAIL_IF_NULL(http_state->files_ts);
+ void *tx = AppLayerParserGetTx(IPPROTO_TCP, ALPROTO_HTTP1, f->alstate, 0);
+ FAIL_IF_NULL(tx);
+ HtpTxUserData *tx_ud = htp_tx_get_user_data(tx);
+ FAIL_IF_NULL(tx_ud);
- FileContainer *files = AppLayerParserGetFiles(p->flow, STREAM_TOSERVER);
+ FileContainer *files = AppLayerParserGetTxFiles(p->flow, tx, STREAM_TOSERVER);
FAIL_IF_NULL(files);
File *file = files->head;
FAIL_IF_NULL(file);
HtpState *http_state = f->alstate;
FAIL_IF_NULL(http_state);
- FAIL_IF_NULL(http_state->files_ts);
+ void *tx = AppLayerParserGetTx(IPPROTO_TCP, ALPROTO_HTTP1, f->alstate, 0);
+ FAIL_IF_NULL(tx);
+ HtpTxUserData *tx_ud = htp_tx_get_user_data(tx);
+ FAIL_IF_NULL(tx_ud);
- FileContainer *files = AppLayerParserGetFiles(p->flow, STREAM_TOSERVER);
+ FileContainer *files = AppLayerParserGetTxFiles(p->flow, tx, STREAM_TOSERVER);
FAIL_IF_NULL(files);
File *file = files->head;
FAIL_IF_NULL(file);
http_state = f->alstate;
FAIL_IF_NULL(http_state);
- FAIL_IF_NULL(http_state->files_ts);
+ tx = AppLayerParserGetTx(IPPROTO_TCP, ALPROTO_HTTP1, f->alstate, 0);
+ FAIL_IF_NULL(tx);
+ tx_ud = htp_tx_get_user_data(tx);
+ FAIL_IF_NULL(tx_ud);
- files = AppLayerParserGetFiles(p->flow, STREAM_TOSERVER);
+ files = AppLayerParserGetTxFiles(p->flow, tx, STREAM_TOSERVER);
FAIL_IF_NULL(files);
file = files->head;
FAIL_IF_NULL(file);
HtpState *http_state = f->alstate;
FAIL_IF_NULL(http_state);
- FAIL_IF_NULL(http_state->files_ts);
+ void *tx = AppLayerParserGetTx(IPPROTO_TCP, ALPROTO_HTTP1, f->alstate, 0);
+ FAIL_IF_NULL(tx);
+ HtpTxUserData *tx_ud = htp_tx_get_user_data(tx);
+ FAIL_IF_NULL(tx_ud);
- FileContainer *files = AppLayerParserGetFiles(p->flow, STREAM_TOSERVER);
+ FileContainer *files = AppLayerParserGetTxFiles(p->flow, tx, STREAM_TOSERVER);
FAIL_IF_NULL(files);
File *file = files->head;
FAIL_IF_NULL(file);
http_state = f->alstate;
FAIL_IF_NULL(http_state);
- FAIL_IF_NULL(http_state->files_ts);
+ tx = AppLayerParserGetTx(IPPROTO_TCP, ALPROTO_HTTP1, f->alstate, 0);
+ FAIL_IF_NULL(tx);
+ tx_ud = htp_tx_get_user_data(tx);
+ FAIL_IF_NULL(tx_ud);
- files = AppLayerParserGetFiles(p->flow, STREAM_TOSERVER);
+ files = AppLayerParserGetTxFiles(p->flow, tx, STREAM_TOSERVER);
FAIL_IF_NULL(files);
file = files->head;
FAIL_IF_NULL(file);
HtpState *http_state = f->alstate;
FAIL_IF_NULL(http_state);
- FAIL_IF_NULL(http_state->files_ts);
+ void *tx = AppLayerParserGetTx(IPPROTO_TCP, ALPROTO_HTTP1, f->alstate, 0);
+ FAIL_IF_NULL(tx);
+ HtpTxUserData *tx_ud = htp_tx_get_user_data(tx);
+ FAIL_IF_NULL(tx_ud);
- FileContainer *files = AppLayerParserGetFiles(p->flow, STREAM_TOSERVER);
+ FileContainer *files = AppLayerParserGetTxFiles(p->flow, tx, STREAM_TOSERVER);
FAIL_IF_NULL(files);
File *file = files->head;
FAIL_IF_NULL(file);
http_state = f->alstate;
FAIL_IF_NULL(http_state);
- FAIL_IF_NULL(http_state->files_ts);
+ tx = AppLayerParserGetTx(IPPROTO_TCP, ALPROTO_HTTP1, f->alstate, 0);
+ FAIL_IF_NULL(tx);
+ tx_ud = htp_tx_get_user_data(tx);
+ FAIL_IF_NULL(tx_ud);
- files = AppLayerParserGetFiles(p->flow, STREAM_TOSERVER);
+ files = AppLayerParserGetTxFiles(p->flow, tx, STREAM_TOSERVER);
FAIL_IF_NULL(files);
file = files->head;
FAIL_IF_NULL(file);
transforms = engine->v2.transforms;
}
- FileContainer *ffc = AppLayerParserGetFiles(f, flags);
+ FileContainer *ffc = AppLayerParserGetTxFiles(f, txv, flags);
if (ffc == NULL) {
- return DETECT_ENGINE_INSPECT_SIG_NO_MATCH;
+ return DETECT_ENGINE_INSPECT_SIG_CANT_MATCH_FILES;
}
int local_file_id = 0;
File *file = ffc->head;
for (; file != NULL; file = file->next) {
- if (file->txid != tx_id)
- continue;
-
InspectionBuffer *buffer = FiledataGetDataCallback(det_ctx, transforms, f, flags, file,
engine->sm_list, engine->sm_list_base, local_file_id, false);
if (buffer == NULL)
const MpmCtx *mpm_ctx = ctx->mpm_ctx;
const int list_id = ctx->list_id;
- FileContainer *ffc = AppLayerParserGetFiles(f, flags);
+ FileContainer *ffc = AppLayerParserGetTxFiles(f, txv, flags);
if (ffc != NULL) {
int local_file_id = 0;
for (File *file = ffc->head; file != NULL; file = file->next) {
- if (file->txid != idx)
- continue;
-
InspectionBuffer *buffer = FiledataGetDataCallback(det_ctx, ctx->transforms, f, flags,
file, list_id, ctx->base_list_id, local_file_id, true);
if (buffer == NULL)
transforms = engine->v2.transforms;
}
- FileContainer *ffc = AppLayerParserGetFiles(f, flags);
+ FileContainer *ffc = AppLayerParserGetTxFiles(f, txv, flags);
if (ffc == NULL) {
- return DETECT_ENGINE_INSPECT_SIG_NO_MATCH;
+ return DETECT_ENGINE_INSPECT_SIG_CANT_MATCH_FILES;
}
uint8_t r = DETECT_ENGINE_INSPECT_SIG_NO_MATCH;
int local_file_id = 0;
for (File *file = ffc->head; file != NULL; file = file->next) {
- if (file->txid != tx_id)
- continue;
-
InspectionBuffer *buffer = FilemagicGetDataCallback(det_ctx,
transforms, f, flags, file, engine->sm_list, local_file_id, false);
if (buffer == NULL)
const MpmCtx *mpm_ctx = ctx->mpm_ctx;
const int list_id = ctx->list_id;
- FileContainer *ffc = AppLayerParserGetFiles(f, flags);
+ FileContainer *ffc = AppLayerParserGetTxFiles(f, txv, flags);
if (ffc != NULL) {
int local_file_id = 0;
for (File *file = ffc->head; file != NULL; file = file->next) {
- if (file->txid != idx)
- continue;
-
InspectionBuffer *buffer = FilemagicGetDataCallback(det_ctx,
ctx->transforms, f, flags, file, list_id, local_file_id, true);
if (buffer == NULL)
transforms = engine->v2.transforms;
}
- FileContainer *ffc = AppLayerParserGetFiles(f, flags);
+ FileContainer *ffc = AppLayerParserGetTxFiles(f, txv, flags);
if (ffc == NULL) {
- return DETECT_ENGINE_INSPECT_SIG_NO_MATCH;
+ return DETECT_ENGINE_INSPECT_SIG_CANT_MATCH_FILES;
}
uint8_t r = DETECT_ENGINE_INSPECT_SIG_NO_MATCH;
int local_file_id = 0;
for (File *file = ffc->head; file != NULL; file = file->next) {
- if (file->txid != tx_id)
- continue;
-
InspectionBuffer *buffer = FilenameGetDataCallback(det_ctx,
transforms, f, flags, file, engine->sm_list, local_file_id, false);
if (buffer == NULL)
const MpmCtx *mpm_ctx = ctx->mpm_ctx;
const int list_id = ctx->list_id;
- FileContainer *ffc = AppLayerParserGetFiles(f, flags);
+ FileContainer *ffc = AppLayerParserGetTxFiles(f, txv, flags);
if (ffc != NULL) {
int local_file_id = 0;
for (File *file = ffc->head; file != NULL; file = file->next) {
- if (file->txid != idx)
- continue;
-
InspectionBuffer *buffer = FilenameGetDataCallback(det_ctx,
ctx->transforms, f, flags, file, list_id, local_file_id, true);
if (buffer == NULL)
-/* Copyright (C) 2007-2020 Open Information Security Foundation
+/* Copyright (C) 2007-2021 Open Information Security Foundation
*
* You can copy, redistribute or modify this Program under the terms of
* the GNU General Public License version 2 as published by the Free
#include "detect-filestore.h"
+#include "util-validate.h"
+
/**
* \brief Regex for parsing our flow options
*/
if (this_file) {
FileStoreFileById(fc, file_id);
} else if (this_tx) {
- /* flag tx all files will be stored */
- if (f->alproto == ALPROTO_HTTP1 && f->alstate != NULL) {
- HtpState *htp_state = f->alstate;
- if (toserver_dir) {
- htp_state->flags |= HTP_FLAG_STORE_FILES_TX_TS;
- FileStoreAllFilesForTx(htp_state->files_ts, tx_id);
- }
- if (toclient_dir) {
- htp_state->flags |= HTP_FLAG_STORE_FILES_TX_TC;
- FileStoreAllFilesForTx(htp_state->files_tc, tx_id);
+ /* set in AppLayerTxData. Parsers and logger will propegate it to the
+ * individual files, both new and current. */
+ void *txv = AppLayerParserGetTx(p->proto, f->alproto, f->alstate, tx_id);
+ DEBUG_VALIDATE_BUG_ON(txv == NULL);
+ if (txv != NULL) {
+ AppLayerTxData *txd = AppLayerParserGetTxData(p->proto, f->alproto, txv);
+ DEBUG_VALIDATE_BUG_ON(txd == NULL);
+ if (txd != NULL) {
+ txd->file_flags |= FLOWFILE_STORE;
}
- htp_state->store_tx_id = tx_id;
}
} else if (this_flow) {
- /* flag flow all files will be stored */
- if (f->alproto == ALPROTO_HTTP1 && f->alstate != NULL) {
- HtpState *htp_state = f->alstate;
- if (toserver_dir) {
- htp_state->flags |= HTP_FLAG_STORE_FILES_TS;
- FileStoreAllFiles(htp_state->files_ts);
- }
- if (toclient_dir) {
- htp_state->flags |= HTP_FLAG_STORE_FILES_TC;
- FileStoreAllFiles(htp_state->files_tc);
- }
+ /* set in flow and AppLayerStateData */
+ f->file_flags |= FLOWFILE_STORE;
+
+ AppLayerStateData *sd = AppLayerParserGetStateData(p->proto, f->alproto, f->alstate);
+ if (sd != NULL) {
+ sd->file_flags |= FLOWFILE_STORE;
}
} else {
FileStoreFileById(fc, file_id);
TcpSessionSetReassemblyDepth(ssn, FileReassemblyDepth());
}
+ SCLogDebug("s->filestore_ctx %p", s->filestore_ctx);
+
const uint8_t flags = STREAM_FLAGS_FOR_PACKET(p);
for (uint16_t u = 0; u < det_ctx->filestore_cnt; u++) {
- AppLayerParserSetStreamDepthFlag(p->flow->proto, p->flow->alproto,
- FlowGetAppState(p->flow),
- det_ctx->filestore[u].tx_id,
- flags);
- }
-
- FileContainer *ffc = AppLayerParserGetFiles(p->flow, flags);
-
- /* filestore for single files only */
- if (s->filestore_ctx == NULL) {
- for (uint16_t u = 0; u < det_ctx->filestore_cnt; u++) {
- FileStoreFileById(ffc, det_ctx->filestore[u].file_id);
- }
- } else {
- for (uint16_t u = 0; u < det_ctx->filestore_cnt; u++) {
- FilestorePostMatchWithOptions(p, p->flow, s->filestore_ctx, ffc,
- det_ctx->filestore[u].file_id, det_ctx->filestore[u].tx_id);
+ AppLayerParserSetStreamDepthFlag(p->flow->proto, p->flow->alproto, FlowGetAppState(p->flow),
+ det_ctx->filestore[u].tx_id, flags);
+
+ void *txv = AppLayerParserGetTx(p->flow->proto, p->flow->alproto, FlowGetAppState(p->flow),
+ det_ctx->filestore[u].tx_id);
+ DEBUG_VALIDATE_BUG_ON(txv == NULL);
+ if (txv) {
+ FileContainer *ffc_tx = AppLayerParserGetTxFiles(p->flow, txv, flags);
+ DEBUG_VALIDATE_BUG_ON(ffc_tx == NULL);
+ if (ffc_tx) {
+ SCLogDebug("u %u txv %p ffc_tx %p file_id %u", u, txv, ffc_tx,
+ det_ctx->filestore[u].file_id);
+
+ /* filestore for single files only */
+ if (s->filestore_ctx == NULL) {
+ FileStoreFileById(ffc_tx, det_ctx->filestore[u].file_id);
+ } else {
+ FilestorePostMatchWithOptions(p, p->flow, s->filestore_ctx, ffc_tx,
+ det_ctx->filestore[u].file_id, det_ctx->filestore[u].tx_id);
+ }
+ }
}
}
-
SCReturnInt(0);
}
SCEnter();
+ if (!RunmodeIsUnittests()) {
+ extern bool g_filedata_logger_enabled;
+ if (!g_filedata_logger_enabled) {
+ SCLogDebug("not storing file match: no filedata logger enabled");
+ SCReturnInt(1);
+ }
+ }
if (det_ctx->filestore_cnt >= DETECT_FILESTORE_MAX) {
SCReturnInt(1);
}
#define __DETECT_H__
#include "suricata-common.h"
-
#include "flow.h"
+#include "app-layer-events.h"
#include "detect-engine-proto.h"
#include "detect-reference.h"
#include "util-file.h"
#include "reputation.h"
-#include "app-layer-events.h"
-
#define DETECT_MAX_RULE_SIZE 8192
#define DETECT_TRANSFORMS_MAX 16
}
}
-static void FlowPruneFiles(Packet *p)
-{
- if (p->flow && p->flow->alstate) {
- Flow *f = p->flow;
- FileContainer *fc = AppLayerParserGetFiles(f,
- PKT_IS_TOSERVER(p) ? STREAM_TOSERVER : STREAM_TOCLIENT);
- if (fc != NULL) {
- FilePrune(fc);
- }
- }
-}
-
/** \brief update stream engine
*
* We can be called from both the flow timeout path as well as from the
// Outputs.
OutputLoggerLog(tv, p, fw->output_thread);
- /* Prune any stored files. */
- FlowPruneFiles(p);
-
FramesPrune(p->flow, p);
+
/* Release tcp segments. Done here after alerting can use them. */
FLOWWORKER_PROFILING_START(p, PROFILE_FLOWWORKER_TCPPRUNE);
StreamTcpPruneSession(p->flow, p->flowflags & FLOW_PKT_TOSERVER ?
// Outputs.
OutputLoggerLog(tv, p, fw->output_thread);
- /* Prune any stored files. */
- FlowPruneFiles(p);
-
/* Release tcp segments. Done here after alerting can use them. */
if (p->flow != NULL) {
DEBUG_ASSERT_FLOW_LOCKED(p->flow);
bool g_file_logger_enabled = false;
-/** per thread data for this module, contains a list of per thread
- * data for the packet loggers. */
-typedef struct OutputFileLoggerThreadData_ {
- OutputLoggerThreadStore *store;
-#ifdef HAVE_MAGIC
- magic_t magic_ctx;
-#endif
-} OutputFileLoggerThreadData;
-
/* logger instance, a module + a output ctx,
* it's perfectly valid that have multiple instances of the same
* log module (e.g. http.log) with different output ctx'. */
return 0;
}
-static void CloseFile(const Packet *p, Flow *f, File *file)
+static void CloseFile(const Packet *p, Flow *f, AppLayerTxData *txd, File *file)
{
DEBUG_VALIDATE_BUG_ON((file->flags & FILE_LOGGED) != 0);
- void *txv = AppLayerParserGetTx(f->proto, f->alproto, f->alstate, file->txid);
- if (txv) {
- AppLayerTxData *txd = AppLayerParserGetTxData(f->proto, f->alproto, txv);
- if (txd) {
- txd->files_logged++;
- DEBUG_VALIDATE_BUG_ON(txd->files_logged > txd->files_opened);
- }
- }
+ DEBUG_VALIDATE_BUG_ON(f->alproto == ALPROTO_SMB && txd->files_logged != 0);
+ DEBUG_VALIDATE_BUG_ON(f->alproto == ALPROTO_FTPDATA && txd->files_logged != 0);
+ txd->files_logged++;
+ DEBUG_VALIDATE_BUG_ON(txd->files_logged > txd->files_opened);
file->flags |= FILE_LOGGED;
+ SCLogDebug("ff %p FILE_LOGGED", file);
}
-static void OutputFileLogFfc(ThreadVars *tv, OutputFileLoggerThreadData *op_thread_data, Packet *p,
- FileContainer *ffc, const bool file_close, const bool file_trunc, uint8_t dir)
+void OutputFileLogFfc(ThreadVars *tv, OutputFileLoggerThreadData *op_thread_data, Packet *p,
+ FileContainer *ffc, void *txv, const uint64_t tx_id, AppLayerTxData *txd,
+ const bool file_close, const bool file_trunc, uint8_t dir)
{
- SCLogDebug("ffc %p", ffc);
- if (ffc != NULL) {
- File *ff;
- for (ff = ffc->head; ff != NULL; ff = ff->next) {
- if (ff->flags & FILE_LOGGED)
- continue;
-
- SCLogDebug("ff %p", ff);
+ if (ffc->head == NULL)
+ return;
- if (file_trunc && ff->state < FILE_STATE_CLOSED)
- ff->state = FILE_STATE_TRUNCATED;
+ SCLogDebug("ffc %p ffc->head %p file_close %d file_trunc %d dir %s", ffc,
+ ffc ? ffc->head : NULL, file_close, file_trunc, dir == STREAM_TOSERVER ? "ts" : "tc");
+ File *ff;
+ for (ff = ffc->head; ff != NULL; ff = ff->next) {
+ SCLogDebug("ff %p pre-FILE_LOGGED", ff);
+ if (ff->flags & FILE_LOGGED)
+ continue;
- if (file_close && ff->state < FILE_STATE_CLOSED)
- ff->state = FILE_STATE_TRUNCATED;
+ FileApplyTxFlags(txd, dir, ff);
- SCLogDebug("ff %p state %u", ff, ff->state);
-
- if (ff->state > FILE_STATE_OPENED) {
- bool file_logged = false;
-#ifdef HAVE_MAGIC
- if (FileForceMagic() && ff->magic == NULL) {
- FilemagicThreadLookup(&op_thread_data->magic_ctx, ff);
- }
-#endif
- const OutputFileLogger *logger = list;
- const OutputLoggerThreadStore *store = op_thread_data->store;
- while (logger && store) {
- DEBUG_VALIDATE_BUG_ON(logger->LogFunc == NULL);
-
- SCLogDebug("logger %p", logger);
- PACKET_PROFILING_LOGGER_START(p, logger->logger_id);
- logger->LogFunc(tv, store->thread_data, (const Packet *)p, (const File *)ff, dir);
- PACKET_PROFILING_LOGGER_END(p, logger->logger_id);
- file_logged = true;
-
- logger = logger->next;
- store = store->next;
-
- DEBUG_VALIDATE_BUG_ON(logger == NULL && store != NULL);
- DEBUG_VALIDATE_BUG_ON(logger != NULL && store == NULL);
- }
+ SCLogDebug("ff %p state %u post-FILE_LOGGED", ff, ff->state);
- if (file_logged) {
- CloseFile(p, p->flow, ff);
- }
- }
+ if (file_trunc && ff->state < FILE_STATE_CLOSED) {
+ SCLogDebug("file_trunc %d ff->state %u => FILE_STATE_TRUNCATED", file_trunc, ff->state);
+ ff->state = FILE_STATE_TRUNCATED;
}
- }
-}
-
-static TmEcode OutputFileLog(ThreadVars *tv, Packet *p, void *thread_data)
-{
- DEBUG_VALIDATE_BUG_ON(thread_data == NULL);
- if (list == NULL) {
- /* No child loggers. */
- return TM_ECODE_OK;
- }
+ if (file_close && ff->state < FILE_STATE_CLOSED) {
+ SCLogDebug("file_close %d ff->state %u => FILE_STATE_TRUNCATED", file_close, ff->state);
+ ff->state = FILE_STATE_TRUNCATED;
+ }
- OutputFileLoggerThreadData *op_thread_data = (OutputFileLoggerThreadData *)thread_data;
+ SCLogDebug("ff %p state %u", ff, ff->state);
- /* no flow, no files */
- Flow * const f = p->flow;
- if (f == NULL || f->alstate == NULL) {
- SCReturnInt(TM_ECODE_OK);
- }
+ if (ff->state > FILE_STATE_OPENED) {
+ SCLogDebug("FILE LOGGING");
+ bool file_logged = false;
+#ifdef HAVE_MAGIC
+ if (FileForceMagic() && ff->magic == NULL) {
+ FilemagicThreadLookup(&op_thread_data->magic_ctx, ff);
+ }
+#endif
+ const OutputFileLogger *logger = list;
+ const OutputLoggerThreadStore *store = op_thread_data->store;
+ while (logger && store) {
+ DEBUG_VALIDATE_BUG_ON(logger->LogFunc == NULL);
+
+ SCLogDebug("logger %p", logger);
+ PACKET_PROFILING_LOGGER_START(p, logger->logger_id);
+ logger->LogFunc(tv, store->thread_data, (const Packet *)p, (const File *)ff, txv,
+ tx_id, dir);
+ PACKET_PROFILING_LOGGER_END(p, logger->logger_id);
+ file_logged = true;
+
+ logger = logger->next;
+ store = store->next;
+
+ DEBUG_VALIDATE_BUG_ON(logger == NULL && store != NULL);
+ DEBUG_VALIDATE_BUG_ON(logger != NULL && store == NULL);
+ }
- if (p->proto == IPPROTO_TCP) {
- const bool file_trunc = StreamTcpReassembleDepthReached(p);
- if (p->flowflags & FLOW_PKT_TOSERVER) {
- const bool file_close_ts = ((p->flags & PKT_PSEUDO_STREAM_END));
- FileContainer *ffc_ts = AppLayerParserGetFiles(f, STREAM_TOSERVER);
- OutputFileLogFfc(
- tv, op_thread_data, p, ffc_ts, file_close_ts, file_trunc, STREAM_TOSERVER);
- } else {
- const bool file_close_tc = ((p->flags & PKT_PSEUDO_STREAM_END));
- FileContainer *ffc_tc = AppLayerParserGetFiles(f, STREAM_TOCLIENT);
- OutputFileLogFfc(
- tv, op_thread_data, p, ffc_tc, file_close_tc, file_trunc, STREAM_TOCLIENT);
+ if (file_logged) {
+ CloseFile(p, p->flow, txd, ff);
+ }
}
- } else if (p->proto == IPPROTO_UDP) {
- FileContainer *ffc_ts = AppLayerParserGetFiles(f, STREAM_TOSERVER);
- OutputFileLogFfc(tv, op_thread_data, p, ffc_ts, false, false, STREAM_TOSERVER);
- FileContainer *ffc_tc = AppLayerParserGetFiles(f, STREAM_TOCLIENT);
- OutputFileLogFfc(tv, op_thread_data, p, ffc_tc, false, false, STREAM_TOCLIENT);
}
- return TM_ECODE_OK;
}
-/** \brief thread init for the tx logger
+/** \brief thread init for the file logger
* This will run the thread init functions for the individual registered
* loggers */
-static TmEcode OutputFileLogThreadInit(ThreadVars *tv, const void *initdata, void **data)
+TmEcode OutputFileLogThreadInit(ThreadVars *tv, OutputFileLoggerThreadData **data)
{
- OutputFileLoggerThreadData *td = SCMalloc(sizeof(*td));
+ OutputFileLoggerThreadData *td = SCCalloc(1, sizeof(*td));
if (td == NULL)
return TM_ECODE_FAILED;
- memset(td, 0x00, sizeof(*td));
-
- *data = (void *)td;
+ *data = td;
#ifdef HAVE_MAGIC
td->magic_ctx = MagicInitContext();
return TM_ECODE_OK;
}
-static TmEcode OutputFileLogThreadDeinit(ThreadVars *tv, void *thread_data)
+TmEcode OutputFileLogThreadDeinit(ThreadVars *tv, OutputFileLoggerThreadData *op_thread_data)
{
- OutputFileLoggerThreadData *op_thread_data = (OutputFileLoggerThreadData *)thread_data;
OutputLoggerThreadStore *store = op_thread_data->store;
OutputFileLogger *logger = list;
return TM_ECODE_OK;
}
-static void OutputFileLogExitPrintStats(ThreadVars *tv, void *thread_data)
-{
- OutputFileLoggerThreadData *op_thread_data = (OutputFileLoggerThreadData *)thread_data;
- OutputLoggerThreadStore *store = op_thread_data->store;
- OutputFileLogger *logger = list;
-
- while (logger && store) {
- if (logger->ThreadExitPrintStats) {
- logger->ThreadExitPrintStats(tv, store->thread_data);
- }
-
- logger = logger->next;
- store = store->next;
- }
-}
-
-static uint32_t OutputFileLoggerGetActiveCount(void)
-{
- uint32_t cnt = 0;
- for (OutputFileLogger *p = list; p != NULL; p = p->next) {
- cnt++;
- }
- return cnt;
-}
-
void OutputFileLoggerRegister(void)
{
- OutputRegisterRootLogger(OutputFileLogThreadInit,
- OutputFileLogThreadDeinit, OutputFileLogExitPrintStats,
- OutputFileLog, OutputFileLoggerGetActiveCount);
}
void OutputFileShutdown(void)
#define __OUTPUT_FILE_H__
#include "decode.h"
+#include "rust.h"
#include "util-file.h"
+/** per thread data for this module, contains a list of per thread
+ * data for the packet loggers. */
+typedef struct OutputFileLoggerThreadData_ {
+ OutputLoggerThreadStore *store;
+#ifdef HAVE_MAGIC
+ magic_t magic_ctx;
+#endif
+} OutputFileLoggerThreadData;
+
+TmEcode OutputFileLogThreadInit(ThreadVars *tv, OutputFileLoggerThreadData **data);
+TmEcode OutputFileLogThreadDeinit(ThreadVars *tv, OutputFileLoggerThreadData *thread_data);
+
+void OutputFileLogFfc(ThreadVars *tv, OutputFileLoggerThreadData *op_thread_data, Packet *p,
+ FileContainer *ffc, void *txv, const uint64_t tx_id, AppLayerTxData *txd,
+ const bool file_close, const bool file_trunc, uint8_t dir);
+
/** file logger function pointer type */
-typedef int (*FileLogger)(ThreadVars *, void *thread_data, const Packet *,
- const File *, uint8_t direction);
+typedef int (*FileLogger)(ThreadVars *, void *thread_data, const Packet *, const File *, void *tx,
+ const uint64_t tx_id, uint8_t direction);
int OutputRegisterFileLogger(LoggerId id, const char *name, FileLogger LogFunc,
OutputCtx *, ThreadInitFunc ThreadInit, ThreadDeinitFunc ThreadDeinit,
bool g_filedata_logger_enabled = false;
-/** per thread data for this module, contains a list of per thread
- * data for the packet loggers. */
-typedef struct OutputFiledataLoggerThreadData_ {
- OutputLoggerThreadStore *store;
-#ifdef HAVE_MAGIC
- magic_t magic_ctx;
-#endif
-} OutputFiledataLoggerThreadData;
-
/* logger instance, a module + a output ctx,
* it's perfectly valid that have multiple instances of the same
* log module (e.g. http.log) with different output ctx'. */
SC_ATOMIC_DECLARE(unsigned int, g_file_store_id);
-static int CallLoggers(ThreadVars *tv, OutputLoggerThreadStore *store_list,
- Packet *p, File *ff,
- const uint8_t *data, uint32_t data_len, uint8_t flags, uint8_t dir)
+static int CallLoggers(ThreadVars *tv, OutputLoggerThreadStore *store_list, Packet *p, File *ff,
+ void *tx, const uint64_t tx_id, const uint8_t *data, uint32_t data_len, uint8_t flags,
+ uint8_t dir)
{
OutputFiledataLogger *logger = list;
OutputLoggerThreadStore *store = store_list;
SCLogDebug("logger %p", logger);
PACKET_PROFILING_LOGGER_START(p, logger->logger_id);
- logger->LogFunc(tv, store->thread_data, (const Packet *)p, ff, data, data_len, flags, dir);
+ logger->LogFunc(tv, store->thread_data, (const Packet *)p, ff, tx, tx_id, data, data_len,
+ flags, dir);
PACKET_PROFILING_LOGGER_END(p, logger->logger_id);
file_logged = 1;
return file_logged;
}
-static void CloseFile(const Packet *p, Flow *f, File *file)
+static void CloseFile(const Packet *p, Flow *f, File *file, void *txv)
{
- void *txv = AppLayerParserGetTx(f->proto, f->alproto, f->alstate, file->txid);
- if (txv) {
- AppLayerTxData *txd = AppLayerParserGetTxData(f->proto, f->alproto, txv);
- if (txd)
- txd->files_stored++;
+ DEBUG_VALIDATE_BUG_ON((file->flags & FILE_STORED) != 0);
+
+ AppLayerTxData *txd = AppLayerParserGetTxData(f->proto, f->alproto, txv);
+ if (txd) {
+ BUG_ON(f->alproto == ALPROTO_SMB && txd->files_logged != 0);
+ txd->files_stored++;
}
file->flags |= FILE_STORED;
}
-static void OutputFiledataLogFfc(ThreadVars *tv, OutputFiledataLoggerThreadData *td, Packet *p,
- FileContainer *ffc, const uint8_t call_flags, const bool file_close, const bool file_trunc,
- const uint8_t dir)
+void OutputFiledataLogFfc(ThreadVars *tv, OutputFiledataLoggerThreadData *td, Packet *p,
+ FileContainer *ffc, void *txv, const uint64_t tx_id, AppLayerTxData *txd,
+ const uint8_t call_flags, const bool file_close, const bool file_trunc, const uint8_t dir)
{
- if (ffc != NULL) {
- OutputLoggerThreadStore *store = td->store;
- File *ff;
- for (ff = ffc->head; ff != NULL; ff = ff->next) {
- uint8_t file_flags = call_flags;
-#ifdef HAVE_MAGIC
- if (FileForceMagic() && ff->magic == NULL) {
- FilemagicThreadLookup(&td->magic_ctx, ff);
- }
-#endif
- SCLogDebug("ff %p", ff);
- if (ff->flags & FILE_STORED) {
- SCLogDebug("stored flag set");
- continue;
- }
+ SCLogDebug("ffc %p", ffc);
- if (!(ff->flags & FILE_STORE)) {
- SCLogDebug("ff FILE_STORE not set");
- continue;
- }
+ OutputLoggerThreadStore *store = td->store;
+ File *ff;
+ for (ff = ffc->head; ff != NULL; ff = ff->next) {
- /* if we have no data chunks left to log, we should still
- * close the logger(s) */
- if (FileDataSize(ff) == ff->content_stored &&
- (file_trunc || file_close)) {
- if (ff->state < FILE_STATE_CLOSED) {
- FileCloseFilePtr(ff, NULL, 0, FILE_TRUNCATED);
- }
- CallLoggers(tv, store, p, ff, NULL, 0, OUTPUT_FILEDATA_FLAG_CLOSE, dir);
- CloseFile(p, p->flow, ff);
- continue;
- }
+ FileApplyTxFlags(txd, dir, ff);
- /* store */
+ uint8_t file_flags = call_flags;
+#ifdef HAVE_MAGIC
+ if (FileForceMagic() && ff->magic == NULL) {
+ FilemagicThreadLookup(&td->magic_ctx, ff);
+ }
+#endif
+ if (ff->flags & FILE_STORED) {
+ continue;
+ }
- /* if file_store_id == 0, this is the first store of this file */
- if (ff->file_store_id == 0) {
- /* new file */
- ff->file_store_id = SC_ATOMIC_ADD(g_file_store_id, 1);
- file_flags |= OUTPUT_FILEDATA_FLAG_OPEN;
- } else {
- /* existing file */
- }
+ if (!(ff->flags & FILE_STORE)) {
+ continue;
+ }
- /* if file needs to be closed or truncated, inform
- * loggers */
- if ((file_close || file_trunc) && ff->state < FILE_STATE_CLOSED) {
+ /* if we have no data chunks left to log, we should still
+ * close the logger(s) */
+ if (FileDataSize(ff) == ff->content_stored && (file_trunc || file_close)) {
+ if (ff->state < FILE_STATE_CLOSED) {
FileCloseFilePtr(ff, NULL, 0, FILE_TRUNCATED);
}
+ CallLoggers(tv, store, p, ff, txv, tx_id, NULL, 0, OUTPUT_FILEDATA_FLAG_CLOSE, dir);
+ CloseFile(p, p->flow, ff, txv);
+ continue;
+ }
- /* tell the logger we're closing up */
- if (ff->state >= FILE_STATE_CLOSED)
- file_flags |= OUTPUT_FILEDATA_FLAG_CLOSE;
-
- /* do the actual logging */
- const uint8_t *data = NULL;
- uint32_t data_len = 0;
-
- StreamingBufferGetDataAtOffset(ff->sb,
- &data, &data_len,
- ff->content_stored);
+ /* store */
- const int file_logged = CallLoggers(tv, store, p, ff, data, data_len, file_flags, dir);
- if (file_logged) {
- ff->content_stored += data_len;
+ /* if file_store_id == 0, this is the first store of this file */
+ if (ff->file_store_id == 0) {
+ /* new file */
+ ff->file_store_id = SC_ATOMIC_ADD(g_file_store_id, 1);
+ file_flags |= OUTPUT_FILEDATA_FLAG_OPEN;
+ } else {
+ /* existing file */
+ }
- /* all done */
- if (file_flags & OUTPUT_FILEDATA_FLAG_CLOSE) {
- CloseFile(p, p->flow, ff);
- }
- }
+ /* if file needs to be closed or truncated, inform
+ * loggers */
+ if ((file_close || file_trunc) && ff->state < FILE_STATE_CLOSED) {
+ FileCloseFilePtr(ff, NULL, 0, FILE_TRUNCATED);
}
- }
-}
-static TmEcode OutputFiledataLog(ThreadVars *tv, Packet *p, void *thread_data)
-{
- DEBUG_VALIDATE_BUG_ON(thread_data == NULL);
+ /* tell the logger we're closing up */
+ if (ff->state >= FILE_STATE_CLOSED)
+ file_flags |= OUTPUT_FILEDATA_FLAG_CLOSE;
- if (list == NULL) {
- /* No child loggers. */
- return TM_ECODE_OK;
- }
+ /* do the actual logging */
+ const uint8_t *data = NULL;
+ uint32_t data_len = 0;
- OutputFiledataLoggerThreadData *op_thread_data = (OutputFiledataLoggerThreadData *)thread_data;
+ StreamingBufferGetDataAtOffset(ff->sb, &data, &data_len, ff->content_stored);
- /* no flow, no files */
- Flow * const f = p->flow;
- if (f == NULL || f->alstate == NULL) {
- SCReturnInt(TM_ECODE_OK);
- }
- /* do not log for ICMP packets related to a TCP/UDP flow */
- if (p->proto != IPPROTO_TCP && p->proto != IPPROTO_UDP) {
- SCReturnInt(TM_ECODE_OK);
- }
+ const int file_logged =
+ CallLoggers(tv, store, p, ff, txv, tx_id, data, data_len, file_flags, dir);
+ if (file_logged) {
+ ff->content_stored += data_len;
- const bool file_trunc = StreamTcpReassembleDepthReached(p);
- if (p->flowflags & FLOW_PKT_TOSERVER) {
- const bool file_close_ts = ((p->flags & PKT_PSEUDO_STREAM_END));
- FileContainer *ffc_ts = AppLayerParserGetFiles(f, STREAM_TOSERVER);
- SCLogDebug("ffc_ts %p", ffc_ts);
- OutputFiledataLogFfc(tv, op_thread_data, p, ffc_ts, STREAM_TOSERVER, file_close_ts,
- file_trunc, STREAM_TOSERVER);
- } else {
- const bool file_close_tc = ((p->flags & PKT_PSEUDO_STREAM_END));
- FileContainer *ffc_tc = AppLayerParserGetFiles(f, STREAM_TOCLIENT);
- SCLogDebug("ffc_tc %p", ffc_tc);
- OutputFiledataLogFfc(tv, op_thread_data, p, ffc_tc, STREAM_TOCLIENT, file_close_tc,
- file_trunc, STREAM_TOCLIENT);
+ /* all done */
+ if (file_flags & OUTPUT_FILEDATA_FLAG_CLOSE) {
+ CloseFile(p, p->flow, ff, txv);
+ }
+ }
}
-
- return TM_ECODE_OK;
}
-/** \brief thread init for the tx logger
+/** \brief thread init for the filedata logger
* This will run the thread init functions for the individual registered
* loggers */
-static TmEcode OutputFiledataLogThreadInit(ThreadVars *tv, const void *initdata, void **data)
+TmEcode OutputFiledataLogThreadInit(ThreadVars *tv, OutputFiledataLoggerThreadData **data)
{
OutputFiledataLoggerThreadData *td = SCMalloc(sizeof(*td));
if (td == NULL)
return TM_ECODE_FAILED;
memset(td, 0x00, sizeof(*td));
-
- *data = (void *)td;
+ *data = td;
#ifdef HAVE_MAGIC
td->magic_ctx = MagicInitContext();
void *retptr = NULL;
if (logger->ThreadInit(tv, (void *)logger->output_ctx, &retptr) == TM_ECODE_OK) {
OutputLoggerThreadStore *ts = SCMalloc(sizeof(*ts));
-/* todo */ BUG_ON(ts == NULL);
+ /* todo */ BUG_ON(ts == NULL);
memset(ts, 0x00, sizeof(*ts));
/* store thread handle */
return TM_ECODE_OK;
}
-static TmEcode OutputFiledataLogThreadDeinit(ThreadVars *tv, void *thread_data)
+TmEcode OutputFiledataLogThreadDeinit(
+ ThreadVars *tv, OutputFiledataLoggerThreadData *op_thread_data)
{
- OutputFiledataLoggerThreadData *op_thread_data = (OutputFiledataLoggerThreadData *)thread_data;
OutputLoggerThreadStore *store = op_thread_data->store;
OutputFiledataLogger *logger = list;
return TM_ECODE_OK;
}
-static void OutputFiledataLogExitPrintStats(ThreadVars *tv, void *thread_data)
-{
- OutputFiledataLoggerThreadData *op_thread_data = (OutputFiledataLoggerThreadData *)thread_data;
- OutputLoggerThreadStore *store = op_thread_data->store;
- OutputFiledataLogger *logger = list;
-
- while (logger && store) {
- if (logger->ThreadExitPrintStats) {
- logger->ThreadExitPrintStats(tv, store->thread_data);
- }
-
- logger = logger->next;
- store = store->next;
- }
-}
-
-static uint32_t OutputFiledataLoggerGetActiveCount(void)
-{
- uint32_t cnt = 0;
- for (OutputFiledataLogger *p = list; p != NULL; p = p->next) {
- cnt++;
- }
- return cnt;
-}
-
void OutputFiledataLoggerRegister(void)
{
- OutputRegisterRootLogger(OutputFiledataLogThreadInit,
- OutputFiledataLogThreadDeinit, OutputFiledataLogExitPrintStats,
- OutputFiledataLog, OutputFiledataLoggerGetActiveCount);
SC_ATOMIC_INIT(g_file_store_id);
SC_ATOMIC_SET(g_file_store_id, 1);
}
#define OUTPUT_FILEDATA_FLAG_OPEN 0x01
#define OUTPUT_FILEDATA_FLAG_CLOSE 0x02
+/** per thread data for this module, contains a list of per thread
+ * data for the packet loggers. */
+typedef struct OutputFiledataLoggerThreadData_ {
+ OutputLoggerThreadStore *store;
+#ifdef HAVE_MAGIC
+ magic_t magic_ctx;
+#endif
+} OutputFiledataLoggerThreadData;
+
+TmEcode OutputFiledataLogThreadInit(ThreadVars *tv, OutputFiledataLoggerThreadData **data);
+TmEcode OutputFiledataLogThreadDeinit(ThreadVars *tv, OutputFiledataLoggerThreadData *thread_data);
+
+void OutputFiledataLogFfc(ThreadVars *tv, OutputFiledataLoggerThreadData *td, Packet *p,
+ FileContainer *ffc, void *txv, const uint64_t tx_id, AppLayerTxData *txd,
+ const uint8_t call_flags, const bool file_close, const bool file_trunc, const uint8_t dir);
+
/** filedata logger function pointer type */
-typedef int (*FiledataLogger)(ThreadVars *, void *thread_data, const Packet *,
- File *, const uint8_t *, uint32_t, uint8_t, uint8_t dir);
+typedef int (*FiledataLogger)(ThreadVars *, void *thread_data, const Packet *, File *, void *tx,
+ const uint64_t tx_id, const uint8_t *, uint32_t, uint8_t, uint8_t dir);
int OutputRegisterFiledataLogger(LoggerId id, const char *name,
FiledataLogger LogFunc, OutputCtx *, ThreadInitFunc ThreadInit,
}
}
-static void OutputFilestoreFinalizeFiles(ThreadVars *tv,
- const OutputFilestoreLogThread *oft, const OutputFilestoreCtx *ctx,
- const Packet *p, File *ff, uint8_t dir) {
+static void OutputFilestoreFinalizeFiles(ThreadVars *tv, const OutputFilestoreLogThread *oft,
+ const OutputFilestoreCtx *ctx, const Packet *p, File *ff, void *tx, const uint64_t tx_id,
+ uint8_t dir)
+{
/* Stringify the SHA256 which will be used in the final
* filename. */
char sha256string[(SC_SHA256_LEN * 2) + 1];
"Failed to write file info record. Output filename truncated.");
} else {
JsonBuilder *js_fileinfo =
- JsonBuildFileInfoRecord(p, ff, true, dir, ctx->xff_cfg, NULL);
+ JsonBuildFileInfoRecord(p, ff, tx, tx_id, true, dir, ctx->xff_cfg, NULL);
if (likely(js_fileinfo != NULL)) {
jb_close(js_fileinfo);
FILE *out = fopen(js_metadata_filename, "w");
}
}
-static int OutputFilestoreLogger(ThreadVars *tv, void *thread_data,
- const Packet *p, File *ff, const uint8_t *data, uint32_t data_len,
- uint8_t flags, uint8_t dir)
+static int OutputFilestoreLogger(ThreadVars *tv, void *thread_data, const Packet *p, File *ff,
+ void *tx, const uint64_t tx_id, const uint8_t *data, uint32_t data_len, uint8_t flags,
+ uint8_t dir)
{
SCEnter();
OutputFilestoreLogThread *aft = (OutputFilestoreLogThread *)thread_data;
ff->fd = -1;
SC_ATOMIC_SUB(filestore_open_file_cnt, 1);
}
- OutputFilestoreFinalizeFiles(tv, aft, ctx, p, ff, dir);
+ OutputFilestoreFinalizeFiles(tv, aft, ctx, p, ff, tx, tx_id, dir);
}
return 0;
static void AlertAddFiles(const Packet *p, JsonBuilder *jb, const uint64_t tx_id)
{
- FileContainer *ffc = AppLayerParserGetFiles(p->flow,
- p->flowflags & FLOW_PKT_TOSERVER ? STREAM_TOSERVER:STREAM_TOCLIENT);
+ const uint8_t direction =
+ (p->flowflags & FLOW_PKT_TOSERVER) ? STREAM_TOSERVER : STREAM_TOCLIENT;
+ FileContainer *ffc = NULL;
+ if (p->flow->alstate != NULL) {
+ void *tx = AppLayerParserGetTx(p->proto, p->flow->alproto, p->flow->alstate, tx_id);
+ if (tx) {
+ ffc = AppLayerParserGetTxFiles(p->flow, tx, direction);
+ }
+ }
if (ffc != NULL) {
File *file = ffc->head;
bool isopen = false;
while (file) {
- if (tx_id == file->txid) {
- if (!isopen) {
- isopen = true;
- jb_open_array(jb, "files");
- }
- jb_start_object(jb);
- EveFileInfo(jb, file, file->flags & FILE_STORED);
- jb_close(jb);
+ if (!isopen) {
+ isopen = true;
+ jb_open_array(jb, "files");
}
+ jb_start_object(jb);
+ EveFileInfo(jb, file, tx_id, file->flags & FILE_STORED);
+ jb_close(jb);
file = file->next;
}
if (isopen) {
OutputJsonThreadCtx *ctx;
} JsonFileLogThread;
-JsonBuilder *JsonBuildFileInfoRecord(const Packet *p, const File *ff, const bool stored,
- uint8_t dir, HttpXFFCfg *xff_cfg, OutputJsonCtx *eve_ctx)
+JsonBuilder *JsonBuildFileInfoRecord(const Packet *p, const File *ff, void *tx,
+ const uint64_t tx_id, const bool stored, uint8_t dir, HttpXFFCfg *xff_cfg,
+ OutputJsonCtx *eve_ctx)
{
enum OutputJsonLogDirection fdir = LOG_DIR_FLOW;
char xff_buffer[XFF_MAXLEN];
if ((xff_cfg != NULL) && !(xff_cfg->flags & XFF_DISABLED)) {
if (FlowGetAppProtocol(p->flow) == ALPROTO_HTTP1) {
- have_xff_ip = HttpXFFGetIPFromTx(p->flow, ff->txid, xff_cfg, xff_buffer, XFF_MAXLEN);
+ have_xff_ip = HttpXFFGetIPFromTx(p->flow, tx_id, xff_cfg, xff_buffer, XFF_MAXLEN);
}
if (have_xff_ip && xff_cfg->flags & XFF_OVERWRITE) {
if (p->flowflags & FLOW_PKT_TOCLIENT) {
switch (p->flow->alproto) {
case ALPROTO_HTTP1:
jb_open_object(js, "http");
- EveHttpAddMetadata(p->flow, ff->txid, js);
+ EveHttpAddMetadata(p->flow, tx_id, js);
jb_close(js);
break;
case ALPROTO_SMTP:
jb_get_mark(js, &mark);
jb_open_object(js, "smtp");
- if (EveSMTPAddMetadata(p->flow, ff->txid, js)) {
+ if (EveSMTPAddMetadata(p->flow, tx_id, js)) {
jb_close(js);
} else {
jb_restore_mark(js, &mark);
}
jb_get_mark(js, &mark);
jb_open_object(js, "email");
- if (EveEmailAddMetadata(p->flow, ff->txid, js)) {
+ if (EveEmailAddMetadata(p->flow, tx_id, js)) {
jb_close(js);
} else {
jb_restore_mark(js, &mark);
/* rpc */
jb_get_mark(js, &mark);
jb_open_object(js, "rpc");
- if (EveNFSAddMetadataRPC(p->flow, ff->txid, js)) {
+ if (EveNFSAddMetadataRPC(p->flow, tx_id, js)) {
jb_close(js);
} else {
jb_restore_mark(js, &mark);
/* nfs */
jb_get_mark(js, &mark);
jb_open_object(js, "nfs");
- if (EveNFSAddMetadata(p->flow, ff->txid, js)) {
+ if (EveNFSAddMetadata(p->flow, tx_id, js)) {
jb_close(js);
} else {
jb_restore_mark(js, &mark);
case ALPROTO_SMB:
jb_get_mark(js, &mark);
jb_open_object(js, "smb");
- if (EveSMBAddMetadata(p->flow, ff->txid, js)) {
+ if (EveSMBAddMetadata(p->flow, tx_id, js)) {
jb_close(js);
} else {
jb_restore_mark(js, &mark);
case ALPROTO_HTTP2:
jb_get_mark(js, &mark);
jb_open_object(js, "http2");
- if (EveHTTP2AddMetadata(p->flow, ff->txid, js)) {
+ if (EveHTTP2AddMetadata(p->flow, tx_id, js)) {
jb_close(js);
} else {
jb_restore_mark(js, &mark);
jb_set_string(js, "app_proto", AppProtoToString(p->flow->alproto));
jb_open_object(js, "fileinfo");
- EveFileInfo(js, ff, stored);
+ EveFileInfo(js, ff, tx_id, stored);
jb_close(js);
/* xff header */
* \internal
* \brief Write meta data on a single line json record
*/
-static void FileWriteJsonRecord(JsonFileLogThread *aft, const Packet *p, const File *ff,
- uint8_t dir, OutputJsonCtx *eve_ctx)
+static void FileWriteJsonRecord(JsonFileLogThread *aft, const Packet *p, const File *ff, void *tx,
+ const uint64_t tx_id, uint8_t dir, OutputJsonCtx *eve_ctx)
{
HttpXFFCfg *xff_cfg = aft->filelog_ctx->xff_cfg != NULL ?
aft->filelog_ctx->xff_cfg : aft->filelog_ctx->parent_xff_cfg;;
JsonBuilder *js = JsonBuildFileInfoRecord(
- p, ff, ff->flags & FILE_STORED ? true : false, dir, xff_cfg, eve_ctx);
+ p, ff, tx, tx_id, ff->flags & FILE_STORED ? true : false, dir, xff_cfg, eve_ctx);
if (unlikely(js == NULL)) {
return;
}
jb_free(js);
}
-static int JsonFileLogger(ThreadVars *tv, void *thread_data, const Packet *p,
- const File *ff, uint8_t dir)
+static int JsonFileLogger(ThreadVars *tv, void *thread_data, const Packet *p, const File *ff,
+ void *tx, const uint64_t tx_id, uint8_t dir)
{
SCEnter();
JsonFileLogThread *aft = (JsonFileLogThread *)thread_data;
SCLogDebug("ff %p", ff);
- FileWriteJsonRecord(aft, p, ff, dir, aft->filelog_ctx->eve_ctx);
+ FileWriteJsonRecord(aft, p, ff, tx, tx_id, dir, aft->filelog_ctx->eve_ctx);
return 0;
}
typedef struct OutputJsonCtx_ OutputJsonCtx;
void JsonFileLogRegister(void);
-JsonBuilder *JsonBuildFileInfoRecord(const Packet *p, const File *ff, const bool stored,
- uint8_t dir, HttpXFFCfg *xff_cfg, OutputJsonCtx *eve_ctx);
+JsonBuilder *JsonBuildFileInfoRecord(const Packet *p, const File *ff, void *tx,
+ const uint64_t tx_id, const bool stored, uint8_t dir, HttpXFFCfg *xff_cfg,
+ OutputJsonCtx *eve_ctx);
#endif /* __OUTPUT_JSON_FILE_H__ */
/* Default Sensor ID value */
static int64_t sensor_id = -1; /* -1 = not defined */
-void EveFileInfo(JsonBuilder *jb, const File *ff, const bool stored)
+void EveFileInfo(JsonBuilder *jb, const File *ff, const uint64_t tx_id, const bool stored)
{
jb_set_string_from_bytes(jb, "filename", ff->name, ff->name_len);
jb_set_uint(jb, "start", ff->start);
jb_set_uint(jb, "end", ff->end);
}
- jb_set_uint(jb, "tx_id", ff->txid);
+ jb_set_uint(jb, "tx_id", tx_id);
}
static void EveAddPacketVars(const Packet *p, JsonBuilder *js_vars)
json_t *SCJsonString(const char *val);
void CreateEveFlowId(JsonBuilder *js, const Flow *f);
-void EveFileInfo(JsonBuilder *js, const File *file, const bool stored);
+void EveFileInfo(JsonBuilder *js, const File *file, const uint64_t tx_id, const bool stored);
void EveTcpFlags(uint8_t flags, JsonBuilder *js);
void EvePacket(const Packet *p, JsonBuilder *js, unsigned long max_length);
JsonBuilder *CreateEveHeader(const Packet *p, enum OutputJsonLogDirection dir,
*
* NOTE p->flow is locked at this point
*/
-static int LuaFileLogger(ThreadVars *tv, void *thread_data, const Packet *p, const File *ff, uint8_t dir)
+static int LuaFileLogger(ThreadVars *tv, void *thread_data, const Packet *p, const File *ff,
+ void *tx, const uint64_t tx_id, uint8_t dir)
{
SCEnter();
LogLuaThreadCtx *td = (LogLuaThreadCtx *)thread_data;
LuaStateSetThreadVars(td->lua_ctx->luastate, tv);
LuaStateSetPacket(td->lua_ctx->luastate, (Packet *)p);
- if (p->flow && p->flow->alstate) {
- void *txptr = AppLayerParserGetTx(p->proto, p->flow->alproto, p->flow->alstate, ff->txid);
- if (txptr) {
- LuaStateSetTX(td->lua_ctx->luastate, txptr, ff->txid);
- }
- }
+ LuaStateSetTX(td->lua_ctx->luastate, tx, tx_id);
LuaStateSetFlow(td->lua_ctx->luastate, p->flow);
LuaStateSetFile(td->lua_ctx->luastate, (File *)ff);
* data for the packet loggers. */
typedef struct OutputTxLoggerThreadData_ {
OutputLoggerThreadStore *store[ALPROTO_MAX];
+
+ /* thread local data from file api */
+ OutputFileLoggerThreadData *file;
+ /* thread local data from filedata api */
+ OutputFiledataLoggerThreadData *filedata;
} OutputTxLoggerThreadData;
/* logger instance, a module + a output ctx,
void (*ThreadExitPrintStats)(ThreadVars *, void *))
{
if (alproto != ALPROTO_UNKNOWN && !(AppLayerParserIsEnabled(alproto))) {
- SCLogNotice("%s logger not enabled: protocol %s is disabled",
- name, AppProtoToString(alproto));
+ SCLogDebug(
+ "%s logger not enabled: protocol %s is disabled", name, AppProtoToString(alproto));
return -1;
}
OutputTxLogger *op = SCMalloc(sizeof(*op));
return 0;
}
+extern bool g_file_logger_enabled;
+extern bool g_filedata_logger_enabled;
+
+/** \brief run the per tx file logging
+ * \todo clean up various end of tx/stream etc indicators
+ */
+static inline void OutputTxLogFiles(ThreadVars *tv, OutputFileLoggerThreadData *file_td,
+ OutputFiledataLoggerThreadData *filedata_td, Packet *p, Flow *f, void *tx,
+ const uint64_t tx_id, AppLayerTxData *txd, const bool tx_complete, const bool ts_ready,
+ const bool tc_ready, const bool ts_eof, const bool tc_eof, const bool eof)
+{
+ int packet_dir;
+ int opposing_dir;
+ bool packet_dir_ready;
+ const bool opposing_dir_ready = eof;
+ bool opposing_tx_ready;
+ if (p->flowflags & FLOW_PKT_TOSERVER) {
+ packet_dir = STREAM_TOSERVER;
+ opposing_dir = STREAM_TOCLIENT;
+ packet_dir_ready = eof | ts_ready | ts_eof;
+ opposing_tx_ready = tc_ready;
+ } else if (p->flowflags & FLOW_PKT_TOCLIENT) {
+ packet_dir = STREAM_TOCLIENT;
+ opposing_dir = STREAM_TOSERVER;
+ packet_dir_ready = eof | tc_ready | tc_eof;
+ opposing_tx_ready = ts_ready;
+ } else {
+ abort();
+ }
+
+ SCLogDebug("eof %d ts_ready %d ts_eof %d", eof, ts_ready, ts_eof);
+ SCLogDebug("eof %d tc_ready %d tc_eof %d", eof, tc_ready, tc_eof);
+
+ SCLogDebug("packet dir %s opposing %s packet_dir_ready %d opposing_dir_ready %d",
+ packet_dir == STREAM_TOSERVER ? "TOSERVER" : "TOCLIENT",
+ opposing_dir == STREAM_TOSERVER ? "TOSERVER" : "TOCLIENT", packet_dir_ready,
+ opposing_dir_ready);
+
+ FileContainer *ffc = AppLayerParserGetTxFiles(f, tx, packet_dir);
+ FileContainer *ffc_opposing = AppLayerParserGetTxFiles(f, tx, opposing_dir);
+
+ /* see if opposing side is finished: if no file support in this direction, of is not
+ * files and tx is done for opposing dir. */
+ bool opposing_finished =
+ ffc_opposing == NULL || (ffc_opposing->head == NULL && opposing_tx_ready);
+ SCLogDebug("opposing_finished %d ffc_opposing %p ffc_opposing->head %p opposing_tx_ready %d",
+ opposing_finished, ffc_opposing, ffc_opposing ? ffc_opposing->head : NULL,
+ opposing_tx_ready);
+
+ if (ffc || ffc_opposing)
+ SCLogDebug("pcap_cnt %" PRIu64 " flow %p tx %p tx_id %" PRIu64
+ " ffc %p ffc_opposing %p tx_complete %d",
+ p->pcap_cnt, f, tx, tx_id, ffc, ffc_opposing, tx_complete);
+
+ if (ffc) {
+ const bool file_close = ((p->flags & PKT_PSEUDO_STREAM_END)) | eof;
+ const bool file_trunc = StreamTcpReassembleDepthReached(p) | eof;
+ SCLogDebug("tx: calling files: ffc %p head %p file_close %d file_trunc %d", ffc, ffc->head,
+ file_close, file_trunc);
+ if (filedata_td)
+ OutputFiledataLogFfc(tv, filedata_td, p, ffc, tx, tx_id, txd, packet_dir, file_close,
+ file_trunc, packet_dir);
+ if (file_td)
+ OutputFileLogFfc(
+ tv, file_td, p, ffc, tx, tx_id, txd, file_close, file_trunc, packet_dir);
+ }
+ /* if EOF and we support files, do a final write out */
+ if (opposing_dir_ready && ffc_opposing != NULL) {
+ const bool file_close = ((p->flags & PKT_PSEUDO_STREAM_END)) | tx_complete | eof;
+ const bool file_trunc = StreamTcpReassembleDepthReached(p) | eof;
+ opposing_finished = true;
+ SCLogDebug("tx: calling for opposing direction files: file_close:%s file_trunc:%s",
+ file_close ? "true" : "false", file_trunc ? "true" : "false");
+ if (filedata_td)
+ OutputFiledataLogFfc(tv, filedata_td, p, ffc_opposing, tx, tx_id, txd, opposing_dir,
+ file_close, file_trunc, opposing_dir);
+ if (file_td)
+ OutputFileLogFfc(tv, file_td, p, ffc_opposing, tx, tx_id, txd, file_close, file_trunc,
+ opposing_dir);
+ }
+
+ const bool tx_done = packet_dir_ready && opposing_finished;
+ SCLogDebug("tx_done %d packet_dir_ready %d opposing_finished %d", tx_done, packet_dir_ready,
+ opposing_finished);
+
+ /* if not a file tx or if tx is done, set logger flags so tx can move on */
+ const bool is_file_tx = (ffc != NULL || ffc_opposing != NULL);
+ if (!is_file_tx || tx_done) {
+ SCLogDebug("is_file_tx %d tx_done %d", is_file_tx, tx_done);
+ if (file_td) {
+ txd->logged.flags |= BIT_U32(LOGGER_FILE);
+ SCLogDebug("setting LOGGER_FILE => %08x", txd->logged.flags);
+ }
+ if (filedata_td) {
+ txd->logged.flags |= BIT_U32(LOGGER_FILEDATA);
+ SCLogDebug("setting LOGGER_FILEDATA => %08x", txd->logged.flags);
+ }
+ } else {
+ SCLogDebug("pcap_cnt %" PRIu64 " flow %p tx %p tx_id %" PRIu64
+ " NOT SETTING FILE FLAGS ffc %p ffc_opposing %p tx_complete %d",
+ p->pcap_cnt, f, tx, tx_id, ffc, ffc_opposing, tx_complete);
+ }
+}
+
static void OutputTxLogList0(ThreadVars *tv, OutputTxLoggerThreadData *op_thread_data, Packet *p,
Flow *f, void *tx, const uint64_t tx_id)
{
}
}
+struct Ctx {
+ uint32_t tx_logged_old;
+ uint32_t tx_logged;
+};
+
+static void OutputTxLogCallLoggers(ThreadVars *tv, OutputTxLoggerThreadData *op_thread_data,
+ const OutputTxLogger *logger, const OutputLoggerThreadStore *store, Packet *p, Flow *f,
+ void *alstate, void *tx, const uint64_t tx_id, const AppProto alproto, const bool eof,
+ const int tx_progress_ts, const int tx_progress_tc, struct Ctx *ctx)
+{
+ DEBUG_VALIDATE_BUG_ON(logger == NULL && store != NULL);
+ DEBUG_VALIDATE_BUG_ON(logger != NULL && store == NULL);
+ // DEBUG_VALIDATE_BUG_ON(logger == NULL && store == NULL);
+
+ while (logger && store) {
+ DEBUG_VALIDATE_BUG_ON(logger->LogFunc == NULL);
+ DEBUG_VALIDATE_BUG_ON(logger->alproto != alproto);
+
+ SCLogDebug("logger %p, Alproto %d LogCondition %p, ts_log_progress %d "
+ "tc_log_progress %d",
+ logger, logger->alproto, logger->LogCondition, logger->ts_log_progress,
+ logger->tc_log_progress);
+ if ((ctx->tx_logged_old & BIT_U32(logger->logger_id)) == 0) {
+ SCLogDebug("alproto match %d, logging tx_id %" PRIu64, logger->alproto, tx_id);
+
+ SCLogDebug("pcap_cnt %" PRIu64 ", tx_id %" PRIu64 " logger %d. EOF %s", p->pcap_cnt,
+ tx_id, logger->logger_id, eof ? "true" : "false");
+
+ if (eof) {
+ SCLogDebug("EOF, so log now");
+ } else {
+ if (logger->LogCondition) {
+ int r = logger->LogCondition(tv, p, alstate, tx, tx_id);
+ if (r == FALSE) {
+ SCLogDebug("conditions not met, not logging");
+ goto next_logger;
+ }
+ } else {
+ if (tx_progress_tc < logger->tc_log_progress) {
+ SCLogDebug("progress not far enough, not logging");
+ goto next_logger;
+ }
+
+ if (tx_progress_ts < logger->ts_log_progress) {
+ SCLogDebug("progress not far enough, not logging");
+ goto next_logger;
+ }
+ }
+ }
+
+ SCLogDebug("Logging tx_id %" PRIu64 " to logger %d", tx_id, logger->logger_id);
+ PACKET_PROFILING_LOGGER_START(p, logger->logger_id);
+ logger->LogFunc(tv, store->thread_data, p, f, alstate, tx, tx_id);
+ PACKET_PROFILING_LOGGER_END(p, logger->logger_id);
+
+ ctx->tx_logged |= BIT_U32(logger->logger_id);
+ }
+
+ next_logger:
+ logger = logger->next;
+ store = store->next;
+
+ DEBUG_VALIDATE_BUG_ON(logger == NULL && store != NULL);
+ DEBUG_VALIDATE_BUG_ON(logger != NULL && store == NULL);
+ }
+}
+
static TmEcode OutputTxLog(ThreadVars *tv, Packet *p, void *thread_data)
{
DEBUG_VALIDATE_BUG_ON(thread_data == NULL);
Flow * const f = p->flow;
const uint8_t ipproto = f->proto;
const AppProto alproto = f->alproto;
-
- if (list[alproto] == NULL && list[ALPROTO_UNKNOWN] == NULL) {
- /* No child loggers registered. */
- return TM_ECODE_OK;
+ SCLogDebug("pcap_cnt %u tx logging %u/%s", (uint32_t)p->pcap_cnt, alproto,
+ AppProtoToString(alproto));
+
+ if (op_thread_data->file == NULL && op_thread_data->filedata == NULL) {
+ if (list[alproto] == NULL && list[ALPROTO_UNKNOWN] == NULL) {
+ SCLogDebug("bail");
+ /* No child loggers registered. */
+ return TM_ECODE_OK;
+ }
+ if (AppLayerParserProtocolHasLogger(p->proto, alproto) == 0)
+ goto end;
}
-
- if (AppLayerParserProtocolHasLogger(p->proto, alproto) == 0)
- goto end;
const LoggerId logger_expectation = AppLayerParserProtocolGetLoggerBits(p->proto, alproto);
- if (logger_expectation == 0)
+ if (logger_expectation == 0) {
+ SCLogDebug("bail: logger_expectation %u. LOGGER_FILE %u LOGGER_FILEDATA %u",
+ logger_expectation, LOGGER_FILE, LOGGER_FILEDATA);
goto end;
-
+ }
void *alstate = f->alstate;
if (alstate == NULL) {
SCLogDebug("no alstate");
goto end;
}
+ SCLogDebug("pcap_cnt %" PRIu64, p->pcap_cnt);
+
const bool last_pseudo = (p->flowflags & FLOW_PKT_LAST_PSEUDO) != 0;
- const bool ts_eof = AppLayerParserStateIssetFlag(f->alparser, APP_LAYER_PARSER_EOF_TS) != 0;
- const bool tc_eof = AppLayerParserStateIssetFlag(f->alparser, APP_LAYER_PARSER_EOF_TC) != 0;
+ const bool ts_eof = AppLayerParserStateIssetFlag(f->alparser,
+ (APP_LAYER_PARSER_EOF_TS | APP_LAYER_PARSER_TRUNC_TS)) != 0;
+ const bool tc_eof = AppLayerParserStateIssetFlag(f->alparser,
+ (APP_LAYER_PARSER_EOF_TC | APP_LAYER_PARSER_TRUNC_TC)) != 0;
+
+ const bool eof = last_pseudo || (ts_eof && tc_eof);
+ SCLogDebug("eof %d last_pseudo %d ts_eof %d tc_eof %d", eof, last_pseudo, ts_eof, tc_eof);
+
const uint8_t ts_disrupt_flags = FlowGetDisruptionFlags(f, STREAM_TOSERVER);
const uint8_t tc_disrupt_flags = FlowGetDisruptionFlags(f, STREAM_TOCLIENT);
+ SCLogDebug("ts_disrupt_flags %02x tc_disrupt_flags %02x", ts_disrupt_flags, tc_disrupt_flags);
const uint64_t total_txs = AppLayerParserGetTxCnt(f, alstate);
uint64_t tx_id = AppLayerParserGetTransactionLogId(f->alparser);
uint64_t max_id = tx_id;
int logged = 0;
int gap = 0;
+ SCLogDebug("tx_id %" PRIu64 " total_txs %" PRIu64, tx_id, total_txs);
+
AppLayerGetTxIteratorFunc IterFunc = AppLayerGetTxIterator(ipproto, alproto);
AppLayerGetTxIterState state;
memset(&state, 0, sizeof(state));
break;
void * const tx = ires.tx_ptr;
tx_id = ires.tx_id;
+ SCLogDebug("STARTING tx_id %" PRIu64 ", tx %p", tx_id, tx);
+
+ const int tx_progress_ts =
+ AppLayerParserGetStateProgress(p->proto, alproto, tx, ts_disrupt_flags);
+ const int tx_progress_tc =
+ AppLayerParserGetStateProgress(p->proto, alproto, tx, tc_disrupt_flags);
+ const bool tx_complete = (tx_progress_ts == AppLayerParserGetStateProgressCompletionStatus(
+ alproto, STREAM_TOSERVER) &&
+ tx_progress_tc == AppLayerParserGetStateProgressCompletionStatus(
+ alproto, STREAM_TOCLIENT));
+ const bool ts_ready = tx_progress_ts == AppLayerParserGetStateProgressCompletionStatus(
+ alproto, STREAM_TOSERVER);
+ const bool tc_ready = tx_progress_tc == AppLayerParserGetStateProgressCompletionStatus(
+ alproto, STREAM_TOCLIENT);
+ SCLogDebug("ts_ready %d tc_ready %d", ts_ready, tc_ready);
+
+ SCLogDebug("file_thread_data %p filedata_thread_data %p", op_thread_data->file,
+ op_thread_data->filedata);
AppLayerTxData *txd = AppLayerParserGetTxData(ipproto, alproto, tx);
+ if (txd && (op_thread_data->file || op_thread_data->filedata) &&
+ AppLayerParserSupportsFiles(p->proto, alproto)) {
+ OutputTxLogFiles(tv, op_thread_data->file, op_thread_data->filedata, p, f, tx, tx_id,
+ txd, tx_complete, ts_ready, tc_ready, ts_eof, tc_eof, eof);
+ }
+ if (txd)
+ SCLogDebug("logger: expect %08x, have %08x", logger_expectation, txd->logged.flags);
+
if (txd == NULL) {
/* make sure this tx, which can't be properly logged is skipped */
logged = 1;
goto next_tx;
}
- SCLogDebug("tx %p/%"PRIu64" txd %p: log_flags %x", tx, tx_id, txd, txd->config.log_flags);
+ SCLogDebug("tx %p/%" PRIu64 " txd %p: log_flags %x logger_expectation %x", tx, tx_id, txd,
+ txd->config.log_flags, logger_expectation);
if (txd->config.log_flags & BIT_U8(CONFIG_TYPE_TX)) {
SCLogDebug("SKIP tx %p/%"PRIu64, tx, tx_id);
goto next_tx;
}
- LoggerId tx_logged = txd->logged.flags;
- const LoggerId tx_logged_old = tx_logged;
- SCLogDebug("logger: expect %08x, have %08x", logger_expectation, tx_logged);
- if (tx_logged == logger_expectation) {
+ if (txd->logged.flags == logger_expectation) {
+ SCLogDebug("fully logged");
/* tx already fully logged */
goto next_tx;
}
- const int tx_progress_ts =
- AppLayerParserGetStateProgress(p->proto, alproto, tx, ts_disrupt_flags);
- const int tx_progress_tc =
- AppLayerParserGetStateProgress(p->proto, alproto, tx, tc_disrupt_flags);
- SCLogDebug("tx_progress_ts %d tx_progress_tc %d",
- tx_progress_ts, tx_progress_tc);
-
+ SCLogDebug("logger: expect %08x, have %08x", logger_expectation, txd->logged.flags);
const OutputTxLogger *logger = list[alproto];
const OutputLoggerThreadStore *store = op_thread_data->store[alproto];
+ struct Ctx ctx = { .tx_logged = txd->logged.flags, .tx_logged_old = txd->logged.flags };
+ SCLogDebug("logger: expect %08x, have %08x", logger_expectation, ctx.tx_logged);
- DEBUG_VALIDATE_BUG_ON(logger == NULL && store != NULL);
- DEBUG_VALIDATE_BUG_ON(logger != NULL && store == NULL);
- DEBUG_VALIDATE_BUG_ON(logger == NULL && store == NULL);
+ OutputTxLogCallLoggers(tv, op_thread_data, logger, store, p, f, alstate, tx, tx_id, alproto,
+ eof, tx_progress_ts, tx_progress_tc, &ctx);
- while (logger && store) {
- DEBUG_VALIDATE_BUG_ON(logger->LogFunc == NULL);
- DEBUG_VALIDATE_BUG_ON(logger->alproto != alproto);
-
- SCLogDebug("logger %p, Alproto %d LogCondition %p, ts_log_progress %d "
- "tc_log_progress %d", logger, logger->alproto, logger->LogCondition,
- logger->ts_log_progress, logger->tc_log_progress);
- if ((tx_logged_old & BIT_U32(logger->logger_id)) == 0) {
- SCLogDebug("alproto match %d, logging tx_id %"PRIu64, logger->alproto, tx_id);
-
- SCLogDebug("pcap_cnt %"PRIu64", tx_id %"PRIu64" logger %d. "
- "EOFs TS %s TC %s LAST PSEUDO %s",
- p->pcap_cnt, tx_id, logger->logger_id,
- ts_eof ? "true" : "false", tc_eof ? "true" : "false",
- last_pseudo ? "true" : "false");
-
- if ((ts_eof && tc_eof) || last_pseudo) {
- SCLogDebug("EOF, so log now");
- } else {
- if (logger->LogCondition) {
- int r = logger->LogCondition(tv, p, alstate, tx, tx_id);
- if (r == FALSE) {
- SCLogDebug("conditions not met, not logging");
- goto next_logger;
- }
- } else {
- if (tx_progress_tc < logger->tc_log_progress) {
- SCLogDebug("progress not far enough, not logging");
- goto next_logger;
- }
-
- if (tx_progress_ts < logger->ts_log_progress) {
- SCLogDebug("progress not far enough, not logging");
- goto next_logger;
- }
- }
- }
-
- SCLogDebug("Logging tx_id %"PRIu64" to logger %d", tx_id, logger->logger_id);
- PACKET_PROFILING_LOGGER_START(p, logger->logger_id);
- logger->LogFunc(tv, store->thread_data, p, f, alstate, tx, tx_id);
- PACKET_PROFILING_LOGGER_END(p, logger->logger_id);
-
- tx_logged |= BIT_U32(logger->logger_id);
- }
-
-next_logger:
- logger = logger->next;
- store = store->next;
-
- DEBUG_VALIDATE_BUG_ON(logger == NULL && store != NULL);
- DEBUG_VALIDATE_BUG_ON(logger != NULL && store == NULL);
- }
-
- if (tx_logged != tx_logged_old) {
- SCLogDebug("logger: storing %08x (was %08x)",
- tx_logged, tx_logged_old);
+ SCLogDebug("logger: expect %08x, have %08x", logger_expectation, ctx.tx_logged);
+ if (ctx.tx_logged != ctx.tx_logged_old) {
+ SCLogDebug("logger: storing %08x (was %08x)", ctx.tx_logged, ctx.tx_logged_old);
DEBUG_VALIDATE_BUG_ON(txd == NULL);
- txd->logged.flags |= tx_logged;
+ txd->logged.flags |= ctx.tx_logged;
}
/* If all loggers logged set a flag and update the last tx_id
* If not all loggers were logged we flag that there was a gap
* so any subsequent transactions in this loop don't increase
* the maximum ID that was logged. */
- if (!gap && tx_logged == logger_expectation) {
+ if (!gap && ctx.tx_logged == logger_expectation) {
+ SCLogDebug("no gap %d, %08x == %08x", gap, ctx.tx_logged, logger_expectation);
logged = 1;
max_id = tx_id;
+ SCLogDebug("max_id %" PRIu64, max_id);
} else {
gap = 1;
}
/** \brief thread init for the tx logger
* This will run the thread init functions for the individual registered
* loggers */
-static TmEcode OutputTxLogThreadInit(ThreadVars *tv, const void *initdata, void **data)
+static TmEcode OutputTxLogThreadInit(ThreadVars *tv, const void *_initdata, void **data)
{
OutputTxLoggerThreadData *td = SCMalloc(sizeof(*td));
if (td == NULL)
logger = logger->next;
}
}
+
+ if (g_file_logger_enabled) {
+ if (OutputFileLogThreadInit(tv, &td->file) != TM_ECODE_OK) {
+ FatalError(SC_ERR_FATAL, "failed to set up file thread data");
+ }
+ }
+ if (g_filedata_logger_enabled) {
+ if (OutputFiledataLogThreadInit(tv, &td->filedata) != TM_ECODE_OK) {
+ FatalError(SC_ERR_FATAL, "failed to set up filedata thread data");
+ }
+ }
+
+ SCLogDebug("file_thread_data %p filedata_thread_data %p", td->file, td->filedata);
+
return TM_ECODE_OK;
}
}
}
+ if (op_thread_data->file) {
+ OutputFileLogThreadDeinit(tv, op_thread_data->file);
+ }
+ if (op_thread_data->filedata) {
+ OutputFiledataLogThreadDeinit(tv, op_thread_data->filedata);
+ }
+
SCFree(op_thread_data);
return TM_ECODE_OK;
}
cnt++;
}
}
+
+ if (g_file_logger_enabled) {
+ cnt++;
+ SCLogDebug("g_file_logger_enabled");
+ }
+ if (g_filedata_logger_enabled) {
+ cnt++;
+ SCLogDebug("g_filedata_logger_enabled");
+ }
+
return cnt;
}
logger = TAILQ_NEXT(logger, entries);
thread_store_node = TAILQ_NEXT(thread_store_node, entries);
}
-
return TM_ECODE_OK;
}
void OutputRegisterRootLoggers(void)
{
OutputPacketLoggerRegister();
- OutputTxLoggerRegister();
OutputFiledataLoggerRegister();
OutputFileLoggerRegister();
+ OutputTxLoggerRegister();
OutputStreamingLoggerRegister();
}
#define DEFAULT_LOG_MODE_APPEND "yes"
#define DEFAULT_LOG_FILETYPE "regular"
+typedef struct OutputLoggerThreadStore_ {
+ void *thread_data;
+ struct OutputLoggerThreadStore_ *next;
+} OutputLoggerThreadStore;
+
#include "output-packet.h"
#include "output-tx.h"
#include "output-file.h"
#include "output-streaming.h"
#include "output-stats.h"
-
-typedef struct OutputLoggerThreadStore_ {
- void *thread_data;
- struct OutputLoggerThreadStore_ *next;
-} OutputLoggerThreadStore;
-
typedef struct OutputInitResult_ {
OutputCtx *ctx;
bool ok;
module->ThreadExitPrintStats);
/* Not used with wild card loggers */
if (module->alproto != ALPROTO_UNKNOWN) {
- logger_bits[module->alproto] |= (1<<module->logger_id);
+ logger_bits[module->alproto] |= BIT_U32(module->logger_id);
}
} else if (module->FiledataLogFunc) {
SCLogDebug("%s is a filedata logger", module->name);
}
}
+extern bool g_file_logger_enabled;
+extern bool g_filedata_logger_enabled;
+
/**
* Initialize the output modules.
*/
/* register the logger bits to the app-layer */
AppProto a;
for (a = 0; a < ALPROTO_MAX; a++) {
+ if (AppLayerParserSupportsFiles(IPPROTO_TCP, a)) {
+ if (g_file_logger_enabled)
+ logger_bits[a] |= BIT_U32(LOGGER_FILE);
+ if (g_filedata_logger_enabled)
+ logger_bits[a] |= BIT_U32(LOGGER_FILEDATA);
+ SCLogDebug("IPPROTO_TCP::%s: g_file_logger_enabled %d g_filedata_logger_enabled %d -> "
+ "%08x",
+ AppProtoToString(a), g_file_logger_enabled, g_filedata_logger_enabled,
+ logger_bits[a]);
+ }
+ if (AppLayerParserSupportsFiles(IPPROTO_UDP, a)) {
+ if (g_file_logger_enabled)
+ logger_bits[a] |= BIT_U32(LOGGER_FILE);
+ if (g_filedata_logger_enabled)
+ logger_bits[a] |= BIT_U32(LOGGER_FILEDATA);
+ }
+
if (logger_bits[a] == 0)
continue;
- const int tcp = AppLayerParserProtocolHasLogger(IPPROTO_TCP, a);
- const int udp = AppLayerParserProtocolHasLogger(IPPROTO_UDP, a);
+ const int tcp = AppLayerParserProtocolHasLogger(IPPROTO_TCP, a) | (g_file_logger_enabled) |
+ (g_filedata_logger_enabled);
+ const int udp = AppLayerParserProtocolHasLogger(IPPROTO_UDP, a) | (g_file_logger_enabled) |
+ (g_filedata_logger_enabled);
+ SCLogDebug("tcp %d udp %d", tcp, udp);
SCLogDebug("logger for %s: %s %s", AppProtoToString(a),
tcp ? "true" : "false", udp ? "true" : "false");
FileAppendGAPById,
FileContainerRecycle,
FilePrune,
- FileContainerSetTx,
AppLayerRegisterParser,
};
const uint8_t *data, uint32_t data_len);
void (*FileContainerRecycle)(FileContainer *ffc);
void (*FilePrune)(FileContainer *ffc);
- void (*FileSetTx)(FileContainer *, uint64_t);
int (*AppLayerRegisterParser)(const struct AppLayerParser *p, AppProto alproto);
LOGGER_JSON_DCERPC,
LOGGER_JSON_HTTP2,
+ LOGGER_FILE,
+ LOGGER_FILEDATA,
+
/** \warning when we exceed what we can express as a u32 flag here we need to update
* LoggerFlags::flags (u32) and `tx_logged` in src/output-tx.c */
return FileFlowFlagsToFlags(flow->file_flags, direction);
}
+void FileApplyTxFlags(const AppLayerTxData *txd, const uint8_t direction, File *file)
+{
+ SCLogDebug("file flags %04x STORE %s NOSTORE %s", file->flags,
+ (file->flags & FILE_STORE) ? "true" : "false",
+ (file->flags & FILE_NOSTORE) ? "true" : "false");
+ uint16_t update_flags = FileFlowFlagsToFlags(txd->file_flags, direction);
+ DEBUG_VALIDATE_BUG_ON(
+ (file->flags & (FILE_STORE | FILE_NOSTORE)) == (FILE_STORE | FILE_NOSTORE));
+ if (file->flags & FILE_STORE)
+ update_flags &= ~FILE_NOSTORE;
+
+ file->flags |= update_flags;
+ SCLogDebug("file flags %04x STORE %s NOSTORE %s", file->flags,
+ (file->flags & FILE_STORE) ? "true" : "false",
+ (file->flags & FILE_NOSTORE) ? "true" : "false");
+ DEBUG_VALIDATE_BUG_ON(
+ (file->flags & (FILE_STORE | FILE_NOSTORE)) == (FILE_STORE | FILE_NOSTORE));
+}
+
static int FileMagicSize(void)
{
/** \todo make this size configurable */
SCReturnInt(0);
}
-/**
- * \brief Set the TX id for a file
- *
- * \param ff The file to store
- * \param txid the tx id
- */
-int FileSetTx(File *ff, uint64_t txid)
-{
- SCLogDebug("ff %p txid %"PRIu64, ff, txid);
- if (ff != NULL)
- ff->txid = txid;
- SCReturnInt(0);
-}
-
-void FileContainerSetTx(FileContainer *ffc, uint64_t tx_id)
-{
- if (ffc && ffc->tail) {
- (void)FileSetTx(ffc->tail, tx_id);
- }
-}
-
/**
* \brief check if we have stored enough
*
f->file_flags, set_file_flags, g_file_flow_mask);
if (set_file_flags != 0 && f->alproto != ALPROTO_UNKNOWN && f->alstate != NULL) {
- uint16_t per_file_flags = 0;
-#ifdef HAVE_MAGIC
- if (set_file_flags & (FLOWFILE_NO_MAGIC_TS|FLOWFILE_NO_MAGIC_TC))
- per_file_flags |= FILE_NOMAGIC;
-#endif
- if (set_file_flags & (FLOWFILE_NO_MD5_TS|FLOWFILE_NO_MD5_TC))
- per_file_flags |= FILE_NOMD5;
- if (set_file_flags & (FLOWFILE_NO_SHA1_TS|FLOWFILE_NO_SHA1_TC))
- per_file_flags |= FILE_NOSHA1;
- if (set_file_flags & (FLOWFILE_NO_SHA256_TS|FLOWFILE_NO_SHA256_TC))
- per_file_flags |= FILE_NOSHA256;
- if (set_file_flags & (FLOWFILE_NO_SIZE_TS|FLOWFILE_NO_SIZE_TC))
- per_file_flags |= FILE_NOTRACK;
- if (set_file_flags & (FLOWFILE_NO_STORE_TS|FLOWFILE_NO_STORE_TC))
- per_file_flags |= FILE_NOSTORE;
-
- FileContainer *ffc = AppLayerParserGetFiles(f, direction);
- if (ffc != NULL) {
- for (File *ptr = ffc->head; ptr != NULL; ptr = ptr->next) {
- ptr->flags |= per_file_flags;
-
- /* destroy any ctx we may have so far */
- if ((per_file_flags & FILE_NOSHA256) &&
- ptr->sha256_ctx != NULL)
- {
- SCSha256Free(ptr->sha256_ctx);
- ptr->sha256_ctx = NULL;
- }
- if ((per_file_flags & FILE_NOSHA1) &&
- ptr->sha1_ctx != NULL)
- {
- SCSha1Free(ptr->sha1_ctx);
- ptr->sha1_ctx = NULL;
- }
- if ((per_file_flags & FILE_NOMD5) &&
- ptr->md5_ctx != NULL)
- {
- SCMd5Free(ptr->md5_ctx);
- ptr->md5_ctx = NULL;
- }
+ AppLayerStateData *sd = AppLayerParserGetStateData(f->proto, f->alproto, f->alstate);
+ if (sd != NULL) {
+ if ((sd->file_flags & f->file_flags) != f->file_flags) {
+ SCLogDebug("state data: updating file_flags %04x with flow file_flags %04x",
+ sd->file_flags, f->file_flags);
+ sd->file_flags |= f->file_flags;
}
}
}
}
-
-
-/**
- * \brief set no store flag, close file if needed
- *
- * \param ff file
- */
-static void FileDisableStoringForFile(File *ff)
-{
- SCEnter();
-
- if (ff == NULL) {
- SCReturn;
- }
-
- SCLogDebug("not storing this file");
- ff->flags |= FILE_NOSTORE;
-
- if (ff->state == FILE_STATE_OPENED && FileDataSize(ff) >= (uint64_t)FileMagicSize()) {
- if (g_file_force_md5 == 0 && g_file_force_sha1 == 0 && g_file_force_sha256 == 0
- && g_file_force_tracking == 0) {
- (void)FileCloseFilePtr(ff, NULL, 0,
- (FILE_TRUNCATED|FILE_NOSTORE));
- }
- }
-}
-
/**
* \brief disable file storing for files in a transaction
*
* \param direction flow direction
* \param tx_id transaction id
*/
-void FileDisableStoringForTransaction(Flow *f, uint8_t direction, uint64_t tx_id)
+void FileDisableStoringForTransaction(Flow *f, const uint8_t direction, void *tx, uint64_t tx_id)
{
- File *ptr = NULL;
-
- DEBUG_ASSERT_FLOW_LOCKED(f);
-
- SCEnter();
-
- FileContainer *ffc = AppLayerParserGetFiles(f, direction);
- if (ffc != NULL) {
- for (ptr = ffc->head; ptr != NULL; ptr = ptr->next) {
- if (ptr->txid == tx_id) {
- if (ptr->flags & FILE_STORE) {
- /* weird, already storing -- let it continue*/
- SCLogDebug("file is already being stored");
- } else {
- FileDisableStoringForFile(ptr);
- }
- }
+ AppLayerTxData *txd = AppLayerParserGetTxData(f->proto, f->alproto, tx);
+ if (txd != NULL) {
+ if (direction & STREAM_TOSERVER) {
+ txd->file_flags |= FLOWFILE_NO_STORE_TS;
+ } else {
+ txd->file_flags |= FLOWFILE_NO_STORE_TC;
}
}
-
- SCReturn;
}
/**
void FileStoreAllFilesForTx(FileContainer *fc, uint64_t tx_id)
{
- File *ptr = NULL;
-
- SCEnter();
-
- if (fc != NULL) {
- for (ptr = fc->head; ptr != NULL; ptr = ptr->next) {
- if (ptr->txid == tx_id) {
- FileStore(ptr);
- }
- }
- }
+ abort();
}
void FileStoreAllFiles(FileContainer *fc)
-/* Copyright (C) 2007-2011 Open Information Security Foundation
+/* Copyright (C) 2007-2021 Open Information Security Foundation
*
* You can copy, redistribute or modify this Program under the terms of
* the GNU General Public License version 2 as published by the Free
#define __UTIL_FILE_H__
#include "conf.h"
-
#include "util-streaming-buffer.h"
/* Hack: Pulling rust.h to get the SCSha256 causes all sorts of problems with
uint16_t name_len;
FileState state;
StreamingBuffer *sb;
- uint64_t txid; /**< tx this file is part of */
uint32_t file_track_id; /**< id used by protocol parser */
uint32_t file_store_id; /**< id used in store file name file.<id> */
int fd; /**< file descriptor for filestore, not
*/
int FileStore(File *);
-/**
- * \brief Set the TX id for a file
- *
- * \param ff The file to store
- * \param txid the tx id
- */
-int FileSetTx(File *, uint64_t txid);
-void FileContainerSetTx(FileContainer *ffc, uint64_t tx_id);
-
/**
* \brief disable file storing for a transaction
*
* \param f flow
+ * \param direction STREAM_TOSERVER or STREAM_TOCLIENT
+ * \param tx transaction pointer
* \param tx_id transaction id
*/
-void FileDisableStoringForTransaction(Flow *f, uint8_t direction, uint64_t tx_id);
+void FileDisableStoringForTransaction(Flow *f, const uint8_t direction, void *tx, uint64_t tx_id);
void FlowFileDisableStoringForTransaction(struct Flow_ *f, uint64_t tx_id);
void FilePrune(FileContainer *ffc);
CASE_CODE (LOGGER_JSON_STATS);
CASE_CODE (LOGGER_PCAP);
CASE_CODE (LOGGER_JSON_METADATA);
+
+ CASE_CODE(LOGGER_FILE);
+ CASE_CODE(LOGGER_FILEDATA);
+
case LOGGER_SIZE:
return "UNKNOWN";
}
projects / thirdparty / suricata.git / commitdiff
