From: Jason Ish Date: Fri, 31 Aug 2018 05:20:21 +0000 (-0600) Subject: rust: app-layer template parser and logger X-Git-Tag: suricata-4.1.0-rc2~40 X-Git-Url: http://git.ipfire.org/gitweb.cgi?a=commitdiff_plain;h=c3f1a35e28f5c7c06b595a95efb3b7c1554151e0;p=thirdparty%2Fsuricata.git rust: app-layer template parser and logger The protocol is a simple request/reply based protocol that can be hand driven with netcat. Request -> 12:Hello World! Response -> 3:Byte Its of the format : where length is the length of the message, not including the length or the delimiter. --- diff --git a/rust/src/applayertemplate/logger.rs b/rust/src/applayertemplate/logger.rs new file mode 100644 index 0000000000..19c429723c --- /dev/null +++ b/rust/src/applayertemplate/logger.rs @@ -0,0 +1,41 @@ +/* Copyright (C) 2018 Open Information Security Foundation + * + * You can copy, redistribute or modify this Program under the terms of + * the GNU General Public License version 2 as published by the Free + * Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * version 2 along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +use libc; +use std; +use json::*; +use super::template::TemplateTransaction; + +fn log_template(tx: &TemplateTransaction) -> Option { + let js = Json::object(); + if let Some(ref request) = tx.request { + js.set_string("request", request); + } + if let Some(ref response) = tx.response { + js.set_string("response", response); + } + return Some(js); +} + +#[no_mangle] +pub extern "C" fn rs_template_logger_log(tx: *mut libc::c_void) -> *mut JsonT { + let tx = cast_pointer!(tx, TemplateTransaction); + match log_template(tx) { + Some(js) => js.unwrap(), + None => std::ptr::null_mut(), + } +} diff --git a/rust/src/applayertemplate/mod.rs b/rust/src/applayertemplate/mod.rs new file mode 100644 index 0000000000..d78388f720 --- /dev/null +++ b/rust/src/applayertemplate/mod.rs @@ -0,0 +1,22 @@ +/* Copyright (C) 2018 Open Information Security Foundation + * + * You can copy, redistribute or modify this Program under the terms of + * the GNU General Public License version 2 as published by the Free + * Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * version 2 along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +pub mod template; +mod parser; +/* TEMPLATE_START_REMOVE */ +pub mod logger; +/* TEMPLATE_END_REMOVE */ diff --git a/rust/src/applayertemplate/parser.rs b/rust/src/applayertemplate/parser.rs new file mode 100644 index 0000000000..acc8bd8416 --- /dev/null +++ b/rust/src/applayertemplate/parser.rs @@ -0,0 +1,64 @@ +/* Copyright (C) 2018 Open Information Security Foundation + * + * You can copy, redistribute or modify this Program under the terms of + * the GNU General Public License version 2 as published by the Free + * Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * version 2 along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +use std; + +fn parse_len(input: &str) -> Result { + input.parse::() +} + +named!(pub parse_message, + do_parse!( + len: map_res!( + map_res!(take_until_s!(":"), std::str::from_utf8), parse_len) >> + sep: take!(1) >> + msg: take_str!(len) >> + ( + msg.to_string() + ) + )); + +#[cfg(test)] +mod tests { + + use nom::*; + use super::*; + + /// Simple test of some valid data. + #[test] + fn test_parse_valid() { + let buf = b"12:Hello World!4:Bye."; + + let result = parse_message(buf); + match result { + IResult::Done(remainder, message) => { + // Check the first message. + assert_eq!(message, "Hello World!"); + + // And we should have 6 bytes left. + assert_eq!(remainder.len(), 6); + } + IResult::Incomplete(_) => { + panic!("Result should not have been incomplete."); + } + IResult::Error(err) => { + panic!("Result should not be an error: {:?}.", err); + } + } + } + +} diff --git a/rust/src/applayertemplate/template.rs b/rust/src/applayertemplate/template.rs new file mode 100644 index 0000000000..865034b8b2 --- /dev/null +++ b/rust/src/applayertemplate/template.rs @@ -0,0 +1,519 @@ +/* Copyright (C) 2018 Open Information Security Foundation + * + * You can copy, redistribute or modify this Program under the terms of + * the GNU General Public License version 2 as published by the Free + * Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * version 2 along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +use std; +use core::{self, ALPROTO_UNKNOWN, AppProto, Flow}; +use libc; +use log::*; +use std::mem::transmute; +use applayer::{self, LoggerFlags}; +use parser::*; +use std::ffi::CString; +use nom; +use super::parser; + +static mut ALPROTO_TEMPLATE: AppProto = ALPROTO_UNKNOWN; + +pub struct TemplateTransaction { + tx_id: u64, + pub request: Option, + pub response: Option, + + logged: LoggerFlags, + de_state: Option<*mut core::DetectEngineState>, + events: *mut core::AppLayerDecoderEvents, +} + +impl TemplateTransaction { + pub fn new() -> TemplateTransaction { + TemplateTransaction { + tx_id: 0, + request: None, + response: None, + logged: LoggerFlags::new(), + de_state: None, + events: std::ptr::null_mut(), + } + } + + pub fn free(&mut self) { + if self.events != std::ptr::null_mut() { + core::sc_app_layer_decoder_events_free_events(&mut self.events); + } + if let Some(state) = self.de_state { + core::sc_detect_engine_state_free(state); + } + } +} + +impl Drop for TemplateTransaction { + fn drop(&mut self) { + self.free(); + } +} + +pub struct TemplateState { + tx_id: u64, + request_buffer: Vec, + response_buffer: Vec, + transactions: Vec, +} + +impl TemplateState { + pub fn new() -> Self { + Self { + tx_id: 0, + request_buffer: Vec::new(), + response_buffer: Vec::new(), + transactions: Vec::new(), + } + } + + // Free a transaction by ID. + fn free_tx(&mut self, tx_id: u64) { + let len = self.transactions.len(); + let mut found = false; + let mut index = 0; + for i in 0..len { + let tx = &self.transactions[i]; + if tx.tx_id == tx_id + 1 { + found = true; + index = i; + break; + } + } + if found { + self.transactions.remove(index); + } + } + + pub fn get_tx(&mut self, tx_id: u64) -> Option<&TemplateTransaction> { + for tx in &mut self.transactions { + if tx.tx_id == tx_id + 1 { + return Some(tx); + } + } + return None; + } + + fn new_tx(&mut self) -> TemplateTransaction { + let mut tx = TemplateTransaction::new(); + self.tx_id += 1; + tx.tx_id = self.tx_id; + return tx; + } + + fn find_request(&mut self) -> Option<&mut TemplateTransaction> { + for tx in &mut self.transactions { + if tx.response.is_none() { + return Some(tx); + } + } + None + } + + fn parse_request(&mut self, input: &[u8]) -> bool { + // We're not interested in empty requests. + if input.len() == 0 { + return true; + } + + // For simplicity, always extend the buffer and work on it. + self.request_buffer.extend(input); + + let tmp: Vec; + let mut current = { + tmp = self.request_buffer.split_off(0); + tmp.as_slice() + }; + + while current.len() > 0 { + match parser::parse_message(current) { + nom::IResult::Done(rem, request) => { + current = rem; + + SCLogNotice!("Request: {}", request); + let mut tx = self.new_tx(); + tx.request = Some(request); + self.transactions.push(tx); + } + nom::IResult::Incomplete(_) => { + self.request_buffer.extend_from_slice(current); + break; + } + nom::IResult::Error(_) => { + return false; + } + } + } + + return true; + } + + fn parse_response(&mut self, input: &[u8]) -> bool { + // We're not interested in empty responses. + if input.len() == 0 { + return true; + } + + // For simplicity, always extend the buffer and work on it. + self.response_buffer.extend(input); + + let tmp: Vec; + let mut current = { + tmp = self.response_buffer.split_off(0); + tmp.as_slice() + }; + + while current.len() > 0 { + match parser::parse_message(current) { + nom::IResult::Done(rem, response) => { + current = rem; + + match self.find_request() { + Some(tx) => { + tx.response = Some(response); + SCLogNotice!("Found response for request:"); + SCLogNotice!("- Request: {:?}", tx.request); + SCLogNotice!("- Response: {:?}", tx.response); + } + None => {} + } + } + nom::IResult::Incomplete(_) => { + self.response_buffer.extend_from_slice(current); + break; + } + nom::IResult::Error(_) => { + return false; + } + } + } + + return true; + } + + fn tx_iterator( + &mut self, + min_tx_id: u64, + state: &mut u64, + ) -> Option<(&TemplateTransaction, u64, bool)> { + let mut index = *state as usize; + let len = self.transactions.len(); + + while index < len { + let tx = &self.transactions[index]; + if tx.tx_id < min_tx_id + 1 { + index += 1; + continue; + } + *state = index as u64 + 1; + return Some((tx, tx.tx_id - 1, (len - index) > 1)); + } + + return None; + } +} + +/// Probe to see if this input looks like a request or response. +/// +/// For the purposes of this template things will be kept simple. The +/// protocol is text based with the leading text being the length of +/// the message in bytes. So simply make sure the first character is +/// between "1" and "9". +fn probe(input: &[u8]) -> bool { + if input.len() > 1 && input[0] >= 49 && input[0] <= 57 { + return true; + } + return false; +} + +// C exports. + +export_tx_get_detect_state!( + rs_template_tx_get_detect_state, + TemplateTransaction +); +export_tx_set_detect_state!( + rs_template_tx_set_detect_state, + TemplateTransaction +); + +/// C entry point for a probing parser. +#[no_mangle] +pub extern "C" fn rs_template_probing_parser( + _flow: *const Flow, + input: *const libc::uint8_t, + input_len: u32, +) -> AppProto { + // Need at least 2 bytes. + if input_len > 1 && input != std::ptr::null_mut() { + let slice = build_slice!(input, input_len as usize); + if probe(slice) { + return unsafe { ALPROTO_TEMPLATE }; + } + } + return ALPROTO_UNKNOWN; +} + +#[no_mangle] +pub extern "C" fn rs_template_state_new() -> *mut libc::c_void { + let state = TemplateState::new(); + let boxed = Box::new(state); + return unsafe { transmute(boxed) }; +} + +#[no_mangle] +pub extern "C" fn rs_template_state_free(state: *mut libc::c_void) { + // Just unbox... + let _drop: Box = unsafe { transmute(state) }; +} + +#[no_mangle] +pub extern "C" fn rs_template_state_tx_free( + state: *mut libc::c_void, + tx_id: libc::uint64_t, +) { + let state = cast_pointer!(state, TemplateState); + state.free_tx(tx_id); +} + +#[no_mangle] +pub extern "C" fn rs_template_parse_request( + _flow: *const Flow, + state: *mut libc::c_void, + pstate: *mut libc::c_void, + input: *const libc::uint8_t, + input_len: u32, + _data: *const libc::c_void, + _flags: u8, +) -> i8 { + let eof = unsafe { + if AppLayerParserStateIssetFlag(pstate, APP_LAYER_PARSER_EOF) > 0 { + true + } else { + false + } + }; + + if eof { + // If needed, handled EOF, or pass it into the parser. + } + + let state = cast_pointer!(state, TemplateState); + let buf = build_slice!(input, input_len as usize); + if state.parse_request(buf) { + return 1; + } + return -1; +} + +#[no_mangle] +pub extern "C" fn rs_template_parse_response( + _flow: *const Flow, + state: *mut libc::c_void, + pstate: *mut libc::c_void, + input: *const libc::uint8_t, + input_len: u32, + _data: *const libc::c_void, + _flags: u8, +) -> i8 { + let _eof = unsafe { + if AppLayerParserStateIssetFlag(pstate, APP_LAYER_PARSER_EOF) > 0 { + true + } else { + false + } + }; + let state = cast_pointer!(state, TemplateState); + let buf = build_slice!(input, input_len as usize); + if state.parse_response(buf) { + return 1; + } + return -1; +} + +#[no_mangle] +pub extern "C" fn rs_template_state_get_tx( + state: *mut libc::c_void, + tx_id: libc::uint64_t, +) -> *mut libc::c_void { + let state = cast_pointer!(state, TemplateState); + match state.get_tx(tx_id) { + Some(tx) => { + return unsafe { transmute(tx) }; + } + None => { + return std::ptr::null_mut(); + } + } +} + +#[no_mangle] +pub extern "C" fn rs_template_state_get_tx_count( + state: *mut libc::c_void, +) -> libc::uint64_t { + let state = cast_pointer!(state, TemplateState); + return state.tx_id; +} + +#[no_mangle] +pub extern "C" fn rs_template_state_progress_completion_status( + _direction: libc::uint8_t, +) -> libc::c_int { + // This parser uses 1 to signal transaction completion status. + return 1; +} + +#[no_mangle] +pub extern "C" fn rs_template_tx_get_alstate_progress( + tx: *mut libc::c_void, + _direction: libc::uint8_t, +) -> libc::c_int { + let tx = cast_pointer!(tx, TemplateTransaction); + + // Transaction is done if we have a response. + if tx.response.is_some() { + return 1; + } + return 0; +} + +#[no_mangle] +pub extern "C" fn rs_template_tx_get_logged( + _state: *mut libc::c_void, + tx: *mut libc::c_void, +) -> u32 { + let tx = cast_pointer!(tx, TemplateTransaction); + return tx.logged.get(); +} + +#[no_mangle] +pub extern "C" fn rs_template_tx_set_logged( + _state: *mut libc::c_void, + tx: *mut libc::c_void, + logged: libc::uint32_t, +) { + let tx = cast_pointer!(tx, TemplateTransaction); + tx.logged.set(logged); +} + +#[no_mangle] +pub extern "C" fn rs_template_state_get_events( + state: *mut libc::c_void, + tx_id: libc::uint64_t, +) -> *mut core::AppLayerDecoderEvents { + let state = cast_pointer!(state, TemplateState); + match state.get_tx(tx_id) { + Some(tx) => tx.events, + _ => std::ptr::null_mut(), + } +} + +#[no_mangle] +pub extern "C" fn rs_template_state_get_event_info( + _event_name: *const libc::c_char, + _event_id: *mut libc::c_int, + _event_type: *mut core::AppLayerEventType, +) -> libc::c_int { + return -1; +} + +#[no_mangle] +pub extern "C" fn rs_template_state_get_tx_iterator( + _ipproto: libc::uint8_t, + _alproto: AppProto, + state: *mut libc::c_void, + min_tx_id: libc::uint64_t, + _max_tx_id: libc::uint64_t, + istate: &mut libc::uint64_t, +) -> applayer::AppLayerGetTxIterTuple { + let state = cast_pointer!(state, TemplateState); + match state.tx_iterator(min_tx_id, istate) { + Some((tx, out_tx_id, has_next)) => { + let c_tx = unsafe { transmute(tx) }; + let ires = applayer::AppLayerGetTxIterTuple::with_values( + c_tx, + out_tx_id, + has_next, + ); + return ires; + } + None => { + return applayer::AppLayerGetTxIterTuple::not_found(); + } + } +} + +// Parser name as a C style string. +const PARSER_NAME: &'static [u8] = b"template-rust\0"; + +#[no_mangle] +pub unsafe extern "C" fn rs_template_register_parser() { + let default_port = CString::new("[7000]").unwrap(); + let parser = RustParser { + name: PARSER_NAME.as_ptr() as *const libc::c_char, + default_port: default_port.as_ptr(), + ipproto: libc::IPPROTO_TCP, + probe_ts: rs_template_probing_parser, + probe_tc: rs_template_probing_parser, + min_depth: 0, + max_depth: 16, + state_new: rs_template_state_new, + state_free: rs_template_state_free, + tx_free: rs_template_state_tx_free, + parse_ts: rs_template_parse_request, + parse_tc: rs_template_parse_response, + get_tx_count: rs_template_state_get_tx_count, + get_tx: rs_template_state_get_tx, + tx_get_comp_st: rs_template_state_progress_completion_status, + tx_get_progress: rs_template_tx_get_alstate_progress, + get_tx_logged: Some(rs_template_tx_get_logged), + set_tx_logged: Some(rs_template_tx_set_logged), + get_de_state: rs_template_tx_get_detect_state, + set_de_state: rs_template_tx_set_detect_state, + get_events: Some(rs_template_state_get_events), + get_eventinfo: Some(rs_template_state_get_event_info), + localstorage_new: None, + localstorage_free: None, + get_tx_mpm_id: None, + set_tx_mpm_id: None, + get_files: None, + get_tx_iterator: Some(rs_template_state_get_tx_iterator), + }; + + let ip_proto_str = CString::new("tcp").unwrap(); + + if AppLayerProtoDetectConfProtoDetectionEnabled( + ip_proto_str.as_ptr(), + parser.name, + ) != 0 + { + let alproto = AppLayerRegisterProtocolDetection(&parser, 1); + ALPROTO_TEMPLATE = alproto; + if AppLayerParserConfParserEnabled( + ip_proto_str.as_ptr(), + parser.name, + ) != 0 + { + let _ = AppLayerRegisterParser(&parser, alproto); + } + SCLogNotice!("Rust template parser registered."); + } else { + SCLogNotice!("Protocol detector and parser disabled for TEMPLATE."); + } +} diff --git a/rust/src/lib.rs b/rust/src/lib.rs index 02db7eb3d9..46f6a2d357 100644 --- a/rust/src/lib.rs +++ b/rust/src/lib.rs @@ -57,3 +57,4 @@ pub mod ikev2; pub mod ntp; pub mod tftp; pub mod dhcp; +pub mod applayertemplate; diff --git a/src/app-layer-template-rust.c b/src/app-layer-template-rust.c index a7d99f4d52..f7f76ce4b4 100644 --- a/src/app-layer-template-rust.c +++ b/src/app-layer-template-rust.c @@ -51,10 +51,12 @@ void RegisterTemplateRustParsers(void) { #ifdef HAVE_RUST + /* TEMPLATE_START_REMOVE */ /* Only register if enabled in config. */ if (ConfGetNode("app-layer.protocols.template-rust") == NULL) { return; } + /* TEMPLATE_END_REMOVE */ SCLogNotice("Registring Rust template parser."); rs_template_register_parser(); #endif