From 98f84d5a9b960b6d5e641f230d84331bb009a240 Mon Sep 17 00:00:00 2001 From: Philippe Antoine Date: Tue, 7 Sep 2021 14:54:57 +0200 Subject: [PATCH] http2: follow range requests Move the content-range parsing code to rust --- rules/http2-events.rules | 1 + rust/cbindgen.toml | 2 + rust/src/core.rs | 14 ++ rust/src/http2/detect.rs | 38 ++++++ rust/src/http2/http2.rs | 77 +++++++++-- rust/src/http2/mod.rs | 1 + rust/src/http2/range.rs | 241 +++++++++++++++++++++++++++++++++ src/app-layer-htp-file.c | 95 ++----------- src/app-layer-htp-file.h | 10 +- src/app-layer-htp-range.c | 43 +++++- src/app-layer-htp-range.h | 14 +- src/output-json-http.c | 2 +- src/rust-context.h | 7 + src/rust.h | 2 + src/suricata.c | 4 + src/tests/app-layer-htp-file.c | 8 +- 16 files changed, 441 insertions(+), 118 deletions(-) create mode 100644 rust/src/http2/range.rs diff --git a/rules/http2-events.rules b/rules/http2-events.rules index 97961da38f..8e45fca38d 100644 --- a/rules/http2-events.rules +++ b/rules/http2-events.rules @@ -15,3 +15,4 @@ alert http2 any any -> any any (msg:"SURICATA HTTP2 too long frame data"; flow:e alert http2 any any -> any any (msg:"SURICATA HTTP2 stream identifier reuse"; flow:established; app-layer-event:http2.stream_id_reuse; classtype:protocol-command-decode; sid:2290007; rev:1;) alert http2 any any -> any any (msg:"SURICATA HTTP2 invalid HTTP1 settings during upgrade"; flow:established; app-layer-event:http2.invalid_http1_settings; classtype:protocol-command-decode; sid:2290008; rev:1;) alert http2 any any -> any any (msg:"SURICATA HTTP2 failed decompression"; flow:established; app-layer-event:http2.failed_decompression; classtype:protocol-command-decode; sid:2290009; rev:1;) +alert http2 any any -> any any (msg:"SURICATA HTTP2 invalid range header"; flow:established; app-layer-event:http2.invalid_range; classtype:protocol-command-decode; sid:2290010; rev:1;) diff --git a/rust/cbindgen.toml b/rust/cbindgen.toml index d3b33312e3..9e58721f36 100644 --- a/rust/cbindgen.toml +++ b/rust/cbindgen.toml @@ -87,6 +87,8 @@ exclude = [ "CLuaState", "DetectEngineState", "Flow", + "StreamingBufferConfig", + "HttpRangeContainerBlock", "FileContainer", "JsonT", "IKEState", diff --git a/rust/src/core.rs b/rust/src/core.rs index 867d9bf7d2..f01a9daac8 100644 --- a/rust/src/core.rs +++ b/rust/src/core.rs @@ -99,6 +99,17 @@ pub type AppLayerDecoderEventsFreeEventsFunc = pub enum StreamingBufferConfig {} +// Opaque flow type (defined in C) +pub enum HttpRangeContainerBlock {} + +pub type SCHttpRangeFreeBlock = extern "C" fn ( + c: *mut HttpRangeContainerBlock); +pub type SCHTPFileCloseHandleRange = extern "C" fn ( + fc: *mut FileContainer, + flags: u16, + c: *mut HttpRangeContainerBlock, + data: *const u8, + data_len: u32); pub type SCFileOpenFileWithId = extern "C" fn ( file_container: &FileContainer, sbcfg: &StreamingBufferConfig, @@ -144,6 +155,9 @@ pub struct SuricataContext { AppLayerDecoderEventsFreeEvents: AppLayerDecoderEventsFreeEventsFunc, pub AppLayerParserTriggerRawStreamReassembly: AppLayerParserTriggerRawStreamReassemblyFunc, + pub HttpRangeFreeBlock: SCHttpRangeFreeBlock, + pub HTPFileCloseHandleRange: SCHTPFileCloseHandleRange, + pub FileOpenFile: SCFileOpenFileWithId, pub FileCloseFile: SCFileCloseFileById, pub FileAppendData: SCFileAppendDataById, diff --git a/rust/src/http2/detect.rs b/rust/src/http2/detect.rs index abc642ba9e..0cbc59c6c9 100644 --- a/rust/src/http2/detect.rs +++ b/rust/src/http2/detect.rs @@ -484,6 +484,44 @@ fn http2_frames_get_header_firstvalue<'a>( return Err(()); } +// same as http2_frames_get_header_value but returns a new Vec +// instead of using the transation to store the result slice +pub fn http2_frames_get_header_value_vec( + tx: &HTTP2Transaction, direction: u8, name: &str, +) -> Result, ()> { + let mut found = 0; + let mut vec = Vec::new(); + let frames = if direction == STREAM_TOSERVER { + &tx.frames_ts + } else { + &tx.frames_tc + }; + for i in 0..frames.len() { + if let Some(blocks) = http2_header_blocks(&frames[i]) { + for block in blocks.iter() { + if block.name == name.as_bytes().to_vec() { + if found == 0 { + vec.extend_from_slice(&block.value); + found = 1; + } else if found == 1 { + vec.extend_from_slice(&[b',', b' ']); + vec.extend_from_slice(&block.value); + found = 2; + } else { + vec.extend_from_slice(&[b',', b' ']); + vec.extend_from_slice(&block.value); + } + } + } + } + } + if found == 0 { + return Err(()); + } else { + return Ok(vec); + } +} + fn http2_frames_get_header_value<'a>( tx: &'a mut HTTP2Transaction, direction: u8, name: &str, ) -> Result<&'a [u8], ()> { diff --git a/rust/src/http2/http2.rs b/rust/src/http2/http2.rs index 1027ff506d..c590bd1a80 100644 --- a/rust/src/http2/http2.rs +++ b/rust/src/http2/http2.rs @@ -16,11 +16,14 @@ */ use super::decompression; +use super::detect; use super::parser; +use super::range; + use crate::applayer::{self, *}; use crate::core::{ - self, AppProto, Flow, SuricataFileContext, ALPROTO_FAILED, ALPROTO_UNKNOWN, IPPROTO_TCP, - STREAM_TOCLIENT, STREAM_TOSERVER, + self, AppProto, Flow, HttpRangeContainerBlock, SuricataFileContext, ALPROTO_FAILED, + ALPROTO_UNKNOWN, IPPROTO_TCP, SC, STREAM_TOCLIENT, STREAM_TOSERVER, }; use crate::filecontainer::*; use crate::filetracker::*; @@ -129,11 +132,12 @@ pub struct HTTP2Transaction { pub frames_ts: Vec, decoder: decompression::HTTP2Decoder, + pub file_range: *mut HttpRangeContainerBlock, de_state: Option<*mut core::DetectEngineState>, events: *mut core::AppLayerDecoderEvents, tx_data: AppLayerTxData, - ft_tc: FileTransferTracker, + pub ft_tc: FileTransferTracker, ft_ts: FileTransferTracker, //temporary escaped header for detection @@ -151,6 +155,7 @@ impl HTTP2Transaction { frames_tc: Vec::new(), frames_ts: Vec::new(), decoder: decompression::HTTP2Decoder::new(), + file_range: std::ptr::null_mut(), de_state: None, events: std::ptr::null_mut(), tx_data: AppLayerTxData::new(), @@ -167,6 +172,27 @@ impl HTTP2Transaction { if let Some(state) = self.de_state { core::sc_detect_engine_state_free(state); } + if self.file_range != std::ptr::null_mut() { + match unsafe { SC } { + None => panic!("BUG no suricata_config"), + Some(c) => { + //TODO get a file container instead of NULL + (c.HTPFileCloseHandleRange)( + std::ptr::null_mut(), + 0, + self.file_range, + std::ptr::null_mut(), + 0, + ); + (c.HttpRangeFreeBlock)(self.file_range); + } + } + } + } + + pub fn set_event(&mut self, event: HTTP2Event) { + let ev = event as u8; + core::sc_app_layer_decoder_events_set_event_raw(&mut self.events, ev); } fn handle_headers(&mut self, blocks: &Vec, dir: u8) { @@ -179,7 +205,7 @@ impl HTTP2Transaction { fn decompress<'a>( &'a mut self, input: &'a [u8], dir: u8, sfcm: &'static SuricataFileContext, over: bool, - files: &mut FileContainer, flags: u16, + files: &mut FileContainer, flags: u16, flow: *const Flow, ) -> io::Result<()> { let mut output = Vec::with_capacity(decompression::HTTP2_DECOMPRESSION_CHUNK_SIZE); let decompressed = self.decoder.decompress(input, &mut output, dir)?; @@ -190,6 +216,31 @@ impl HTTP2Transaction { // we are now sure that new_chunk will open a file // even if it may close it right afterwards self.tx_data.incr_files_opened(); + if let Ok(value) = detect::http2_frames_get_header_value_vec( + self, + STREAM_TOCLIENT, + "content-range", + ) { + match range::http2_parse_content_range(&value) { + Ok((_, v)) => { + range::http2_range_open(self, &v, flow, sfcm, flags, decompressed); + if over { + range::http2_range_close(self, files, flags, &[]) + } + } + _ => { + self.set_event(HTTP2Event::InvalidRange); + } + } + } + } else { + if self.file_range != std::ptr::null_mut() { + if over { + range::http2_range_close(self, files, flags, decompressed) + } else { + range::http2_range_append(self.file_range, decompressed) + } + } } self.ft_tc.new_chunk( sfcm, @@ -321,6 +372,7 @@ pub enum HTTP2Event { StreamIdReuse, InvalidHTTP1Settings, FailedDecompression, + InvalidRange, } pub struct HTTP2DynTable { @@ -750,7 +802,9 @@ impl HTTP2State { } } - fn parse_frames(&mut self, mut input: &[u8], il: usize, dir: u8) -> AppLayerResult { + fn parse_frames( + &mut self, mut input: &[u8], il: usize, dir: u8, flow: *const Flow, + ) -> AppLayerResult { while input.len() > 0 { match parser::http2_parse_frame_header(input) { Ok((rem, head)) => { @@ -825,6 +879,7 @@ impl HTTP2State { over, files, flags, + flow, ) { Err(_e) => { self.set_event(HTTP2Event::FailedDecompression); @@ -854,7 +909,7 @@ impl HTTP2State { return AppLayerResult::ok(); } - fn parse_ts(&mut self, mut input: &[u8]) -> AppLayerResult { + fn parse_ts(&mut self, mut input: &[u8], flow: *const Flow) -> AppLayerResult { //very first : skip magic let mut magic_consumed = 0; if self.progress < HTTP2ConnectionState::Http2StateMagicDone { @@ -894,7 +949,7 @@ impl HTTP2State { } //then parse all we can - let r = self.parse_frames(input, il, STREAM_TOSERVER); + let r = self.parse_frames(input, il, STREAM_TOSERVER, flow); if r.status == 1 { //adds bytes consumed by banner to incomplete result return AppLayerResult::incomplete(r.consumed + magic_consumed as u32, r.needed); @@ -903,7 +958,7 @@ impl HTTP2State { } } - fn parse_tc(&mut self, mut input: &[u8]) -> AppLayerResult { + fn parse_tc(&mut self, mut input: &[u8], flow: *const Flow) -> AppLayerResult { //first consume frame bytes let il = input.len(); if self.response_frame_size > 0 { @@ -918,7 +973,7 @@ impl HTTP2State { } } //then parse all we can - return self.parse_frames(input, il, STREAM_TOCLIENT); + return self.parse_frames(input, il, STREAM_TOCLIENT, flow); } fn tx_iterator( @@ -1025,7 +1080,7 @@ pub unsafe extern "C" fn rs_http2_parse_ts( state.files.flags_ts = FileFlowToFlags(flow, STREAM_TOSERVER); state.files.flags_ts = state.files.flags_ts | FILE_USE_DETECT; - return state.parse_ts(buf); + return state.parse_ts(buf, flow); } #[no_mangle] @@ -1037,7 +1092,7 @@ pub unsafe extern "C" fn rs_http2_parse_tc( let buf = build_slice!(input, input_len as usize); state.files.flags_tc = FileFlowToFlags(flow, STREAM_TOCLIENT); state.files.flags_tc = state.files.flags_tc | FILE_USE_DETECT; - return state.parse_tc(buf); + return state.parse_tc(buf, flow); } #[no_mangle] diff --git a/rust/src/http2/mod.rs b/rust/src/http2/mod.rs index 08a5f32770..9aa6bbaac0 100644 --- a/rust/src/http2/mod.rs +++ b/rust/src/http2/mod.rs @@ -21,3 +21,4 @@ pub mod http2; mod huffman; pub mod logger; mod parser; +mod range; diff --git a/rust/src/http2/range.rs b/rust/src/http2/range.rs new file mode 100644 index 0000000000..5533820905 --- /dev/null +++ b/rust/src/http2/range.rs @@ -0,0 +1,241 @@ +/* Copyright (C) 2021 Open Information Security Foundation +* +* You can copy, redistribute or modify this Program under the terms of +* the GNU General Public License version 2 as published by the Free +* Software Foundation. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU General Public License for more details. +* +* You should have received a copy of the GNU General Public License +* version 2 along with this program; if not, write to the Free Software +* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA +* 02110-1301, USA. +*/ + +use super::detect; +use crate::core::{ + Flow, HttpRangeContainerBlock, StreamingBufferConfig, SuricataFileContext, SC, STREAM_TOSERVER, +}; +use crate::filecontainer::FileContainer; +use crate::http2::http2::HTTP2Transaction; + +use nom::character::complete::digit1; +use nom::IResult; +use std::os::raw::c_uchar; +use std::str::FromStr; + +#[derive(Debug)] +#[repr(C)] +pub struct HTTPContentRange { + pub start: i64, + pub end: i64, + pub size: i64, +} + +pub fn http2_parse_content_range_star<'a>(input: &'a [u8]) -> IResult<&'a [u8], HTTPContentRange> { + let (i2, _) = char!(input, '*')?; + let (i2, _) = char!(i2, '/')?; + let (i2, size) = map_res!(i2, map_res!(digit1, std::str::from_utf8), i64::from_str)?; + return Ok(( + i2, + HTTPContentRange { + start: -1, + end: -1, + size: size, + }, + )); +} + +pub fn http2_parse_content_range_def<'a>(input: &'a [u8]) -> IResult<&'a [u8], HTTPContentRange> { + let (i2, start) = map_res!(input, map_res!(digit1, std::str::from_utf8), i64::from_str)?; + let (i2, _) = char!(i2, '-')?; + let (i2, end) = map_res!(i2, map_res!(digit1, std::str::from_utf8), i64::from_str)?; + let (i2, _) = char!(i2, '/')?; + let (i2, size) = alt!( + i2, + value!(-1, char!('*')) | map_res!(map_res!(digit1, std::str::from_utf8), i64::from_str) + )?; + return Ok(( + i2, + HTTPContentRange { + start: start, + end: end, + size: size, + }, + )); +} + +pub fn http2_parse_content_range<'a>(input: &'a [u8]) -> IResult<&'a [u8], HTTPContentRange> { + let (i2, _) = take_while!(input, |c| c == b' ')?; + let (i2, _) = take_till!(i2, |c| c == b' ')?; + let (i2, _) = take_while!(i2, |c| c == b' ')?; + return alt!( + i2, + http2_parse_content_range_star | http2_parse_content_range_def + ); +} + +#[no_mangle] +pub unsafe extern "C" fn rs_http_parse_content_range( + cr: &mut HTTPContentRange, buffer: *const u8, buffer_len: u32, +) -> std::os::raw::c_int { + let slice = build_slice!(buffer, buffer_len as usize); + match http2_parse_content_range(slice) { + Ok((_, c)) => { + *cr = c; + return 0; + } + _ => { + return -1; + } + } +} + +fn http2_range_key_get(tx: &mut HTTP2Transaction) -> Result<(Vec, usize), ()> { + let hostv = detect::http2_frames_get_header_value_vec(tx, STREAM_TOSERVER, ":authority")?; + let mut hostv = &hostv[..]; + match hostv.iter().position(|&x| x == b':') { + Some(p) => { + hostv = &hostv[..p]; + } + None => {} + } + let uriv = detect::http2_frames_get_header_value_vec(tx, STREAM_TOSERVER, ":path")?; + let mut uriv = &uriv[..]; + match uriv.iter().position(|&x| x == b'?') { + Some(p) => { + uriv = &uriv[..p]; + } + None => {} + } + match uriv.iter().rposition(|&x| x == b'/') { + Some(p) => { + uriv = &uriv[p..]; + } + None => {} + } + let mut r = Vec::with_capacity(hostv.len() + uriv.len()); + r.extend_from_slice(hostv); + r.extend_from_slice(uriv); + return Ok((r, hostv.len())); +} + +pub fn http2_range_open( + tx: &mut HTTP2Transaction, v: &HTTPContentRange, flow: *const Flow, + cfg: &'static SuricataFileContext, flags: u16, data: &[u8], +) { + if let Ok((key, index)) = http2_range_key_get(tx) { + let name = &key[index..]; + tx.file_range = unsafe { + HttpRangeContainerOpenFile( + key.as_ptr(), + key.len() as u32, + flow, + v, + cfg.files_sbcfg, + name.as_ptr(), + name.len() as u16, + flags, + data.as_ptr(), + data.len() as u32, + ) + }; + } +} + +pub fn http2_range_append(fr: *mut HttpRangeContainerBlock, data: &[u8]) { + unsafe { + HttpRangeAppendData(fr, data.as_ptr(), data.len() as u32); + } +} + +pub fn http2_range_close( + tx: &mut HTTP2Transaction, files: &mut FileContainer, flags: u16, data: &[u8], +) { + match unsafe { SC } { + None => panic!("BUG no suricata_config"), + Some(c) => { + (c.HTPFileCloseHandleRange)( + files, + flags, + tx.file_range, + data.as_ptr(), + data.len() as u32, + ); + (c.HttpRangeFreeBlock)(tx.file_range); + } + } + tx.file_range = std::ptr::null_mut(); +} + +// Defined in app-layer-htp-range.h +extern "C" { + pub fn HttpRangeContainerOpenFile( + key: *const c_uchar, keylen: u32, f: *const Flow, cr: &HTTPContentRange, + sbcfg: *const StreamingBufferConfig, name: *const c_uchar, name_len: u16, flags: u16, + data: *const c_uchar, data_len: u32, + ) -> *mut HttpRangeContainerBlock; + pub fn HttpRangeAppendData( + c: *mut HttpRangeContainerBlock, data: *const c_uchar, data_len: u32, + ) -> std::os::raw::c_int; +} + +#[cfg(test)] +mod tests { + + use super::*; + + #[test] + fn test_http2_parse_content_range() { + let buf0: &[u8] = " bytes */100".as_bytes(); + let r0 = http2_parse_content_range(buf0); + match r0 { + Ok((rem, rg)) => { + // Check the first message. + assert_eq!(rg.start, -1); + assert_eq!(rg.end, -1); + assert_eq!(rg.size, 100); + // And we should have no bytes left. + assert_eq!(rem.len(), 0); + } + _ => { + panic!("Result should have been ok."); + } + } + + let buf1: &[u8] = " bytes 10-20/200".as_bytes(); + let r1 = http2_parse_content_range(buf1); + match r1 { + Ok((rem, rg)) => { + // Check the first message. + assert_eq!(rg.start, 10); + assert_eq!(rg.end, 20); + assert_eq!(rg.size, 200); + // And we should have no bytes left. + assert_eq!(rem.len(), 0); + } + _ => { + panic!("Result should have been ok."); + } + } + + let buf2: &[u8] = " bytes 30-68/*".as_bytes(); + let r2 = http2_parse_content_range(buf2); + match r2 { + Ok((rem, rg)) => { + // Check the first message. + assert_eq!(rg.start, 30); + assert_eq!(rg.end, 68); + assert_eq!(rg.size, -1); + // And we should have no bytes left. + assert_eq!(rem.len(), 0); + } + _ => { + panic!("Result should have been ok."); + } + } + } +} diff --git a/src/app-layer-htp-file.c b/src/app-layer-htp-file.c index e489298f1a..45b9dcfe1b 100644 --- a/src/app-layer-htp-file.c +++ b/src/app-layer-htp-file.c @@ -161,60 +161,10 @@ end: * * @return HTP_OK on success, HTP_ERROR on failure. */ -int HTPParseContentRange(bstr * rawvalue, HtpContentRange *range) +int HTPParseContentRange(bstr *rawvalue, HTTPContentRange *range) { - unsigned char *data = bstr_ptr(rawvalue); - size_t len = bstr_len(rawvalue); - size_t pos = 0; - size_t last_pos; - - // skip spaces and units - while (pos < len && data[pos] == ' ') - pos++; - while (pos < len && data[pos] != ' ') - pos++; - while (pos < len && data[pos] == ' ') - pos++; - - // initialize to unseen - range->start = -1; - range->end = -1; - range->size = -1; - - if (pos == len) { - // missing data - return -1; - } - - if (data[pos] == '*') { - // case with size only - if (len <= pos + 1 || data[pos+1] != '/') { - range->size = -1; - return -1; - } - pos += 2; - range->size = bstr_util_mem_to_pint(data + pos, len - pos, 10, &last_pos); - } else { - // case with start and end - range->start = bstr_util_mem_to_pint(data + pos, len - pos, 10, &last_pos); - pos += last_pos; - if (len <= pos + 1 || data[pos] != '-') { - return -1; - } - pos++; - range->end = bstr_util_mem_to_pint(data + pos, len - pos, 10, &last_pos); - pos += last_pos; - if (len <= pos + 1 || data[pos] != '/') { - return -1; - } - pos++; - if (data[pos] != '*') { - // case with size - range->size = bstr_util_mem_to_pint(data + pos, len - pos, 10, &last_pos); - } - } - - return 0; + uint32_t len = bstr_len(rawvalue); + return rs_http_parse_content_range(range, bstr_ptr(rawvalue), len); } /** @@ -226,7 +176,7 @@ int HTPParseContentRange(bstr * rawvalue, HtpContentRange *range) * @return HTP_OK on success, HTP_ERROR, -2, -3 on failure. */ static int HTPParseAndCheckContentRange( - bstr *rawvalue, HtpContentRange *range, HtpState *s, HtpTxUserData *htud) + bstr *rawvalue, HTTPContentRange *range, HtpState *s, HtpTxUserData *htud) { int r = HTPParseContentRange(rawvalue, range); if (r != 0) { @@ -269,7 +219,7 @@ int HTPFileOpenWithRange(HtpState *s, HtpTxUserData *txud, const uint8_t *filena DEBUG_VALIDATE_BUG_ON(s == NULL); // This function is only called STREAM_TOCLIENT from HtpResponseBodyHandle - HtpContentRange crparsed; + HTTPContentRange crparsed; if (HTPParseAndCheckContentRange(rawvalue, &crparsed, s, htud) != 0) { // range is invalid, fall back to classic open return HTPFileOpen( @@ -325,38 +275,13 @@ int HTPFileOpenWithRange(HtpState *s, HtpTxUserData *txud, const uint8_t *filena // do not reassemble file without host info SCReturnInt(0); } - HttpRangeContainerFile *file_range_container = - HttpRangeContainerUrlGet(keyurl, keylen, &s->f->lastts); - SCFree(keyurl); - if (file_range_container == NULL) { - // probably reached memcap - SCReturnInt(-1); - } DEBUG_VALIDATE_BUG_ON(s->file_range); - s->file_range = HttpRangeOpenFile(file_range_container, crparsed.start, crparsed.end, - crparsed.size, &s->cfg->response.sbcfg, filename, filename_len, flags, data, data_len); - SCLogDebug("s->file_range == %p", s->file_range); + s->file_range = HttpRangeContainerOpenFile(keyurl, keylen, s->f, &crparsed, + &s->cfg->response.sbcfg, filename, filename_len, flags, data, data_len); + SCFree(keyurl); if (s->file_range == NULL) { - SCLogDebug("s->file_range == NULL"); - THashDecrUsecnt(file_range_container->hdata); - DEBUG_VALIDATE_BUG_ON( - SC_ATOMIC_GET(file_range_container->hdata->use_cnt) > (uint32_t)INT_MAX); - THashDataUnlock(file_range_container->hdata); - - // probably reached memcap SCReturnInt(-1); - /* in some cases we don't take a reference, so decr use cnt */ - } else if (s->file_range->container == NULL) { - THashDecrUsecnt(file_range_container->hdata); - } else { - SCLogDebug("container %p use_cnt %u", s->file_range, - SC_ATOMIC_GET(s->file_range->container->hdata->use_cnt)); - DEBUG_VALIDATE_BUG_ON( - SC_ATOMIC_GET(s->file_range->container->hdata->use_cnt) > (uint32_t)INT_MAX); } - - /* we're done, so unlock. But since we have a reference in s->file_range keep use_cnt. */ - THashDataUnlock(file_range_container->hdata); SCReturnInt(0); } @@ -415,7 +340,7 @@ end: SCReturnInt(retval); } -void HTPFileCloseHandleRange(FileContainer *files, const uint8_t flags, HttpRangeContainerBlock *c, +void HTPFileCloseHandleRange(FileContainer *files, const uint16_t flags, HttpRangeContainerBlock *c, const uint8_t *data, uint32_t data_len) { if (HttpRangeAppendData(c, data, data_len) < 0) { @@ -429,7 +354,7 @@ void HTPFileCloseHandleRange(FileContainer *files, const uint8_t flags, HttpRang SCLogDebug("range in ERROR state"); } File *ranged = HttpRangeClose(c, flags); - if (ranged) { + if (ranged && files) { /* HtpState owns the constructed file now */ FileContainerAdd(files, ranged); } diff --git a/src/app-layer-htp-file.h b/src/app-layer-htp-file.h index 67620186ab..66522a3f2f 100644 --- a/src/app-layer-htp-file.h +++ b/src/app-layer-htp-file.h @@ -25,19 +25,13 @@ #ifndef __APP_LAYER_HTP_FILE_H__ #define __APP_LAYER_HTP_FILE_H__ -typedef struct HtpContentRange_ { - int64_t start; - int64_t end; - int64_t size; -} HtpContentRange; - int HTPFileOpen(HtpState *, HtpTxUserData *, const uint8_t *, uint16_t, const uint8_t *, uint32_t, uint64_t, uint8_t); -int HTPParseContentRange(bstr * rawvalue, HtpContentRange *range); +int HTPParseContentRange(bstr *rawvalue, HTTPContentRange *range); int HTPFileOpenWithRange(HtpState *, HtpTxUserData *, const uint8_t *, uint16_t, const uint8_t *, uint32_t, uint64_t, bstr *rawvalue, HtpTxUserData *htud); void HTPFileCloseHandleRange( - FileContainer *, const uint8_t, HttpRangeContainerBlock *, const uint8_t *, uint32_t); + FileContainer *, const uint16_t, HttpRangeContainerBlock *, const uint8_t *, uint32_t); int HTPFileStoreChunk(HtpState *, const uint8_t *, uint32_t, uint8_t); int HTPFileClose(HtpState *, const uint8_t *, uint32_t, uint8_t, uint8_t); diff --git a/src/app-layer-htp-range.c b/src/app-layer-htp-range.c index 0c508f1cf2..271aa10bd4 100644 --- a/src/app-layer-htp-range.c +++ b/src/app-layer-htp-range.c @@ -236,8 +236,9 @@ uint32_t HttpRangeContainersTimeoutHash(struct timeval *ts) /** * \returns locked data */ -void *HttpRangeContainerUrlGet(const uint8_t *key, size_t keylen, struct timeval *ts) +static void *HttpRangeContainerUrlGet(const uint8_t *key, uint32_t keylen, const Flow *f) { + const struct timeval *ts = &f->lastts; HttpRangeContainerFile lookup; memset(&lookup, 0, sizeof(lookup)); // cast so as not to have const in the structure @@ -345,9 +346,9 @@ static HttpRangeContainerBlock *HttpRangeOpenFileAux(HttpRangeContainerFile *c, return curf; } -HttpRangeContainerBlock *HttpRangeOpenFile(HttpRangeContainerFile *c, uint64_t start, uint64_t end, - uint64_t total, const StreamingBufferConfig *sbcfg, const uint8_t *name, uint16_t name_len, - uint16_t flags, const uint8_t *data, uint32_t len) +static HttpRangeContainerBlock *HttpRangeOpenFile(HttpRangeContainerFile *c, uint64_t start, + uint64_t end, uint64_t total, const StreamingBufferConfig *sbcfg, const uint8_t *name, + uint16_t name_len, uint16_t flags, const uint8_t *data, uint32_t len) { HttpRangeContainerBlock *r = HttpRangeOpenFileAux(c, start, end, total, sbcfg, name, name_len, flags); @@ -357,6 +358,40 @@ HttpRangeContainerBlock *HttpRangeOpenFile(HttpRangeContainerFile *c, uint64_t s return r; } +HttpRangeContainerBlock *HttpRangeContainerOpenFile(const uint8_t *key, uint32_t keylen, + const Flow *f, const HTTPContentRange *crparsed, const StreamingBufferConfig *sbcfg, + const uint8_t *name, uint16_t name_len, uint16_t flags, const uint8_t *data, + uint32_t data_len) +{ + HttpRangeContainerFile *file_range_container = HttpRangeContainerUrlGet(key, keylen, f); + if (file_range_container == NULL) { + // probably reached memcap + return NULL; + } + HttpRangeContainerBlock *r = HttpRangeOpenFile(file_range_container, crparsed->start, + crparsed->end, crparsed->size, sbcfg, name, name_len, flags, data, data_len); + SCLogDebug("s->file_range == %p", r); + if (r == NULL) { + THashDecrUsecnt(file_range_container->hdata); + DEBUG_VALIDATE_BUG_ON( + SC_ATOMIC_GET(file_range_container->hdata->use_cnt) > (uint32_t)INT_MAX); + THashDataUnlock(file_range_container->hdata); + + // probably reached memcap + return NULL; + /* in some cases we don't take a reference, so decr use cnt */ + } else if (r->container == NULL) { + THashDecrUsecnt(file_range_container->hdata); + } else { + SCLogDebug("container %p use_cnt %u", r, SC_ATOMIC_GET(r->container->hdata->use_cnt)); + DEBUG_VALIDATE_BUG_ON(SC_ATOMIC_GET(r->container->hdata->use_cnt) > (uint32_t)INT_MAX); + } + + /* we're done, so unlock. But since we have a reference in s->file_range keep use_cnt. */ + THashDataUnlock(file_range_container->hdata); + return r; +} + int HttpRangeAppendData(HttpRangeContainerBlock *c, const uint8_t *data, uint32_t len) { if (len == 0) { diff --git a/src/app-layer-htp-range.h b/src/app-layer-htp-range.h index 455d6d2e34..cda202294d 100644 --- a/src/app-layer-htp-range.h +++ b/src/app-layer-htp-range.h @@ -18,14 +18,16 @@ #ifndef __APP_LAYER_HTP_RANGE_H__ #define __APP_LAYER_HTP_RANGE_H__ +#include "suricata-common.h" +#include "app-layer-parser.h" + #include "util-thash.h" +#include "rust-bindings.h" void HttpRangeContainersInit(void); void HttpRangeContainersDestroy(void); uint32_t HttpRangeContainersTimeoutHash(struct timeval *ts); -void *HttpRangeContainerUrlGet(const uint8_t *key, size_t keylen, struct timeval *ts); - // linked list of ranges : buffer with offset typedef struct HttpRangeContainerBuffer { /** red and black tree */ @@ -98,9 +100,11 @@ typedef struct HttpRangeContainerBlock { int HttpRangeAppendData(HttpRangeContainerBlock *c, const uint8_t *data, uint32_t len); File *HttpRangeClose(HttpRangeContainerBlock *c, uint16_t flags); -HttpRangeContainerBlock *HttpRangeOpenFile(HttpRangeContainerFile *c, uint64_t start, uint64_t end, - uint64_t total, const StreamingBufferConfig *sbcfg, const uint8_t *name, uint16_t name_len, - uint16_t flags, const uint8_t *data, uint32_t len); +// HttpRangeContainerBlock but trouble with headers inclusion order +HttpRangeContainerBlock *HttpRangeContainerOpenFile(const unsigned char *key, uint32_t keylen, + const Flow *f, const HTTPContentRange *cr, const StreamingBufferConfig *sbcfg, + const unsigned char *name, uint16_t name_len, uint16_t flags, const unsigned char *data, + uint32_t data_len); void HttpRangeFreeBlock(HttpRangeContainerBlock *b); diff --git a/src/output-json-http.c b/src/output-json-http.c index bc38254d3b..ddddf597c5 100644 --- a/src/output-json-http.c +++ b/src/output-json-http.c @@ -252,7 +252,7 @@ static void EveHttpLogJSONBasic(JsonBuilder *js, htp_tx_t *tx) jb_open_object(js, "content_range"); jb_set_string_from_bytes( js, "raw", bstr_ptr(h_content_range->value), bstr_len(h_content_range->value)); - HtpContentRange crparsed; + HTTPContentRange crparsed; if (HTPParseContentRange(h_content_range->value, &crparsed) == 0) { if (crparsed.start >= 0) jb_set_uint(js, "start", crparsed.start); diff --git a/src/rust-context.h b/src/rust-context.h index 4c07423aa2..20d8ce7ef3 100644 --- a/src/rust-context.h +++ b/src/rust-context.h @@ -25,6 +25,9 @@ #include "app-layer-snmp.h" //SNMPState, SNMPTransaction #include "app-layer-tftp.h" //TFTPState, TFTPTransaction +// hack for include orders cf SCSha256 +typedef struct HttpRangeContainerBlock HttpRangeContainerBlock; + struct AppLayerParser; typedef struct SuricataContext_ { @@ -36,6 +39,10 @@ typedef struct SuricataContext_ { void (*AppLayerDecoderEventsFreeEvents)(AppLayerDecoderEvents **); void (*AppLayerParserTriggerRawStreamReassembly)(Flow *, int direction); + void (*HttpRangeFreeBlock)(HttpRangeContainerBlock *); + void (*HTPFileCloseHandleRange)( + FileContainer *, const uint16_t, HttpRangeContainerBlock *, const uint8_t *, uint32_t); + int (*FileOpenFileWithId)(FileContainer *, const StreamingBufferConfig *, uint32_t track_id, const uint8_t *name, uint16_t name_len, const uint8_t *data, uint32_t data_len, uint16_t flags); diff --git a/src/rust.h b/src/rust.h index 6888460039..a01e6e602b 100644 --- a/src/rust.h +++ b/src/rust.h @@ -19,6 +19,8 @@ #define __RUST_H__ #include "util-lua.h" +// hack for include orders cf SCSha256 +typedef struct HttpRangeContainerBlock HttpRangeContainerBlock; #include "rust-context.h" #include "rust-bindings.h" diff --git a/src/suricata.c b/src/suricata.c index 3c8f581611..a0df21db36 100644 --- a/src/suricata.c +++ b/src/suricata.c @@ -123,6 +123,7 @@ #include "app-layer-enip.h" #include "app-layer-dnp3.h" #include "app-layer-smb.h" +#include "app-layer-htp-file.h" #include "output-filestore.h" @@ -2666,6 +2667,9 @@ int InitGlobal(void) { suricata_context.AppLayerParserTriggerRawStreamReassembly = AppLayerParserTriggerRawStreamReassembly; + suricata_context.HttpRangeFreeBlock = HttpRangeFreeBlock; + suricata_context.HTPFileCloseHandleRange = HTPFileCloseHandleRange; + suricata_context.FileOpenFileWithId = FileOpenFileWithId; suricata_context.FileCloseFileById = FileCloseFileById; suricata_context.FileAppendDataById = FileAppendDataById; diff --git a/src/tests/app-layer-htp-file.c b/src/tests/app-layer-htp-file.c index a1701c0d36..869c04944b 100644 --- a/src/tests/app-layer-htp-file.c +++ b/src/tests/app-layer-htp-file.c @@ -26,7 +26,7 @@ static int AppLayerHtpFileParseContentRangeTest01 (void) { - HtpContentRange range; + HTTPContentRange range; bstr * rawvalue = bstr_dup_c("bytes 12-25/100"); FAIL_IF_NOT(HTPParseContentRange(rawvalue, &range) == 0); FAIL_IF_NOT(range.start == 12); @@ -43,7 +43,7 @@ static int AppLayerHtpFileParseContentRangeTest01 (void) static int AppLayerHtpFileParseContentRangeTest02 (void) { - HtpContentRange range; + HTTPContentRange range; bstr * rawvalue = bstr_dup_c("bytes 15335424-27514354/"); FAIL_IF(HTPParseContentRange(rawvalue, &range) == 0); bstr_free(rawvalue); @@ -57,7 +57,7 @@ static int AppLayerHtpFileParseContentRangeTest02 (void) static int AppLayerHtpFileParseContentRangeTest03 (void) { - HtpContentRange range; + HTTPContentRange range; bstr * rawvalue = bstr_dup_c("bytes 15335424-"); FAIL_IF(HTPParseContentRange(rawvalue, &range) == 0); bstr_free(rawvalue); @@ -72,7 +72,7 @@ static int AppLayerHtpFileParseContentRangeTest03 (void) static int AppLayerHtpFileParseContentRangeTest04 (void) { - HtpContentRange range; + HTTPContentRange range; bstr * rawvalue = bstr_dup_c("bytes 24-42/*"); FAIL_IF_NOT(HTPParseContentRange(rawvalue, &range) == 0); FAIL_IF_NOT(range.start == 24); -- 2.47.2