From: Philippe Antoine Date: Thu, 12 Nov 2020 08:24:36 +0000 (+0100) Subject: http2: decompression for files X-Git-Tag: suricata-7.0.0-beta1~1839 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=refs%2Fpull%2F5780%2Fhead;p=thirdparty%2Fsuricata.git http2: decompression for files gzip and brotli decompression for files --- diff --git a/rules/http2-events.rules b/rules/http2-events.rules index bb2d08ded5..97961da38f 100644 --- a/rules/http2-events.rules +++ b/rules/http2-events.rules @@ -14,3 +14,4 @@ alert http2 any any -> any any (msg:"SURICATA HTTP2 header frame with extra data alert http2 any any -> any any (msg:"SURICATA HTTP2 too long frame data"; flow:established; app-layer-event:http2.long_frame_data; classtype:protocol-command-decode; sid:2290006; rev:1;) alert http2 any any -> any any (msg:"SURICATA HTTP2 stream identifier reuse"; flow:established; app-layer-event:http2.stream_id_reuse; classtype:protocol-command-decode; sid:2290007; rev:1;) alert http2 any any -> any any (msg:"SURICATA HTTP2 invalid HTTP1 settings during upgrade"; flow:established; app-layer-event:http2.invalid_http1_settings; classtype:protocol-command-decode; sid:2290008; rev:1;) +alert http2 any any -> any any (msg:"SURICATA HTTP2 failed decompression"; flow:established; app-layer-event:http2.failed_decompression; classtype:protocol-command-decode; sid:2290009; rev:1;) diff --git a/rust/Cargo.toml.in b/rust/Cargo.toml.in index 4596a27c6f..28237ccbcf 100644 --- a/rust/Cargo.toml.in +++ b/rust/Cargo.toml.in @@ -29,6 +29,8 @@ num = "0.2" num-derive = "0.2" num-traits = "0.2" widestring = "0.4" +flate2 = "1.0" +brotli = "3.3.0" der-parser = "4.0" kerberos-parser = "0.5" diff --git a/rust/src/http2/decompression.rs b/rust/src/http2/decompression.rs new file mode 100644 index 0000000000..205b527a14 --- /dev/null +++ b/rust/src/http2/decompression.rs @@ -0,0 +1,216 @@ +/* Copyright (C) 2021 Open Information Security Foundation +* +* You can copy, redistribute or modify this Program under the terms of +* the GNU General Public License version 2 as published by the Free +* Software Foundation. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU General Public License for more details. +* +* You should have received a copy of the GNU General Public License +* version 2 along with this program; if not, write to the Free Software +* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA +* 02110-1301, USA. +*/ + +use crate::core::STREAM_TOCLIENT; +use brotli; +use flate2::read::GzDecoder; +use std; +use std::io; +use std::io::{Cursor, Read, Write}; + +pub const HTTP2_DECOMPRESSION_CHUNK_SIZE: usize = 0x1000; // 4096 + +#[repr(u8)] +#[derive(Copy, Clone, PartialOrd, PartialEq, Debug)] +pub enum HTTP2ContentEncoding { + HTTP2ContentEncodingUnknown = 0, + HTTP2ContentEncodingGzip = 1, + HTTP2ContentEncodingBr = 2, + HTTP2ContentEncodingUnrecognized = 3, +} + +//a cursor turning EOF into blocking errors +pub struct HTTP2cursor { + pub cursor: Cursor>, +} + +impl HTTP2cursor { + pub fn new() -> HTTP2cursor { + HTTP2cursor { + cursor: Cursor::new(Vec::new()), + } + } + + #[cfg(feature = "debug-validate")] + pub fn position(&self) -> u64 { + return self.cursor.position(); + } + + pub fn set_position(&mut self, pos: u64) { + return self.cursor.set_position(pos); + } +} + +// we need to implement this as flate2 and brotli crates +// will read from this object +impl Read for HTTP2cursor { + fn read(&mut self, buf: &mut [u8]) -> io::Result { + //use the cursor, except it turns eof into blocking error + let r = self.cursor.read(buf); + match r { + Err(ref err) => { + if err.kind() == io::ErrorKind::UnexpectedEof { + return Err(io::ErrorKind::WouldBlock.into()); + } + } + Ok(0) => { + //regular EOF turned into blocking error + return Err(io::ErrorKind::WouldBlock.into()); + } + Ok(_n) => {} + } + return r; + } +} + +pub enum HTTP2Decompresser { + UNASSIGNED, + GZIP(GzDecoder), + BROTLI(brotli::Decompressor), +} + +struct HTTP2DecoderHalf { + encoding: HTTP2ContentEncoding, + decoder: HTTP2Decompresser, +} + +pub trait GetMutCursor { + fn get_mut(&mut self) -> &mut HTTP2cursor; +} + +impl GetMutCursor for GzDecoder { + fn get_mut(&mut self) -> &mut HTTP2cursor { + return self.get_mut(); + } +} + +impl GetMutCursor for brotli::Decompressor { + fn get_mut(&mut self) -> &mut HTTP2cursor { + return self.get_mut(); + } +} + +fn http2_decompress<'a>( + decoder: &mut (impl Read + GetMutCursor), input: &'a [u8], output: &'a mut Vec, +) -> io::Result<&'a [u8]> { + match decoder.get_mut().cursor.write_all(input) { + Ok(()) => {} + Err(e) => { + return Err(e); + } + } + let mut offset = 0; + decoder.get_mut().set_position(0); + output.resize(HTTP2_DECOMPRESSION_CHUNK_SIZE, 0); + loop { + match decoder.read(&mut output[offset..]) { + Ok(0) => { + break; + } + Ok(n) => { + offset += n; + if offset == output.len() { + output.resize(output.len() + HTTP2_DECOMPRESSION_CHUNK_SIZE, 0); + } + } + Err(e) => { + if e.kind() == io::ErrorKind::WouldBlock { + break; + } + return Err(e); + } + } + } + //checks all input was consumed + debug_validate_bug_on!(decoder.get_mut().position() < (input.len() as u64)); + decoder.get_mut().set_position(0); + return Ok(&output[..offset]); +} + +impl HTTP2DecoderHalf { + pub fn new() -> HTTP2DecoderHalf { + HTTP2DecoderHalf { + encoding: HTTP2ContentEncoding::HTTP2ContentEncodingUnknown, + decoder: HTTP2Decompresser::UNASSIGNED, + } + } + + pub fn http2_encoding_fromvec(&mut self, input: &Vec) { + //use first encoding... + if self.encoding == HTTP2ContentEncoding::HTTP2ContentEncodingUnknown { + if *input == "gzip".as_bytes().to_vec() { + self.encoding = HTTP2ContentEncoding::HTTP2ContentEncodingGzip; + self.decoder = HTTP2Decompresser::GZIP(GzDecoder::new(HTTP2cursor::new())); + } else if *input == "br".as_bytes().to_vec() { + self.encoding = HTTP2ContentEncoding::HTTP2ContentEncodingBr; + self.decoder = HTTP2Decompresser::BROTLI(brotli::Decompressor::new( + HTTP2cursor::new(), + HTTP2_DECOMPRESSION_CHUNK_SIZE, + )); + } else { + self.encoding = HTTP2ContentEncoding::HTTP2ContentEncodingUnrecognized; + } + } + } + + pub fn decompress<'a>( + &'a mut self, input: &'a [u8], output: &'a mut Vec, + ) -> io::Result<&'a [u8]> { + match self.decoder { + HTTP2Decompresser::GZIP(ref mut gzip_decoder) => { + return http2_decompress(gzip_decoder, input, output); + } + HTTP2Decompresser::BROTLI(ref mut br_decoder) => { + return http2_decompress(br_decoder, input, output); + } + _ => {} + } + return Ok(input); + } +} + +pub struct HTTP2Decoder { + decoder_tc: HTTP2DecoderHalf, + decoder_ts: HTTP2DecoderHalf, +} + +impl HTTP2Decoder { + pub fn new() -> HTTP2Decoder { + HTTP2Decoder { + decoder_tc: HTTP2DecoderHalf::new(), + decoder_ts: HTTP2DecoderHalf::new(), + } + } + + pub fn http2_encoding_fromvec(&mut self, input: &Vec, dir: u8) { + if dir == STREAM_TOCLIENT { + self.decoder_tc.http2_encoding_fromvec(input); + } else { + self.decoder_ts.http2_encoding_fromvec(input); + } + } + + pub fn decompress<'a>( + &'a mut self, input: &'a [u8], output: &'a mut Vec, dir: u8, + ) -> io::Result<&'a [u8]> { + if dir == STREAM_TOCLIENT { + return self.decoder_tc.decompress(input, output); + } else { + return self.decoder_ts.decompress(input, output); + } + } +} diff --git a/rust/src/http2/http2.rs b/rust/src/http2/http2.rs index 7c883fe049..47cb73231b 100644 --- a/rust/src/http2/http2.rs +++ b/rust/src/http2/http2.rs @@ -15,6 +15,7 @@ * 02110-1301, USA. */ +use super::decompression; use super::parser; use crate::applayer::{self, *}; use crate::core::{ @@ -27,6 +28,7 @@ use nom; use std; use std::ffi::{CStr, CString}; use std::fmt; +use std::io; use std::mem::transmute; static mut ALPROTO_HTTP2: AppProto = ALPROTO_UNKNOWN; @@ -124,6 +126,8 @@ pub struct HTTP2Transaction { pub frames_tc: Vec, pub frames_ts: Vec, + decoder: decompression::HTTP2Decoder, + de_state: Option<*mut core::DetectEngineState>, events: *mut core::AppLayerDecoderEvents, tx_data: AppLayerTxData, @@ -143,6 +147,7 @@ impl HTTP2Transaction { state: HTTP2TransactionState::HTTP2StateIdle, frames_tc: Vec::new(), frames_ts: Vec::new(), + decoder: decompression::HTTP2Decoder::new(), de_state: None, events: std::ptr::null_mut(), tx_data: AppLayerTxData::new(), @@ -160,6 +165,36 @@ impl HTTP2Transaction { } } + fn handle_headers(&mut self, blocks: &Vec, dir: u8) { + for i in 0..blocks.len() { + if blocks[i].name == "content-encoding".as_bytes().to_vec() { + self.decoder.http2_encoding_fromvec(&blocks[i].value, dir); + } + } + } + + fn decompress<'a>( + &'a mut self, input: &'a [u8], dir: u8, sfcm: &'static SuricataFileContext, over: bool, + files: &mut FileContainer, flags: u16, + ) -> io::Result<()> { + let mut output = Vec::with_capacity(decompression::HTTP2_DECOMPRESSION_CHUNK_SIZE); + let decompressed = self.decoder.decompress(input, &mut output, dir)?; + let xid: u32 = self.tx_id as u32; + self.ft.new_chunk( + sfcm, + files, + flags, + b"", + decompressed, + self.ft.tracked, //offset = append + decompressed.len() as u32, + 0, + over, + &xid, + ); + return Ok(()); + } + fn handle_frame( &mut self, header: &parser::HTTP2FrameHeader, data: &HTTP2FrameTypeData, dir: u8, ) { @@ -173,18 +208,21 @@ impl HTTP2Transaction { } self.state = HTTP2TransactionState::HTTP2StateReserved; } + self.handle_headers(&hs.blocks, dir); } - HTTP2FrameTypeData::CONTINUATION(_) => { + HTTP2FrameTypeData::CONTINUATION(hs) => { if dir == STREAM_TOCLIENT && header.flags & parser::HTTP2_FLAG_HEADER_END_HEADERS != 0 { self.child_stream_id = 0; } + self.handle_headers(&hs.blocks, dir); } - HTTP2FrameTypeData::HEADERS(_) => { + HTTP2FrameTypeData::HEADERS(hs) => { if dir == STREAM_TOCLIENT { self.child_stream_id = 0; } + self.handle_headers(&hs.blocks, dir); } HTTP2FrameTypeData::RSTSTREAM(_) => { self.child_stream_id = 0; @@ -253,6 +291,7 @@ pub enum HTTP2Event { LongFrameData, StreamIdReuse, InvalidHTTP1Settings, + FailedDecompression, } impl HTTP2Event { @@ -267,6 +306,7 @@ impl HTTP2Event { 6 => Some(HTTP2Event::LongFrameData), 7 => Some(HTTP2Event::StreamIdReuse), 8 => Some(HTTP2Event::InvalidHTTP1Settings), + 9 => Some(HTTP2Event::FailedDecompression), _ => None, } } @@ -767,21 +807,21 @@ impl HTTP2State { let index = self.find_tx_index(sid); if index > 0 { let mut tx_same = &mut self.transactions[index - 1]; - let xid: u32 = tx_same.tx_id as u32; tx_same.ft.tx_id = tx_same.tx_id - 1; let (files, flags) = self.files.get(dir); - tx_same.ft.new_chunk( + match tx_same.decompress( + &rem[..hlsafe], + dir, sfcm, + over, files, flags, - b"", - &rem[..hlsafe], - tx_same.ft.tracked, //offset = append - hlsafe as u32, - 0, - over, - &xid, - ); + ) { + Err(_e) => { + self.set_event(HTTP2Event::FailedDecompression); + } + _ => {} + } } } None => panic!("no SURICATA_HTTP2_FILE_CONFIG"), @@ -1052,6 +1092,7 @@ pub extern "C" fn rs_http2_state_get_event_info( "long_frame_data" => HTTP2Event::LongFrameData as i32, "stream_id_reuse" => HTTP2Event::StreamIdReuse as i32, "invalid_http1_settings" => HTTP2Event::InvalidHTTP1Settings as i32, + "failed_decompression" => HTTP2Event::FailedDecompression as i32, _ => -1, // unknown event } } @@ -1080,6 +1121,7 @@ pub extern "C" fn rs_http2_state_get_event_info_by_id( HTTP2Event::LongFrameData => "long_frame_data\0", HTTP2Event::StreamIdReuse => "stream_id_reuse\0", HTTP2Event::InvalidHTTP1Settings => "invalid_http1_settings\0", + HTTP2Event::FailedDecompression => "failed_decompression\0", }; unsafe { *event_name = estr.as_ptr() as *const std::os::raw::c_char; diff --git a/rust/src/http2/mod.rs b/rust/src/http2/mod.rs index f6f5aba1f2..08a5f32770 100644 --- a/rust/src/http2/mod.rs +++ b/rust/src/http2/mod.rs @@ -15,6 +15,7 @@ * 02110-1301, USA. */ +mod decompression; pub mod detect; pub mod http2; mod huffman; diff --git a/src/suricata.c b/src/suricata.c index 0d3e8a67ea..548d4c238c 100644 --- a/src/suricata.c +++ b/src/suricata.c @@ -694,6 +694,8 @@ static void PrintBuildInfo(void) #endif /* For compatibility, just say we have HAVE_NSS. */ strlcat(features, "HAVE_NSS ", sizeof(features)); + /* HTTP2_DECOMPRESSION is not an optional feature in this major version */ + strlcat(features, "HTTP2_DECOMPRESSION ", sizeof(features)); #ifdef HAVE_LUA strlcat(features, "HAVE_LUA ", sizeof(features)); #endif