]> git.ipfire.org Git - thirdparty/suricata.git/commitdiff
http2: decompression for files
authorPhilippe Antoine <contact@catenacyber.fr>
Thu, 12 Nov 2020 08:24:36 +0000 (09:24 +0100)
committerVictor Julien <victor@inliniac.net>
Mon, 7 Jun 2021 08:58:05 +0000 (10:58 +0200)
gzip and brotli decompression for files

(cherry picked from commit d8612282149df27be47f7aaba906ea08dcc3952e)

rules/http2-events.rules
rust/Cargo.toml.in
rust/src/http2/decompression.rs [new file with mode: 0644]
rust/src/http2/http2.rs
rust/src/http2/mod.rs
src/suricata.c

index bb2d08ded58393171148cb1fa15ce286c2537ea4..97961da38fc620eba7c3d761adc0b9aaeab3b202 100644 (file)
@@ -14,3 +14,4 @@ alert http2 any any -> any any (msg:"SURICATA HTTP2 header frame with extra data
 alert http2 any any -> any any (msg:"SURICATA HTTP2 too long frame data"; flow:established; app-layer-event:http2.long_frame_data; classtype:protocol-command-decode; sid:2290006; rev:1;)
 alert http2 any any -> any any (msg:"SURICATA HTTP2 stream identifier reuse"; flow:established; app-layer-event:http2.stream_id_reuse; classtype:protocol-command-decode; sid:2290007; rev:1;)
 alert http2 any any -> any any (msg:"SURICATA HTTP2 invalid HTTP1 settings during upgrade"; flow:established; app-layer-event:http2.invalid_http1_settings; classtype:protocol-command-decode; sid:2290008; rev:1;)
+alert http2 any any -> any any (msg:"SURICATA HTTP2 failed decompression"; flow:established; app-layer-event:http2.failed_decompression; classtype:protocol-command-decode; sid:2290009; rev:1;)
index c1b7e9d9ba5c2ad445366eb562e5648b6f844dc1..2dff06ee3acf73489631d9aba8da3799f3197ad3 100644 (file)
@@ -30,6 +30,8 @@ num-derive = "0.2"
 num-traits = "0.2"
 widestring = "0.4"
 md5 = "0.7.0"
+flate2 = "1.0"
+brotli = "3.3.0"
 
 der-parser = "4.0"
 kerberos-parser = "0.5"
diff --git a/rust/src/http2/decompression.rs b/rust/src/http2/decompression.rs
new file mode 100644 (file)
index 0000000..205b527
--- /dev/null
@@ -0,0 +1,216 @@
+/* Copyright (C) 2021 Open Information Security Foundation
+*
+* You can copy, redistribute or modify this Program under the terms of
+* the GNU General Public License version 2 as published by the Free
+* Software Foundation.
+*
+* This program is distributed in the hope that it will be useful,
+* but WITHOUT ANY WARRANTY; without even the implied warranty of
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+* GNU General Public License for more details.
+*
+* You should have received a copy of the GNU General Public License
+* version 2 along with this program; if not, write to the Free Software
+* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+* 02110-1301, USA.
+*/
+
+use crate::core::STREAM_TOCLIENT;
+use brotli;
+use flate2::read::GzDecoder;
+use std;
+use std::io;
+use std::io::{Cursor, Read, Write};
+
+pub const HTTP2_DECOMPRESSION_CHUNK_SIZE: usize = 0x1000; // 4096
+
+#[repr(u8)]
+#[derive(Copy, Clone, PartialOrd, PartialEq, Debug)]
+pub enum HTTP2ContentEncoding {
+    HTTP2ContentEncodingUnknown = 0,
+    HTTP2ContentEncodingGzip = 1,
+    HTTP2ContentEncodingBr = 2,
+    HTTP2ContentEncodingUnrecognized = 3,
+}
+
+//a cursor turning EOF into blocking errors
+pub struct HTTP2cursor {
+    pub cursor: Cursor<Vec<u8>>,
+}
+
+impl HTTP2cursor {
+    pub fn new() -> HTTP2cursor {
+        HTTP2cursor {
+            cursor: Cursor::new(Vec::new()),
+        }
+    }
+
+    #[cfg(feature = "debug-validate")]
+    pub fn position(&self) -> u64 {
+        return self.cursor.position();
+    }
+
+    pub fn set_position(&mut self, pos: u64) {
+        return self.cursor.set_position(pos);
+    }
+}
+
+// we need to implement this as flate2 and brotli crates
+// will read from this object
+impl Read for HTTP2cursor {
+    fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
+        //use the cursor, except it turns eof into blocking error
+        let r = self.cursor.read(buf);
+        match r {
+            Err(ref err) => {
+                if err.kind() == io::ErrorKind::UnexpectedEof {
+                    return Err(io::ErrorKind::WouldBlock.into());
+                }
+            }
+            Ok(0) => {
+                //regular EOF turned into blocking error
+                return Err(io::ErrorKind::WouldBlock.into());
+            }
+            Ok(_n) => {}
+        }
+        return r;
+    }
+}
+
+pub enum HTTP2Decompresser {
+    UNASSIGNED,
+    GZIP(GzDecoder<HTTP2cursor>),
+    BROTLI(brotli::Decompressor<HTTP2cursor>),
+}
+
+struct HTTP2DecoderHalf {
+    encoding: HTTP2ContentEncoding,
+    decoder: HTTP2Decompresser,
+}
+
+pub trait GetMutCursor {
+    fn get_mut(&mut self) -> &mut HTTP2cursor;
+}
+
+impl GetMutCursor for GzDecoder<HTTP2cursor> {
+    fn get_mut(&mut self) -> &mut HTTP2cursor {
+        return self.get_mut();
+    }
+}
+
+impl GetMutCursor for brotli::Decompressor<HTTP2cursor> {
+    fn get_mut(&mut self) -> &mut HTTP2cursor {
+        return self.get_mut();
+    }
+}
+
+fn http2_decompress<'a>(
+    decoder: &mut (impl Read + GetMutCursor), input: &'a [u8], output: &'a mut Vec<u8>,
+) -> io::Result<&'a [u8]> {
+    match decoder.get_mut().cursor.write_all(input) {
+        Ok(()) => {}
+        Err(e) => {
+            return Err(e);
+        }
+    }
+    let mut offset = 0;
+    decoder.get_mut().set_position(0);
+    output.resize(HTTP2_DECOMPRESSION_CHUNK_SIZE, 0);
+    loop {
+        match decoder.read(&mut output[offset..]) {
+            Ok(0) => {
+                break;
+            }
+            Ok(n) => {
+                offset += n;
+                if offset == output.len() {
+                    output.resize(output.len() + HTTP2_DECOMPRESSION_CHUNK_SIZE, 0);
+                }
+            }
+            Err(e) => {
+                if e.kind() == io::ErrorKind::WouldBlock {
+                    break;
+                }
+                return Err(e);
+            }
+        }
+    }
+    //checks all input was consumed
+    debug_validate_bug_on!(decoder.get_mut().position() < (input.len() as u64));
+    decoder.get_mut().set_position(0);
+    return Ok(&output[..offset]);
+}
+
+impl HTTP2DecoderHalf {
+    pub fn new() -> HTTP2DecoderHalf {
+        HTTP2DecoderHalf {
+            encoding: HTTP2ContentEncoding::HTTP2ContentEncodingUnknown,
+            decoder: HTTP2Decompresser::UNASSIGNED,
+        }
+    }
+
+    pub fn http2_encoding_fromvec(&mut self, input: &Vec<u8>) {
+        //use first encoding...
+        if self.encoding == HTTP2ContentEncoding::HTTP2ContentEncodingUnknown {
+            if *input == "gzip".as_bytes().to_vec() {
+                self.encoding = HTTP2ContentEncoding::HTTP2ContentEncodingGzip;
+                self.decoder = HTTP2Decompresser::GZIP(GzDecoder::new(HTTP2cursor::new()));
+            } else if *input == "br".as_bytes().to_vec() {
+                self.encoding = HTTP2ContentEncoding::HTTP2ContentEncodingBr;
+                self.decoder = HTTP2Decompresser::BROTLI(brotli::Decompressor::new(
+                    HTTP2cursor::new(),
+                    HTTP2_DECOMPRESSION_CHUNK_SIZE,
+                ));
+            } else {
+                self.encoding = HTTP2ContentEncoding::HTTP2ContentEncodingUnrecognized;
+            }
+        }
+    }
+
+    pub fn decompress<'a>(
+        &'a mut self, input: &'a [u8], output: &'a mut Vec<u8>,
+    ) -> io::Result<&'a [u8]> {
+        match self.decoder {
+            HTTP2Decompresser::GZIP(ref mut gzip_decoder) => {
+                return http2_decompress(gzip_decoder, input, output);
+            }
+            HTTP2Decompresser::BROTLI(ref mut br_decoder) => {
+                return http2_decompress(br_decoder, input, output);
+            }
+            _ => {}
+        }
+        return Ok(input);
+    }
+}
+
+pub struct HTTP2Decoder {
+    decoder_tc: HTTP2DecoderHalf,
+    decoder_ts: HTTP2DecoderHalf,
+}
+
+impl HTTP2Decoder {
+    pub fn new() -> HTTP2Decoder {
+        HTTP2Decoder {
+            decoder_tc: HTTP2DecoderHalf::new(),
+            decoder_ts: HTTP2DecoderHalf::new(),
+        }
+    }
+
+    pub fn http2_encoding_fromvec(&mut self, input: &Vec<u8>, dir: u8) {
+        if dir == STREAM_TOCLIENT {
+            self.decoder_tc.http2_encoding_fromvec(input);
+        } else {
+            self.decoder_ts.http2_encoding_fromvec(input);
+        }
+    }
+
+    pub fn decompress<'a>(
+        &'a mut self, input: &'a [u8], output: &'a mut Vec<u8>, dir: u8,
+    ) -> io::Result<&'a [u8]> {
+        if dir == STREAM_TOCLIENT {
+            return self.decoder_tc.decompress(input, output);
+        } else {
+            return self.decoder_ts.decompress(input, output);
+        }
+    }
+}
index 99c6165493d85ebd76f7fb1887b45d57de109b0d..a726bfeff4e9aba4a1c19939c11900ceed310ee4 100644 (file)
@@ -16,6 +16,7 @@
  */
 
 use super::files::*;
+use super::decompression;
 use super::parser;
 use crate::applayer::{self, *};
 use crate::core::{
@@ -28,6 +29,7 @@ use nom;
 use std;
 use std::ffi::{CStr, CString};
 use std::fmt;
+use std::io;
 use std::mem::transmute;
 
 static mut ALPROTO_HTTP2: AppProto = ALPROTO_UNKNOWN;
@@ -125,6 +127,8 @@ pub struct HTTP2Transaction {
     pub frames_tc: Vec<HTTP2Frame>,
     pub frames_ts: Vec<HTTP2Frame>,
 
+    decoder: decompression::HTTP2Decoder,
+
     de_state: Option<*mut core::DetectEngineState>,
     events: *mut core::AppLayerDecoderEvents,
     tx_data: AppLayerTxData,
@@ -144,6 +148,7 @@ impl HTTP2Transaction {
             state: HTTP2TransactionState::HTTP2StateIdle,
             frames_tc: Vec::new(),
             frames_ts: Vec::new(),
+            decoder: decompression::HTTP2Decoder::new(),
             de_state: None,
             events: std::ptr::null_mut(),
             tx_data: AppLayerTxData::new(),
@@ -161,6 +166,36 @@ impl HTTP2Transaction {
         }
     }
 
+    fn handle_headers(&mut self, blocks: &Vec<parser::HTTP2FrameHeaderBlock>, dir: u8) {
+        for i in 0..blocks.len() {
+            if blocks[i].name == "content-encoding".as_bytes().to_vec() {
+                self.decoder.http2_encoding_fromvec(&blocks[i].value, dir);
+            }
+        }
+    }
+
+    fn decompress<'a>(
+        &'a mut self, input: &'a [u8], dir: u8, sfcm: &'static SuricataFileContext, over: bool,
+        files: &mut FileContainer, flags: u16,
+    ) -> io::Result<()> {
+        let mut output = Vec::with_capacity(decompression::HTTP2_DECOMPRESSION_CHUNK_SIZE);
+        let decompressed = self.decoder.decompress(input, &mut output, dir)?;
+        let xid: u32 = self.tx_id as u32;
+        self.ft.new_chunk(
+            sfcm,
+            files,
+            flags,
+            b"",
+            decompressed,
+            self.ft.tracked, //offset = append
+            decompressed.len() as u32,
+            0,
+            over,
+            &xid,
+        );
+        return Ok(());
+    }
+
     fn handle_frame(
         &mut self, header: &parser::HTTP2FrameHeader, data: &HTTP2FrameTypeData, dir: u8,
     ) {
@@ -174,18 +209,21 @@ impl HTTP2Transaction {
                     }
                     self.state = HTTP2TransactionState::HTTP2StateReserved;
                 }
+                self.handle_headers(&hs.blocks, dir);
             }
-            HTTP2FrameTypeData::CONTINUATION(_) => {
+            HTTP2FrameTypeData::CONTINUATION(hs) => {
                 if dir == STREAM_TOCLIENT
                     && header.flags & parser::HTTP2_FLAG_HEADER_END_HEADERS != 0
                 {
                     self.child_stream_id = 0;
                 }
+                self.handle_headers(&hs.blocks, dir);
             }
-            HTTP2FrameTypeData::HEADERS(_) => {
+            HTTP2FrameTypeData::HEADERS(hs) => {
                 if dir == STREAM_TOCLIENT {
                     self.child_stream_id = 0;
                 }
+                self.handle_headers(&hs.blocks, dir);
             }
             HTTP2FrameTypeData::RSTSTREAM(_) => {
                 self.child_stream_id = 0;
@@ -254,6 +292,7 @@ pub enum HTTP2Event {
     LongFrameData,
     StreamIdReuse,
     InvalidHTTP1Settings,
+    FailedDecompression,
 }
 
 impl HTTP2Event {
@@ -268,6 +307,7 @@ impl HTTP2Event {
             6 => Some(HTTP2Event::LongFrameData),
             7 => Some(HTTP2Event::StreamIdReuse),
             8 => Some(HTTP2Event::InvalidHTTP1Settings),
+            9 => Some(HTTP2Event::FailedDecompression),
             _ => None,
         }
     }
@@ -768,21 +808,21 @@ impl HTTP2State {
                                 let index = self.find_tx_index(sid);
                                 if index > 0 {
                                     let mut tx_same = &mut self.transactions[index - 1];
-                                    let xid: u32 = tx_same.tx_id as u32;
                                     tx_same.ft.tx_id = tx_same.tx_id - 1;
                                     let (files, flags) = self.files.get(dir);
-                                    tx_same.ft.new_chunk(
+                                    match tx_same.decompress(
+                                        &rem[..hlsafe],
+                                        dir,
                                         sfcm,
+                                        over,
                                         files,
                                         flags,
-                                        b"",
-                                        &rem[..hlsafe],
-                                        tx_same.ft.tracked, //offset = append
-                                        hlsafe as u32,
-                                        0,
-                                        over,
-                                        &xid,
-                                    );
+                                    ) {
+                                        Err(_e) => {
+                                            self.set_event(HTTP2Event::FailedDecompression);
+                                        }
+                                        _ => {}
+                                    }
                                 }
                             }
                             None => panic!("no SURICATA_HTTP2_FILE_CONFIG"),
@@ -1058,6 +1098,7 @@ pub extern "C" fn rs_http2_state_get_event_info(
                 "long_frame_data" => HTTP2Event::LongFrameData as i32,
                 "stream_id_reuse" => HTTP2Event::StreamIdReuse as i32,
                 "invalid_http1_settings" => HTTP2Event::InvalidHTTP1Settings as i32,
+                "failed_decompression" => HTTP2Event::FailedDecompression as i32,
                 _ => -1, // unknown event
             }
         }
@@ -1086,6 +1127,7 @@ pub extern "C" fn rs_http2_state_get_event_info_by_id(
             HTTP2Event::LongFrameData => "long_frame_data\0",
             HTTP2Event::StreamIdReuse => "stream_id_reuse\0",
             HTTP2Event::InvalidHTTP1Settings => "invalid_http1_settings\0",
+            HTTP2Event::FailedDecompression => "failed_decompression\0",
         };
         unsafe {
             *event_name = estr.as_ptr() as *const std::os::raw::c_char;
index ad86bdf291ce97aba4054d7e94ee4bc982d88628..d6dcb55eca474c5fc8a0d6cf0cfdb229befaf6ed 100644 (file)
@@ -15,6 +15,7 @@
  * 02110-1301, USA.
  */
 
+mod decompression;
 pub mod detect;
 pub mod files;
 pub mod http2;
index 5728da50251ae089eaf9e56ea59bd6da1113e874..3187e0a9dd03bde5041b36a577220de909815b1e 100644 (file)
@@ -704,6 +704,8 @@ static void PrintBuildInfo(void)
 #ifdef HAVE_NSS
     strlcat(features, "HAVE_NSS ", sizeof(features));
 #endif
+    /* HTTP2_DECOMPRESSION is not an optional feature in this major version */
+    strlcat(features, "HTTP2_DECOMPRESSION ", sizeof(features));
 #ifdef HAVE_LUA
     strlcat(features, "HAVE_LUA ", sizeof(features));
 #endif