From: Pranav Bhalerao (prbhaler) Date: Tue, 29 Sep 2020 05:51:40 +0000 (+0000) Subject: Merge pull request #2463 in SNORT/snort3 from ~ABHPAL/snort3:feature/custom_xff_heade... X-Git-Tag: 3.0.3-2~21 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=e23be1b306539363624a3ff90030df27076f3972;p=thirdparty%2Fsnort3.git Merge pull request #2463 in SNORT/snort3 from ~ABHPAL/snort3:feature/custom_xff_header_support to master Squashed commit of the following: commit 7aec7eef7656af547f44efe8fcd9ab1dcb31a948 Author: Abhijit Pal Date: Mon Sep 7 08:01:04 2020 -0400 http_inspect: support for custom xff type headers --- diff --git a/doc/user/http_inspect.txt b/doc/user/http_inspect.txt old mode 100644 new mode 100755 index 674248e82..9733c2d74 --- a/doc/user/http_inspect.txt +++ b/doc/user/http_inspect.txt @@ -163,6 +163,20 @@ the unescape, decodeURI, or decodeURIComponent are %XX, %uXXXX, XX and uXXXXi. http_inspect also replaces consecutive whitespaces with a single space and normalizes the plus by concatenating the strings. +===== xff_headers + +This configuration supports defining custom x-forwarded-for type headers. In a +multi-vendor world, it is quite possible that the header name carrying the +original client IP could be vendor-specific. This is due to the absence +of standardization which would otherwise standardize the header name. +In such a scenario, this configuration provides a way with which such headers +can be introduced to HI. The default value of this configuration +is "x-forwarded-for true-client-ip". The default definition introduces the +two commonly known headers and is preferred in the same order by the +inspector as they are defined, e.g "x-forwarded-for" will be preferred than +"true-client-ip" if both headers are present in the stream. The header names +should be delimited by a space. + ===== URI processing Normalization and inspection of the URI in the HTTP request message is a @@ -454,8 +468,9 @@ to http_header when no specific header is specified. This provides the original IP address of the client sending the request as it was stored by a proxy in the request message headers. Specifically it -is the last IP address listed in the X-Forwarded-For or True-Client-IP -header. If both headers are present the former is used. +is the last IP address listed in the X-Forwarded-For, True-Client-IP or +any other custom x-forwarded-for type header. If multiple headers are present the +preference defined in xff_headers configuration is considered. ===== http_client_body diff --git a/src/service_inspectors/http_inspect/dev_notes.txt b/src/service_inspectors/http_inspect/dev_notes.txt old mode 100644 new mode 100755 index fa91439af..5833d8003 --- a/src/service_inspectors/http_inspect/dev_notes.txt +++ b/src/service_inspectors/http_inspect/dev_notes.txt @@ -85,6 +85,19 @@ Message sections implement the Just-In-Time (JIT) principle for work products. A essential processing is done under process(). Other work products are derived and stored the first time detection or some other customer asks for them. +HI also supports defining custom "x-forwarded-for" type headers. In a multi-vendor world, it is +quite possible that the header name carrying the original client IP could be vendor-specific. This +is due to the absence of standardization which would otherwise standardize the header name. In such +a scenario, it is important to provide a configuration with which such x-forwarded-for type headers +can be introduced to HI. The headers can be introduced with the xff_headers configuration. The default +value of this configuration is "x-forwarded-for true-client-ip". The default definition introduces +the two commonly known "x-forwarded-for" type headers and is preferred in the same order by the +inspector as they are defined, e.g "x-forwarded-for" will be preferred than "true-client-ip" if +both headers are present in the stream. Every HTTP Header is mapped to an ID internally. The +custom headers are mapped to a dynamically generated ID and the mapping is appended at the end +of the mapping of the known HTTP headers. Every HI instance can have its own list of custom +headers and thus an instance of HTTP header mapping list is also associated with an HI instance. + The Field class is an important tool for managing JIT. It consists of a pointer to a raw message field or derived work product with a length field. Various negative length values specify the status of the field. For instance STAT_NOTCOMPUTE means the item has not been computed yet, diff --git a/src/service_inspectors/http_inspect/http_enum.h b/src/service_inspectors/http_inspect/http_enum.h old mode 100644 new mode 100755 index 8bc7e142e..4e0a8b956 --- a/src/service_inspectors/http_inspect/http_enum.h +++ b/src/service_inspectors/http_inspect/http_enum.h @@ -32,6 +32,9 @@ static const uint32_t HTTP_GID = 119; static const int GZIP_WINDOW_BITS = 31; static const int DEFLATE_WINDOW_BITS = 15; static const int MAX_FIELD_NAME_LENGTH = 100; +// Plan to support max 8 xff headers +static const uint8_t MAX_XFF_HEADERS = 8; +static const uint8_t MAX_CUSTOM_HEADERS = MAX_XFF_HEADERS; // This can grow into a bitmap for the get_buf() form parameter static const uint64_t FORM_REQUEST = 0x1; diff --git a/src/service_inspectors/http_inspect/http_inspect.cc b/src/service_inspectors/http_inspect/http_inspect.cc old mode 100644 new mode 100755 index 1967e336b..0241e3ccf --- a/src/service_inspectors/http_inspect/http_inspect.cc +++ b/src/service_inspectors/http_inspect/http_inspect.cc @@ -82,6 +82,24 @@ static std::string GetBadChars(const ByteBitSet& bitset) return str; } + +static std::string GetXFFHeaders(const StrCode *header_list) +{ + std::string hdr_list; + for (int idx = 0; header_list[idx].code; idx++) + { + hdr_list += header_list[idx].name; + hdr_list += " "; + } + + // Remove the trailing whitespace, if any + if (hdr_list.length()) + { + hdr_list.pop_back(); + } + return hdr_list; +} + HttpInspect::HttpInspect(const HttpParaList* params_) : params(params_), xtra_trueip_id(Stream::reg_xtra_data_cb(get_xtra_trueip)), @@ -122,6 +140,7 @@ void HttpInspect::show(const SnortConfig*) const auto unreserved_chars = GetUnreservedChars(params->uri_param.unreserved_char); auto bad_chars = GetBadChars(params->uri_param.bad_characters); + auto xff_headers = GetXFFHeaders(params->xff_headers); ConfigLogger::log_limit("request_depth", params->request_depth, -1LL); ConfigLogger::log_limit("response_depth", params->response_depth, -1LL); @@ -148,6 +167,7 @@ void HttpInspect::show(const SnortConfig*) const ConfigLogger::log_flag("backslash_to_slash", params->uri_param.backslash_to_slash); ConfigLogger::log_flag("plus_to_space", params->uri_param.plus_to_space); ConfigLogger::log_flag("simplify_path", params->uri_param.simplify_path); + ConfigLogger::log_value("xff_headers", xff_headers.c_str()); } InspectSection HttpInspect::get_latest_is(const Packet* p) diff --git a/src/service_inspectors/http_inspect/http_module.cc b/src/service_inspectors/http_inspect/http_module.cc old mode 100644 new mode 100755 index af6016b68..98eba4905 --- a/src/service_inspectors/http_inspect/http_module.cc +++ b/src/service_inspectors/http_inspect/http_module.cc @@ -29,6 +29,7 @@ #include "http_enum.h" #include "http_js_norm.h" #include "http_uri_norm.h" +#include "http_msg_head_shared.h" using namespace snort; using namespace HttpEnums; @@ -139,6 +140,9 @@ const Parameter HttpModule::http_params[] = { "simplify_path", Parameter::PT_BOOL, nullptr, "true", "reduce URI directory path to simplest form" }, + { "xff_headers", Parameter::PT_STRING, nullptr, "x-forwarded-for true-client-ip", + "specifies the xff type headers to parse and consider in the same order " + "of preference as defined" }, #ifdef REG_TEST { "test_input", Parameter::PT_BOOL, nullptr, "false", "read HTTP messages from text file" }, @@ -282,6 +286,42 @@ bool HttpModule::set(const char*, Value& val, SnortConfig*) params->uri_param.uri_char[(uint8_t)'/'] = val.get_bool() ? CHAR_PATH : CHAR_NORMAL; params->uri_param.uri_char[(uint8_t)'.'] = val.get_bool() ? CHAR_PATH : CHAR_NORMAL; } + else if (val.is("xff_headers")) + { + std::string header; + int custom_id_idx = 1; + int hdr_idx; + StrCode end_header = {0, nullptr}; + + // Delete the default params if any + for (int idx = 0; params->xff_headers[idx].code; idx++) + { + params->xff_headers[idx].code = 0; + delete[] params->xff_headers[idx].name; + } + + // The configured text should be converted to lower case as the header + // text comparison is lower case sensitive + val.lower(); + + // Tokenize the entered config. Every space separated value is a custom xff header and is + // preferred in the order in which it is configured + val.set_first_token(); + for (hdr_idx = 0; val.get_next_token(header) && (hdr_idx < MAX_XFF_HEADERS); hdr_idx++) + { + int hdr_id; + hdr_id = str_to_code(header.c_str(), HttpMsgHeadShared::header_list); + hdr_id = (hdr_id != HttpCommon::STAT_OTHER) ? hdr_id : (HEAD__MAX_VALUE + custom_id_idx++); + + // Copy the custom header params to the params list. The custom + // headers from this list would be appended to the instance specific + // header_list + params->xff_headers[hdr_idx].code = hdr_id; + params->xff_headers[hdr_idx].name = new char[header.length() + 1]; + strcpy(const_cast(params->xff_headers[hdr_idx].name), header.c_str()); + } + params->xff_headers[hdr_idx] = end_header; + } #ifdef REG_TEST else if (val.is("test_input")) { @@ -315,6 +355,31 @@ bool HttpModule::set(const char*, Value& val, SnortConfig*) return true; } +static void prepare_http_header_list(HttpParaList* params) +{ + int32_t hdr_idx; + StrCode end_header = {0, nullptr}; + + // Copy the global header_list + for (hdr_idx = 0; HttpMsgHeadShared::header_list[hdr_idx].code ; hdr_idx++) + { + params->header_list[hdr_idx] = HttpMsgHeadShared::header_list[hdr_idx]; + } + + // Copy the custom xff headers to the header list except the known headers + for (int32_t idx = 0; params->xff_headers[idx].code; idx++) + { + int32_t code = str_to_code(params->xff_headers[idx].name, HttpMsgHeadShared::header_list); + if (code == HttpCommon::STAT_OTHER) + { + params->header_list[hdr_idx++] = params->xff_headers[idx]; + } + } + + // A dummy header object to mark the end of the list + params->header_list[hdr_idx] = end_header; +} + bool HttpModule::end(const char*, int, SnortConfig*) { if (!params->uri_param.utf8 && params->uri_param.utf8_bare_byte) @@ -343,9 +408,20 @@ bool HttpModule::end(const char*, int, SnortConfig*) params->js_norm_param.js_norm = new HttpJsNorm(params->js_norm_param.max_javascript_whitespaces, params->uri_param); } + + prepare_http_header_list(params); + return true; } +HttpParaList::~HttpParaList() +{ + for (int idx = 0; xff_headers[idx].code; idx++) + { + delete[] xff_headers[idx].name; + } +} + HttpParaList::JsNormParam::~JsNormParam() { delete js_norm; diff --git a/src/service_inspectors/http_inspect/http_module.h b/src/service_inspectors/http_inspect/http_module.h old mode 100644 new mode 100755 index 24d4834ff..b20ea5cdb --- a/src/service_inspectors/http_inspect/http_module.h +++ b/src/service_inspectors/http_inspect/http_module.h @@ -28,6 +28,7 @@ #include "profiler/profiler.h" #include "http_enum.h" +#include "http_str_to_code.h" #define HTTP_NAME "http_inspect" #define HTTP_HELP "HTTP inspector" @@ -35,6 +36,7 @@ struct HttpParaList { public: + ~HttpParaList(); int64_t request_depth = -1; int64_t response_depth = -1; @@ -82,6 +84,19 @@ public: }; UriParam uri_param; + // This will store list of custom xff headers. These are stored in the + // order of the header preference. The default header preference only + // consists of known XFF Headers in the below order + // 1. X-Forwarded-For + // 2. True-Client-IP + // Rest of the custom XFF Headers would be added to this list and will be + // positioned based on the preference of the headers. + // As of now, plan is to support a maximum of 8 xff type headers. + StrCode xff_headers[HttpEnums::MAX_XFF_HEADERS + 1] = {}; + // The below header_list contains the list of known static header along with + // any custom headers mapped with the their respective Header IDs. + StrCode header_list[HttpEnums::HEAD__MAX_VALUE + HttpEnums::MAX_CUSTOM_HEADERS + 1] = {}; + #ifdef REG_TEST int64_t print_amount = 1200; diff --git a/src/service_inspectors/http_inspect/http_msg_head_shared.cc b/src/service_inspectors/http_inspect/http_msg_head_shared.cc old mode 100644 new mode 100755 index d5d625fd0..b80f3c4f4 --- a/src/service_inspectors/http_inspect/http_msg_head_shared.cc +++ b/src/service_inspectors/http_inspect/http_msg_head_shared.cc @@ -239,7 +239,7 @@ void HttpMsgHeadShared::derive_header_name_id(int index) create_event(EVENT_HEAD_NAME_WHITESPACE); } } - header_name_id[index] = (HeaderId)str_to_code(lower_name, lower_length, header_list); + header_name_id[index] = (HeaderId)str_to_code(lower_name, lower_length, params->header_list); delete[] lower_name; } diff --git a/src/service_inspectors/http_inspect/http_msg_head_shared.h b/src/service_inspectors/http_inspect/http_msg_head_shared.h old mode 100644 new mode 100755 index f8aebc146..fe06b4d96 --- a/src/service_inspectors/http_inspect/http_msg_head_shared.h +++ b/src/service_inspectors/http_inspect/http_msg_head_shared.h @@ -74,7 +74,7 @@ protected: #endif private: - static const int MAX = HttpEnums::HEAD__MAX_VALUE; + static const int MAX = HttpEnums::HEAD__MAX_VALUE + HttpEnums::MAX_CUSTOM_HEADERS; // Header normalization strategies. There should be one defined for every different way we can // process a header field value. diff --git a/src/service_inspectors/http_inspect/http_msg_header.cc b/src/service_inspectors/http_inspect/http_msg_header.cc old mode 100644 new mode 100755 index e34dc66a9..74ab1e0ee --- a/src/service_inspectors/http_inspect/http_msg_header.cc +++ b/src/service_inspectors/http_inspect/http_msg_header.cc @@ -70,22 +70,24 @@ const Field& HttpMsgHeader::get_true_ip() if (true_ip.length() != STAT_NOT_COMPUTE) return true_ip; - const Field* header_to_use; - const Field& xff = get_header_value_norm(HEAD_X_FORWARDED_FOR); - if (xff.length() > 0) - header_to_use = &xff; - else + const Field* header_to_use = nullptr; + + for (int idx = 0; params->xff_headers[idx].code; idx++) { - const Field& tcip = get_header_value_norm(HEAD_TRUE_CLIENT_IP); - if (tcip.length() > 0) - header_to_use = &tcip; - else + const Field& xff = get_header_value_norm((HeaderId)params->xff_headers[idx].code); + if (xff.length() > 0) { - true_ip.set(STAT_NOT_PRESENT); - return true_ip; + header_to_use = &xff; + break; } } + if (!header_to_use) + { + true_ip.set(STAT_NOT_PRESENT); + return true_ip; + } + // This is potentially a comma-separated list of IP addresses. Take the last one in the list. // Since this is a normalized header field any whitespace will be an actual space. int32_t length; diff --git a/src/service_inspectors/http_inspect/http_str_to_code.cc b/src/service_inspectors/http_inspect/http_str_to_code.cc old mode 100644 new mode 100755 index 82bb5391c..230ec5972 --- a/src/service_inspectors/http_inspect/http_str_to_code.cc +++ b/src/service_inspectors/http_inspect/http_str_to_code.cc @@ -27,6 +27,11 @@ #include "http_common.h" +int32_t str_to_code(const char* text, const StrCode table[]) +{ + return str_to_code((const uint8_t*)text, strlen(text), table); +} + // Need to replace this simple algorithm for better performance FIXIT-P int32_t str_to_code(const uint8_t* text, const int32_t text_len, const StrCode table[]) { diff --git a/src/service_inspectors/http_inspect/http_str_to_code.h b/src/service_inspectors/http_inspect/http_str_to_code.h old mode 100644 new mode 100755 index 1efcfc93a..8e783ad86 --- a/src/service_inspectors/http_inspect/http_str_to_code.h +++ b/src/service_inspectors/http_inspect/http_str_to_code.h @@ -28,6 +28,7 @@ struct StrCode const char* name; }; +int32_t str_to_code(const char* text, const StrCode table[]); int32_t str_to_code(const uint8_t* text, const int32_t text_len, const StrCode table[]); int32_t substr_to_code(const uint8_t* text, const int32_t text_len, const StrCode table[]); diff --git a/src/service_inspectors/http_inspect/http_tables.cc b/src/service_inspectors/http_inspect/http_tables.cc old mode 100644 new mode 100755 index a0c12f688..683e82c4d --- a/src/service_inspectors/http_inspect/http_tables.cc +++ b/src/service_inspectors/http_inspect/http_tables.cc @@ -216,7 +216,7 @@ const HeaderNormalizer HttpMsgHeadShared::NORMALIZER_CONTENT_LENGTH const HeaderNormalizer HttpMsgHeadShared::NORMALIZER_CHARSET { EVENT__NONE, INF__NONE, false, norm_remove_quotes_lws, norm_to_lower, nullptr }; -const HeaderNormalizer* const HttpMsgHeadShared::header_norms[HEAD__MAX_VALUE] = { +const HeaderNormalizer* const HttpMsgHeadShared::header_norms[HEAD__MAX_VALUE + MAX_CUSTOM_HEADERS + 1] = { &NORMALIZER_BASIC, // 0 &NORMALIZER_BASIC, // HEAD__OTHER &NORMALIZER_TOKEN_LIST, // HEAD_CACHE_CONTROL @@ -275,6 +275,15 @@ const HeaderNormalizer* const HttpMsgHeadShared::header_norms[HEAD__MAX_VALUE] = &NORMALIZER_BASIC, // HEAD_MIME_VERSION &NORMALIZER_BASIC, // HEAD_PROXY_AGENT &NORMALIZER_BASIC, // HEAD_CONTENT_DISPOSITION + &NORMALIZER_BASIC, // HEAD__MAX_VALUE + &NORMALIZER_BASIC, // HEAD_CUSTOM_XFF_HEADER + &NORMALIZER_BASIC, // HEAD_CUSTOM_XFF_HEADER + &NORMALIZER_BASIC, // HEAD_CUSTOM_XFF_HEADER + &NORMALIZER_BASIC, // HEAD_CUSTOM_XFF_HEADER + &NORMALIZER_BASIC, // HEAD_CUSTOM_XFF_HEADER + &NORMALIZER_BASIC, // HEAD_CUSTOM_XFF_HEADER + &NORMALIZER_BASIC, // HEAD_CUSTOM_XFF_HEADER + &NORMALIZER_BASIC, // HEAD_CUSTOM_XFF_HEADER }; const RuleMap HttpModule::http_events[] = diff --git a/src/service_inspectors/http_inspect/test/http_module_test.cc b/src/service_inspectors/http_inspect/test/http_module_test.cc old mode 100644 new mode 100755 index 03667238c..48776f604 --- a/src/service_inspectors/http_inspect/test/http_module_test.cc +++ b/src/service_inspectors/http_inspect/test/http_module_test.cc @@ -45,6 +45,9 @@ void ParseWarning(WarningGroup, const char*, ...) {} void ParseError(const char*, ...) {} void Value::get_bits(std::bitset<256ul>&) const {} +void Value::set_first_token() {} +bool Value::get_next_token(std::string& ) { return false; } + int DetectionEngine::queue_event(unsigned int, unsigned int, Actions::Type) { return 0; } LiteralSearch::Handle* LiteralSearch::setup() { return nullptr; } void LiteralSearch::cleanup(LiteralSearch::Handle*) {} @@ -55,6 +58,7 @@ LiteralSearch* LiteralSearch::instantiate(LiteralSearch::Handle*, const uint8_t* void show_stats(PegCount*, const PegInfo*, unsigned, const char*) { } void show_stats(PegCount*, const PegInfo*, const IndexVec&, const char*, FILE*) { } +int32_t str_to_code(const char*, const StrCode []) { return 0; } int32_t str_to_code(const uint8_t*, const int32_t, const StrCode []) { return 0; } int32_t substr_to_code(const uint8_t*, const int32_t, const StrCode []) { return 0; } long HttpTestManager::print_amount {}; diff --git a/src/service_inspectors/http_inspect/test/http_uri_norm_test.cc b/src/service_inspectors/http_inspect/test/http_uri_norm_test.cc old mode 100644 new mode 100755 index c1db2a57e..295f7f5df --- a/src/service_inspectors/http_inspect/test/http_uri_norm_test.cc +++ b/src/service_inspectors/http_inspect/test/http_uri_norm_test.cc @@ -41,6 +41,8 @@ namespace snort void ParseWarning(WarningGroup, const char*, ...) {} void ParseError(const char*, ...) {} void Value::get_bits(std::bitset<256ul>&) const {} +void Value::set_first_token() {} +bool Value::get_next_token(std::string& ) { return false; } int DetectionEngine::queue_event(unsigned int, unsigned int, Actions::Type) { return 0; } LiteralSearch::Handle* LiteralSearch::setup() { return nullptr; } void LiteralSearch::cleanup(LiteralSearch::Handle*) {}