From: Oleksii. Shumeiko -X (oshumeik - SOFTSERVE INC at Cisco) Date: Wed, 31 Jan 2024 17:04:33 +0000 (+0000) Subject: Pull request #4182: Kaizen X-Git-Tag: 3.1.79.0~2 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=e85ae49a91613c9469947c87d73949d70677089a;p=thirdparty%2Fsnort3.git Pull request #4182: Kaizen Merge in SNORT/snort3 from ~OSHUMEIK/snort3:feature/kaizen to master Squashed commit of the following: commit a127d8b0b075aba335bf216c69c2a09cd52f0919 Author: Oleksii Shumeiko Date: Tue Jan 30 16:01:12 2024 +0200 kaizen: add dev_notes.txt commit 558dee4cdd82850d875e868ebb7ce8f2b2e820ba Author: Oleksii Shumeiko Date: Tue Jan 30 15:43:58 2024 +0200 kaizen: update copyright commit f4fd702f97b35604c3eca073de23dd902a8928b4 Merge: e8332d15da d6d691cb89 Author: Vitalii Serhiiovych Horbatov -X (vhorbato - SOFTSERVE INC at Cisco) Date: Thu Jan 25 14:02:09 2024 +0000 Pull request #949: kaizen: change GID to 411 Merge in FIREPOWER/snort3 from ~VHORBATO/snort3:kaizen_gid_411 to feature/kaizen * commit 'd6d691cb890dde908fe832c517075aee9588bd31': kaizen: change kaizen gid to 411 commit d6d691cb890dde908fe832c517075aee9588bd31 Author: vhorbato Date: Thu Jan 25 11:41:20 2024 +0200 kaizen: change kaizen gid to 411 commit e8332d15da6a91b648d853ceaf0389143845e1e3 Merge: 67d683fb0f 6c4e69b643 Author: Vitalii Serhiiovych Horbatov -X (vhorbato - SOFTSERVE INC at Cisco) Date: Thu Jan 25 10:32:30 2024 +0000 Pull request #948: kaizen: change default value of uri_depth to -1 Merge in FIREPOWER/snort3 from ~VHORBATO/snort3:kaizen_uri_default to feature/kaizen * commit '6c4e69b6435b1ed0e4052d877f51f80cd84c0f28': kaizen: change default value of uri_depth to -1 commit 6c4e69b6435b1ed0e4052d877f51f80cd84c0f28 Author: vhorbato Date: Thu Jan 25 11:07:31 2024 +0200 kaizen: change default value of uri_depth to -1 commit 67d683fb0ffdd446076d9cb64ab3db4d2a05eeac Merge: 32685a7bf3 8acf22fb16 Author: Vitalii Serhiiovych Horbatov -X (vhorbato - SOFTSERVE INC at Cisco) Date: Wed Jan 24 16:32:28 2024 +0000 Pull request #942: kaizen: make kaizen configurable per policy Merge in FIREPOWER/snort3 from ~VHORBATO/snort3:kaizen_fixes to feature/kaizen * commit '8acf22fb16055acd87073e352817ada8a7c5ed03': kaizen: change kaizen gid to 155 kaizen: make kaizen configurable per policy commit 8acf22fb16055acd87073e352817ada8a7c5ed03 Author: vhorbato Date: Fri Jan 19 22:19:39 2024 +0200 kaizen: change kaizen gid to 155 commit d4f65497bc32587e4209e8680d6fc9d405e6db76 Author: vhorbato Date: Wed Dec 20 18:29:49 2023 +0200 kaizen: make kaizen configurable per policy commit 32685a7bf359e01d69d586127adb42cb295e4016 Author: Oleksii Shumeiko Date: Tue Dec 19 16:02:06 2023 +0200 kaizen: extend mock object with simple matching mechanism commit c7a02041f358c5e2ac916078524f10a924803379 Author: Oleksii Shumeiko Date: Mon Dec 18 14:27:16 2023 +0200 kaizen: register module only when LibML present or REG_TEST defined commit ad5ee55b22f421c01037ffabdbe57f019da9b7c0 Author: Yehor Velykozhon -X (yvelykoz - SOFTSERVE INC at Cisco) Date: Tue Dec 19 09:33:10 2023 +0000 Pull request #931: configure_cmake.sh: fix conflict-resolve issue Merge in FIREPOWER/snort3 from ~YVELYKOZ/snort3:kaizen_fix_config to feature/kaizen Squashed commit of the following: commit 6111cb812e852e7e0bff10a4494363f37a5ff390 Author: Yehor Velykozhon Date: Mon Dec 18 17:54:05 2023 +0200 build: fix configure_cmake.sh after incomplete rebase commit cf6f22e251b9bfe57f573129eae3ece00c1a0d02 Author: Brandon Stultz Date: Fri Sep 22 16:50:39 2023 -0400 network_inspectors: add kaizen ML based exploit detector --- diff --git a/CMakeLists.txt b/CMakeLists.txt index c094dc045..525e51113 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -204,4 +204,12 @@ else () NUMA: OFF") endif () +if (HAVE_LIBML) + message("\ + LibML: ON") +else () + message("\ + LibML: OFF") +endif () + message("-------------------------------------------------------\n") diff --git a/cmake/FindML.cmake b/cmake/FindML.cmake new file mode 100644 index 000000000..e5d7165c5 --- /dev/null +++ b/cmake/FindML.cmake @@ -0,0 +1,14 @@ +find_library(ML_LIBRARIES NAMES ml_static HINTS ${ML_LIBRARIES_DIR_HINT}) +find_path(ML_INCLUDE_DIRS libml.h HINTS ${ML_INCLUDE_DIR_HINT}) + +include(FindPackageHandleStandardArgs) + +find_package_handle_standard_args(ML + DEFAULT_MSG + ML_LIBRARIES + ML_INCLUDE_DIRS +) + +if (ML_FOUND) + set(HAVE_LIBML TRUE) +endif() diff --git a/cmake/include_libraries.cmake b/cmake/include_libraries.cmake index 1a0b197f8..244c6452d 100644 --- a/cmake/include_libraries.cmake +++ b/cmake/include_libraries.cmake @@ -28,3 +28,4 @@ find_package(ICONV QUIET) find_package(UUID QUIET) find_package(Libunwind) find_package(NUMA QUIET) +find_package(ML QUIET) diff --git a/config.cmake.h.in b/config.cmake.h.in index 78205f5cf..045bb0acd 100644 --- a/config.cmake.h.in +++ b/config.cmake.h.in @@ -150,6 +150,9 @@ /* numa available */ #cmakedefine HAVE_NUMA 1 +/* libml available */ +#cmakedefine HAVE_LIBML 1 + /* Availability of specific library functions */ /* Define to 1 if you have the `malloc_trim' function. */ diff --git a/configure_cmake.sh b/configure_cmake.sh index 1e28e1907..ddabeab59 100755 --- a/configure_cmake.sh +++ b/configure_cmake.sh @@ -120,6 +120,10 @@ Optional Packages: libuuid include directory --with-uuid-libraries=DIR libuuid library directory + --with-libml-includes=DIR + libml include directory + --with-libml-libraries=DIR + libml library directory Some influential variable definitions: SIGNAL_SNORT_RELOAD= @@ -463,6 +467,12 @@ while [ $# -ne 0 ]; do --with-toolchain=*) append_cache_entry CMAKE_TOOLCHAIN_FILE PATH $optarg ;; + --with-libml-includes=*) + append_cache_entry ML_INCLUDE_DIR_HINT PATH $optarg + ;; + --with-libml-libraries=*) + append_cache_entry ML_LIBRARIES_DIR_HINT PATH $optarg + ;; SIGNAL_SNORT_RELOAD=*) append_cache_entry SIGNAL_SNORT_RELOAD STRING $optarg ;; diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 4cf2456cc..e0ffa4103 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -72,6 +72,11 @@ if ( USE_TIRPC ) LIST(APPEND EXTERNAL_INCLUDES ${TIRPC_INCLUDE_DIRS}) endif () +if ( HAVE_LIBML ) + LIST(APPEND EXTERNAL_LIBRARIES ${ML_LIBRARIES}) + LIST(APPEND EXTERNAL_INCLUDES ${ML_INCLUDE_DIRS}) +endif () + include_directories(BEFORE ${LUAJIT_INCLUDE_DIR}) include_directories(AFTER ${EXTERNAL_INCLUDES}) diff --git a/src/main/shell.cc b/src/main/shell.cc index 04a17cdce..fdf0a9fab 100644 --- a/src/main/shell.cc +++ b/src/main/shell.cc @@ -42,6 +42,10 @@ #include #endif +#ifdef HAVE_LIBML +#include +#endif + extern "C" { #include } @@ -91,6 +95,9 @@ static const char* dep_versions[] = { #endif #ifdef HAVE_LZMA "LZMA", +#endif +#ifdef HAVE_LIBML + "LIBML", #endif nullptr }; @@ -157,6 +164,9 @@ static void install_dependencies_strings(Shell* sh, lua_State* L) #ifdef HAVE_LZMA vs.push_back(lzma_version_string()); #endif +#ifdef HAVE_LIBML + vs.push_back(libml_version()); +#endif lua_createtable(L, 0, vs.size()); for (int i = 0; dep_versions[i + 1];) diff --git a/src/network_inspectors/CMakeLists.txt b/src/network_inspectors/CMakeLists.txt index de1f5727c..deefca433 100644 --- a/src/network_inspectors/CMakeLists.txt +++ b/src/network_inspectors/CMakeLists.txt @@ -2,6 +2,7 @@ add_subdirectory(appid) add_subdirectory(arp_spoof) add_subdirectory(binder) +add_subdirectory(kaizen) add_subdirectory(normalize) add_subdirectory(packet_capture) add_subdirectory(packet_tracer) @@ -21,6 +22,7 @@ endif() set(STATIC_NETWORK_INSPECTOR_PLUGINS $ $ + $ $ $ $ diff --git a/src/network_inspectors/dev_notes.txt b/src/network_inspectors/dev_notes.txt index 4c9c8f72a..fabcd45f8 100644 --- a/src/network_inspectors/dev_notes.txt +++ b/src/network_inspectors/dev_notes.txt @@ -20,6 +20,13 @@ normalizations. packet_capture - A tool for dumping the wire packets that Snort receives. +kaizen - Machine learning based exploit detector capable of detecting novel +attacks fitting known vulnerability types. Kaizen uses a neural network +provided by a model file to detect exploit patterns. The Kaizen Snort module +subscribes to HTTP events published by the HTTP inspector, performs inference +on HTTP queries/posts, and generates events if the neural network detects +an exploit. + This entire set of inspectors is instantiated as a group via network_inspectors.cc diff --git a/src/network_inspectors/kaizen/CMakeLists.txt b/src/network_inspectors/kaizen/CMakeLists.txt new file mode 100644 index 000000000..bc590578d --- /dev/null +++ b/src/network_inspectors/kaizen/CMakeLists.txt @@ -0,0 +1,8 @@ +add_library(kaizen OBJECT + kaizen_engine.cc + kaizen_engine.h + kaizen_inspector.cc + kaizen_inspector.h + kaizen_module.cc + kaizen_module.h +) diff --git a/src/network_inspectors/kaizen/dev_notes.txt b/src/network_inspectors/kaizen/dev_notes.txt new file mode 100644 index 000000000..5778f5f0c --- /dev/null +++ b/src/network_inspectors/kaizen/dev_notes.txt @@ -0,0 +1,21 @@ +Kaizen is a neural network-based exploit detector for the Snort intrusion +prevention system. It is designed to not only learn to detect known attacks +from training data, but also learn to detect attacks it has never seen before. + +Kaizen uses TensorFlow, included as LibML library. + +Global configuration sets the trained network model to use. For example: + + kaizen_engine.http_param_model = { 'model.file' } + +While per policy configuration sets data source and inspection depth in +the selected Inspection policy. The following example enables two sources, +HTTP URI and HTTP body: + + kaizen.uri_depth = -1 + kaizen.client_body_depth = 100 + +Trace messages are available: + +* trace.modules.kaizen.classifier turns on messages from Kaizen + diff --git a/src/network_inspectors/kaizen/kaizen_engine.cc b/src/network_inspectors/kaizen/kaizen_engine.cc new file mode 100644 index 000000000..1977c3ff7 --- /dev/null +++ b/src/network_inspectors/kaizen/kaizen_engine.cc @@ -0,0 +1,235 @@ +//-------------------------------------------------------------------------- +// Copyright (C) 2023-2024 Cisco and/or its affiliates. All rights reserved. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License Version 2 as published +// by the Free Software Foundation. You may not use, modify or distribute +// this program under any other version of the GNU General Public License. +// +// This program is distributed in the hope that it will be useful, but +// WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with this program; if not, write to the Free Software Foundation, Inc., +// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +//-------------------------------------------------------------------------- +// kaizen_engine.cc author Vitalii Horbatov + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "kaizen_engine.h" + +#include +#include + +#ifdef HAVE_LIBML +#include +#endif + +#include "framework/decode_data.h" +#include "log/messages.h" +#include "main/reload_tuner.h" +#include "main/snort.h" +#include "main/snort_config.h" +#include "parser/parse_conf.h" +#include "utils/util.h" + +using namespace snort; +using namespace std; + +static THREAD_LOCAL BinaryClassifier* classifier = nullptr; + +static bool build_classifier(const string& model, BinaryClassifier*& dst) +{ + dst = new BinaryClassifier(); + + return dst->build(model); +} + +//-------------------------------------------------------------------------- +// module +//-------------------------------------------------------------------------- + +static const Parameter kaizen_engine_params[] = +{ + { "http_param_model", Parameter::PT_STRING, nullptr, nullptr, "path to the model file" }, + { nullptr, Parameter::PT_MAX, nullptr, nullptr, nullptr } +}; + +KaizenEngineModule::KaizenEngineModule() : Module(KZ_ENGINE_NAME, KZ_ENGINE_HELP, kaizen_engine_params) {} + +bool KaizenEngineModule::set(const char*, Value& v, SnortConfig*) +{ + if (v.is("http_param_model")) + conf.http_param_model_path = v.get_string(); + + return true; +} + +//-------------------------------------------------------------------------- +// reload tuner +//-------------------------------------------------------------------------- + +class KaizenReloadTuner : public snort::ReloadResourceTuner +{ +public: + explicit KaizenReloadTuner(const string& http_param_model) : http_param_model(http_param_model) {} + ~KaizenReloadTuner() override = default; + + bool tinit() override + { + delete classifier; + + if (!build_classifier(http_param_model, classifier)) + ErrorMessage("Can't build the classifier model.\n"); + + return false; + } + + bool tune_packet_context() override + { return true; } + + bool tune_idle_context() override + { return true; } + +private: + const string& http_param_model; +}; + +//-------------------------------------------------------------------------- +// inspector +//-------------------------------------------------------------------------- + +KaizenEngine::KaizenEngine(const KaizenEngineConfig& c) : config(c) +{ + http_param_model = read_model(); + + if (!validate_model()) + ParseError("Can't build the classifier model %s.", config.http_param_model_path.c_str()); +} + +void KaizenEngine::show(const SnortConfig*) const +{ ConfigLogger::log_value("http_param_model", config.http_param_model_path.c_str()); } + +string KaizenEngine::read_model() +{ + const char* hint = config.http_param_model_path.c_str(); + string path; + size_t size = 0; + + if (!get_config_file(hint, path) || !get_file_size(path, size)) + { + ParseError("kaizen_engine: could not read model file: %s", hint); + return {}; + } + + ifstream file(path, ios::binary); + + if (!file.is_open()) + { + ParseError("kaizen_engine: could not read model file: %s", hint); + return {}; + } + + if (size == 0) + { + ParseError("kaizen_engine: empty model file: %s", hint); + return {}; + } + + string buffer(size, '\0'); + file.read(&buffer[0], streamsize(size)); + return buffer; +} + +bool KaizenEngine::validate_model() +{ + BinaryClassifier* test_classifier = nullptr; + bool res = build_classifier(http_param_model, test_classifier); + delete test_classifier; + + return res; +} + +void KaizenEngine::tinit() +{ build_classifier(http_param_model, classifier); } + +void KaizenEngine::tterm() +{ + delete classifier; + classifier = nullptr; +} + +void KaizenEngine::install_reload_handler(SnortConfig* sc) +{ sc->register_reload_handler(new KaizenReloadTuner(http_param_model)); } + +BinaryClassifier* KaizenEngine::get_classifier() +{ return classifier; } + +//-------------------------------------------------------------------------- +// api stuff +//-------------------------------------------------------------------------- + +static Module* mod_ctor() +{ return new KaizenEngineModule; } + +static void mod_dtor(Module* m) +{ delete m; } + +static Inspector* kaizen_engine_ctor(Module* m) +{ + KaizenEngineModule* mod = (KaizenEngineModule*)m; + return new KaizenEngine(mod->get_config()); +} + +static void kaizen_engine_dtor(Inspector* p) +{ + assert(p); + delete p; +} + +static const InspectApi kaizen_engine_api = +{ + { +#if defined(HAVE_LIBML) || defined(REG_TEST) + PT_INSPECTOR, +#else + PT_MAX, +#endif + sizeof(InspectApi), + INSAPI_VERSION, + 0, + API_RESERVED, + API_OPTIONS, + KZ_ENGINE_NAME, + KZ_ENGINE_HELP, + mod_ctor, + mod_dtor + }, + IT_PASSIVE, + PROTO_BIT__NONE, // proto_bits; + nullptr, // buffers + nullptr, // service + nullptr, // pinit + nullptr, // pterm + nullptr, // tinit + nullptr, // tterm + kaizen_engine_ctor, + kaizen_engine_dtor, + nullptr, // ssn + nullptr // reset +}; + +#ifdef BUILDING_SO +SO_PUBLIC const BaseApi* snort_plugins[] = +#else +const BaseApi* nin_kaizen_engine[] = +#endif +{ + &kaizen_engine_api.base, + nullptr +}; diff --git a/src/network_inspectors/kaizen/kaizen_engine.h b/src/network_inspectors/kaizen/kaizen_engine.h new file mode 100644 index 000000000..1c62dd199 --- /dev/null +++ b/src/network_inspectors/kaizen/kaizen_engine.h @@ -0,0 +1,103 @@ +//-------------------------------------------------------------------------- +// Copyright (C) 2023-2024 Cisco and/or its affiliates. All rights reserved. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License Version 2 as published +// by the Free Software Foundation. You may not use, modify or distribute +// this program under any other version of the GNU General Public License. +// +// This program is distributed in the hope that it will be useful, but +// WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with this program; if not, write to the Free Software Foundation, Inc., +// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +//-------------------------------------------------------------------------- +// kaizen_engine.h author Vitalii Horbatov + +#ifndef KAIZEN_ENGINE_H +#define KAIZEN_ENGINE_H + +#include "framework/module.h" +#include "framework/inspector.h" + + +#define KZ_ENGINE_NAME "kaizen_engine" +#define KZ_ENGINE_HELP "configure machine learning engine settings" + +class BinaryClassifier; +struct KaizenEngineConfig +{ + std::string http_param_model_path; +}; + +class KaizenEngineModule : public snort::Module +{ +public: + KaizenEngineModule(); + + bool set(const char*, snort::Value&, snort::SnortConfig*) override; + + Usage get_usage() const override + { return GLOBAL; } + + const KaizenEngineConfig& get_config() + { return conf; } + +private: + KaizenEngineConfig conf; +}; + + +class KaizenEngine : public snort::Inspector +{ +public: + KaizenEngine(const KaizenEngineConfig&); + + void show(const snort::SnortConfig*) const override; + void eval(snort::Packet*) override {} + + void tinit() override; + void tterm() override; + + void install_reload_handler(snort::SnortConfig*) override; + + static BinaryClassifier* get_classifier(); + +private: + std::string read_model(); + bool validate_model(); + + KaizenEngineConfig config; + std::string http_param_model; +}; + + +// Mock Classifier for tests if LibML absents. +// However, when REG_TEST is undefined, the entire code below won't be executed. +// Check the plugin type provided in kaizen_engine_api in the cc file +#ifndef HAVE_LIBML +class BinaryClassifier +{ +public: + bool build(const std::string& model) + { + pattern = model; + return pattern != "error"; + } + + bool run(const char* ptr, size_t len, float& threshold) + { + std::string data(ptr, len); + threshold = std::string::npos == data.find(pattern) ? 0.0f : 1.0f; + return pattern != "fail"; + } + +private: + std::string pattern; +}; +#endif + +#endif diff --git a/src/network_inspectors/kaizen/kaizen_inspector.cc b/src/network_inspectors/kaizen/kaizen_inspector.cc new file mode 100644 index 000000000..559e30db6 --- /dev/null +++ b/src/network_inspectors/kaizen/kaizen_inspector.cc @@ -0,0 +1,251 @@ +//-------------------------------------------------------------------------- +// Copyright (C) 2023-2024 Cisco and/or its affiliates. All rights reserved. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License Version 2 as published +// by the Free Software Foundation. You may not use, modify or distribute +// this program under any other version of the GNU General Public License. +// +// This program is distributed in the hope that it will be useful, but +// WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with this program; if not, write to the Free Software Foundation, Inc., +// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +//-------------------------------------------------------------------------- +// kaizen_inspector.cc author Brandon Stultz + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "kaizen_inspector.h" + +#include + +#ifdef HAVE_LIBML +#include +#endif + +#include "detection/detection_engine.h" +#include "log/messages.h" +#include "managers/inspector_manager.h" +#include "pub_sub/http_events.h" +#include "pub_sub/http_request_body_event.h" +#include "utils/util.h" + +#include "kaizen_engine.h" + +using namespace snort; +using namespace std; + +THREAD_LOCAL KaizenStats kaizen_stats; +THREAD_LOCAL ProfileStats kaizen_prof; + +//-------------------------------------------------------------------------- +// HTTP body event handler +//-------------------------------------------------------------------------- + +class HttpBodyHandler : public DataHandler +{ +public: + HttpBodyHandler(Kaizen& kz) + : DataHandler(KZ_NAME), inspector(kz) {} + + void handle(DataEvent& de, Flow*) override; + +private: + Kaizen& inspector; +}; + +void HttpBodyHandler::handle(DataEvent& de, Flow*) +{ + // cppcheck-suppress unreadVariable + Profile profile(kaizen_prof); + + BinaryClassifier* classifier = KaizenEngine::get_classifier(); + KaizenConfig config = inspector.get_config(); + HttpRequestBodyEvent* he = (HttpRequestBodyEvent*)&de; + + if (he->is_mime()) + return; + + int32_t body_len = 0; + + const char* body = (const char*)he->get_client_body(body_len); + + body_len = std::min(config.client_body_depth, body_len); + + if (!body || body_len <= 0) + return; + + assert(classifier); + + float output = 0.0; + + kaizen_stats.libml_calls++; + + if (!classifier->run(body, (size_t)body_len, output)) + return; + + kaizen_stats.client_body_bytes += body_len; + + debug_logf(kaizen_trace, TRACE_CLASSIFIER, nullptr, "input (body): %.*s\n", body_len, body); + debug_logf(kaizen_trace, TRACE_CLASSIFIER, nullptr, "output: %f\n", output); + + if ((double)output > config.http_param_threshold) + { + kaizen_stats.client_body_alerts++; + debug_logf(kaizen_trace, TRACE_CLASSIFIER, nullptr, "\n"); + DetectionEngine::queue_event(KZ_GID, KZ_SID); + } +} + +//-------------------------------------------------------------------------- +// HTTP uri event handler +//-------------------------------------------------------------------------- + +class HttpUriHandler : public DataHandler +{ +public: + HttpUriHandler(Kaizen& kz) + : DataHandler(KZ_NAME), inspector(kz) {} + + void handle(DataEvent&, Flow*) override; + +private: + Kaizen& inspector; +}; + +void HttpUriHandler::handle(DataEvent& de, Flow*) +{ + // cppcheck-suppress unreadVariable + Profile profile(kaizen_prof); + + BinaryClassifier* classifier = KaizenEngine::get_classifier(); + KaizenConfig config = inspector.get_config(); + HttpEvent* he = (HttpEvent*)&de; + + int32_t query_len = 0; + const char* query = (const char*)he->get_uri_query(query_len); + + query_len = std::min(config.uri_depth, query_len); + + if (!query || query_len <= 0) + return; + + assert(classifier); + + float output = 0.0; + + kaizen_stats.libml_calls++; + + if (!classifier->run(query, (size_t)query_len, output)) + return; + + kaizen_stats.uri_bytes += query_len; + + debug_logf(kaizen_trace, TRACE_CLASSIFIER, nullptr, "input (query): %.*s\n", query_len, query); + debug_logf(kaizen_trace, TRACE_CLASSIFIER, nullptr, "output: %f\n", output); + + if ((double)output > config.http_param_threshold) + { + kaizen_stats.uri_alerts++; + debug_logf(kaizen_trace, TRACE_CLASSIFIER, nullptr, "\n"); + DetectionEngine::queue_event(KZ_GID, KZ_SID); + } +} + +//-------------------------------------------------------------------------- +// inspector +//-------------------------------------------------------------------------- + +void Kaizen::show(const SnortConfig*) const +{ + ConfigLogger::log_value("uri_depth", config.uri_depth); + ConfigLogger::log_value("client_body_depth", config.client_body_depth); + ConfigLogger::log_value("http_param_threshold", config.http_param_threshold); +} + +bool Kaizen::configure(SnortConfig* sc) +{ + if (config.uri_depth > 0) + DataBus::subscribe(http_pub_key, HttpEventIds::REQUEST_HEADER, new HttpUriHandler(*this)); + + if (config.client_body_depth > 0) + DataBus::subscribe(http_pub_key, HttpEventIds::REQUEST_BODY, new HttpBodyHandler(*this)); + + if(!InspectorManager::get_inspector(KZ_ENGINE_NAME, true, sc)) + { + ParseError("kaizen requires %s to be configured in the global policy.", KZ_ENGINE_NAME); + return false; + } + + return true; +} + +//-------------------------------------------------------------------------- +// api stuff +//-------------------------------------------------------------------------- + +static Module* mod_ctor() +{ return new KaizenModule; } + +static void mod_dtor(Module* m) +{ delete m; } + +static Inspector* kaizen_ctor(Module* m) +{ + KaizenModule* km = (KaizenModule*)m; + return new Kaizen(km->get_conf()); +} + +static void kaizen_dtor(Inspector* p) +{ + assert(p); + delete p; +} + +static const InspectApi kaizen_api = +{ + { +#if defined(HAVE_LIBML) || defined(REG_TEST) + PT_INSPECTOR, +#else + PT_MAX, +#endif + sizeof(InspectApi), + INSAPI_VERSION, + 0, + API_RESERVED, + API_OPTIONS, + KZ_NAME, + KZ_HELP, + mod_ctor, + mod_dtor + }, + IT_PASSIVE, + PROTO_BIT__ANY_IP, // proto_bits; + nullptr, // buffers + nullptr, // service + nullptr, // pinit + nullptr, // pterm + nullptr, // tinit + nullptr, // tterm + kaizen_ctor, + kaizen_dtor, + nullptr, // ssn + nullptr // reset +}; + +#ifdef BUILDING_SO +SO_PUBLIC const BaseApi* snort_plugins[] = +#else +const BaseApi* nin_kaizen[] = +#endif +{ + &kaizen_api.base, + nullptr +}; diff --git a/src/network_inspectors/kaizen/kaizen_inspector.h b/src/network_inspectors/kaizen/kaizen_inspector.h new file mode 100644 index 000000000..90dcf7e5d --- /dev/null +++ b/src/network_inspectors/kaizen/kaizen_inspector.h @@ -0,0 +1,46 @@ +//-------------------------------------------------------------------------- +// Copyright (C) 2023-2024 Cisco and/or its affiliates. All rights reserved. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License Version 2 as published +// by the Free Software Foundation. You may not use, modify or distribute +// this program under any other version of the GNU General Public License. +// +// This program is distributed in the hope that it will be useful, but +// WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with this program; if not, write to the Free Software Foundation, Inc., +// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +//-------------------------------------------------------------------------- +// kaizen_inspector.h author Brandon Stultz + +#ifndef KAIZEN_INSPECTOR_H +#define KAIZEN_INSPECTOR_H + +#include +#include + +#include "framework/inspector.h" + +#include "kaizen_module.h" + +class Kaizen : public snort::Inspector +{ +public: + Kaizen(const KaizenConfig& c) : config(c) { }; + + void show(const snort::SnortConfig*) const override; + void eval(snort::Packet*) override {} + bool configure(snort::SnortConfig*) override; + + const KaizenConfig& get_config() + { return config; } +private: + KaizenConfig config; +}; + +#endif + diff --git a/src/network_inspectors/kaizen/kaizen_module.cc b/src/network_inspectors/kaizen/kaizen_module.cc new file mode 100644 index 000000000..c58bb99c8 --- /dev/null +++ b/src/network_inspectors/kaizen/kaizen_module.cc @@ -0,0 +1,133 @@ +//-------------------------------------------------------------------------- +// Copyright (C) 2023-2024 Cisco and/or its affiliates. All rights reserved. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License Version 2 as published +// by the Free Software Foundation. You may not use, modify or distribute +// this program under any other version of the GNU General Public License. +// +// This program is distributed in the hope that it will be useful, but +// WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with this program; if not, write to the Free Software Foundation, Inc., +// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +//-------------------------------------------------------------------------- +// kaizen_module.cc author Brandon Stultz + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "kaizen_module.h" + +#include "log/messages.h" +#include "service_inspectors/http_inspect/http_field.h" + +using namespace snort; + +THREAD_LOCAL const Trace* kaizen_trace = nullptr; + +static const Parameter kaizen_params[] = +{ + { "uri_depth", Parameter::PT_INT, "-1:max31", "-1", + "number of input HTTP URI bytes to scan (-1 unlimited)" }, + + { "client_body_depth", Parameter::PT_INT, "-1:max31", "0", + "number of input HTTP client body bytes to scan (-1 unlimited)" }, + + { "http_param_threshold", Parameter::PT_REAL, "0:1", "0.95", + "alert threshold for http_param_model" }, + + { nullptr, Parameter::PT_MAX, nullptr, nullptr, nullptr } +}; + +static const RuleMap kaizen_rules[] = +{ + { KZ_SID, "exploit payload detected" }, + { 0, nullptr } +}; + +static const PegInfo peg_names[] = +{ + { CountType::SUM, "uri_alerts", "total number of alerts triggered on HTTP URI" }, + { CountType::SUM, "client_body_alerts", "total number of alerts triggered on HTTP client body" }, + { CountType::SUM, "uri_bytes", "total number of HTTP URI bytes processed" }, + { CountType::SUM, "client_body_bytes", "total number of HTTP client body bytes processed" }, + { CountType::SUM, "libml_calls", "total libml calls" }, + { CountType::END, nullptr, nullptr } +}; + +#ifdef DEBUG_MSGS +static const TraceOption kaizen_trace_options[] = +{ + { "classifier", TRACE_CLASSIFIER, "enable Kaizen classifier trace logging" }, + { nullptr, 0, nullptr } +}; +#endif + +//-------------------------------------------------------------------------- +// module +//-------------------------------------------------------------------------- + +KaizenModule::KaizenModule() : Module(KZ_NAME, KZ_HELP, kaizen_params) {} + +bool KaizenModule::set(const char*, Value& v, SnortConfig*) +{ + static_assert(std::is_same::value, + "Field::length maximum value should not exceed uri_depth type range"); + static_assert(std::is_same::value, + "Field::length maximum value should not exceed client_body_depth type range"); + + if (v.is("uri_depth")) + { + conf.uri_depth = v.get_int32(); + if (conf.uri_depth == -1) + conf.uri_depth = INT32_MAX; + } + else if (v.is("client_body_depth")) + { + conf.client_body_depth = v.get_int32(); + if (conf.client_body_depth == -1) + conf.client_body_depth = INT32_MAX; + } + else if (v.is("http_param_threshold")) + conf.http_param_threshold = v.get_real(); + + return true; +} + +bool KaizenModule::end(const char*, int, snort::SnortConfig*) +{ + if (!conf.uri_depth && !conf.client_body_depth) + ParseWarning(WARN_CONF, + "Neither of Kaizen source depth is set, Kaizen won't process traffic."); + + return true; +} + +const RuleMap* KaizenModule::get_rules() const +{ return kaizen_rules; } + +const PegInfo* KaizenModule::get_pegs() const +{ return peg_names; } + +PegCount* KaizenModule::get_counts() const +{ return (PegCount*)&kaizen_stats; } + +ProfileStats* KaizenModule::get_profile() const +{ return &kaizen_prof; } + +void KaizenModule::set_trace(const Trace* trace) const +{ kaizen_trace = trace; } + +const TraceOption* KaizenModule::get_trace_options() const +{ +#ifndef DEBUG_MSGS + return nullptr; +#else + return kaizen_trace_options; +#endif +} diff --git a/src/network_inspectors/kaizen/kaizen_module.h b/src/network_inspectors/kaizen/kaizen_module.h new file mode 100644 index 000000000..68fb6daeb --- /dev/null +++ b/src/network_inspectors/kaizen/kaizen_module.h @@ -0,0 +1,89 @@ +//-------------------------------------------------------------------------- +// Copyright (C) 2023-2024 Cisco and/or its affiliates. All rights reserved. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License Version 2 as published +// by the Free Software Foundation. You may not use, modify or distribute +// this program under any other version of the GNU General Public License. +// +// This program is distributed in the hope that it will be useful, but +// WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with this program; if not, write to the Free Software Foundation, Inc., +// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +//-------------------------------------------------------------------------- +// kaizen_module.h author Brandon Stultz + +#ifndef KAIZEN_MODULE_H +#define KAIZEN_MODULE_H + +#include "framework/module.h" +#include "main/thread.h" +#include "profiler/profiler.h" +#include "trace/trace_api.h" + +#define KZ_GID 411 +#define KZ_SID 1 + +#define KZ_NAME "kaizen" +#define KZ_HELP "machine learning based exploit detector" + +enum { TRACE_CLASSIFIER }; + +struct KaizenStats +{ + PegCount uri_alerts; + PegCount client_body_alerts; + PegCount uri_bytes; + PegCount client_body_bytes; + PegCount libml_calls; +}; + +extern THREAD_LOCAL KaizenStats kaizen_stats; +extern THREAD_LOCAL snort::ProfileStats kaizen_prof; +extern THREAD_LOCAL const snort::Trace* kaizen_trace; + +struct KaizenConfig +{ + std::string http_param_model_path; + double http_param_threshold; + int32_t uri_depth; + int32_t client_body_depth; +}; + +class KaizenModule : public snort::Module +{ +public: + KaizenModule(); + + bool set(const char*, snort::Value&, snort::SnortConfig*) override; + bool end(const char*, int, snort::SnortConfig*) override; + + const KaizenConfig& get_conf() const + { return conf; } + + unsigned get_gid() const override + { return KZ_GID; } + + const snort::RuleMap* get_rules() const override; + + const PegInfo* get_pegs() const override; + PegCount* get_counts() const override; + + Usage get_usage() const override + { return INSPECT; } + + snort::ProfileStats* get_profile() const override; + + void set_trace(const snort::Trace*) const override; + const snort::TraceOption* get_trace_options() const override; + +private: + KaizenConfig conf = {}; +}; + +#endif + diff --git a/src/network_inspectors/network_inspectors.cc b/src/network_inspectors/network_inspectors.cc index 3d7232ae0..399f1fd18 100644 --- a/src/network_inspectors/network_inspectors.cc +++ b/src/network_inspectors/network_inspectors.cc @@ -31,6 +31,8 @@ extern const BaseApi* nin_normalize; extern const BaseApi* nin_reputation; extern const BaseApi* nin_appid[]; +extern const BaseApi* nin_kaizen_engine[]; +extern const BaseApi* nin_kaizen[]; extern const BaseApi* nin_port_scan[]; extern const BaseApi* nin_rna[]; @@ -52,6 +54,8 @@ void load_network_inspectors() { PluginManager::load_plugins(network_inspectors); PluginManager::load_plugins(nin_appid); + PluginManager::load_plugins(nin_kaizen_engine); + PluginManager::load_plugins(nin_kaizen); PluginManager::load_plugins(nin_port_scan); PluginManager::load_plugins(nin_rna); diff --git a/src/utils/util.cc b/src/utils/util.cc index eaeb5a8bb..afcf75dc0 100644 --- a/src/utils/util.cc +++ b/src/utils/util.cc @@ -45,6 +45,10 @@ #include #endif +#ifdef HAVE_LIBML +#include +#endif + extern "C" { #include } @@ -128,6 +132,9 @@ int DisplayBanner() #endif #ifdef HAVE_LZMA LogMessage(" Using LZMA version %s\n", lzma_version_string()); +#endif +#ifdef HAVE_LIBML + LogMessage(" Using LibML version %s\n", libml_version()); #endif LogMessage("\n");