From: Mike Stepanek (mstepane) Date: Thu, 9 Sep 2021 12:22:57 +0000 (+0000) Subject: Merge pull request #3045 in SNORT/snort3 from ~OSERHIIE/snort3:catch_benchmark to... X-Git-Tag: 3.1.13.0~18 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=571cbf61608789a4de4091e689b209fea1e25b49;p=thirdparty%2Fsnort3.git Merge pull request #3045 in SNORT/snort3 from ~OSERHIIE/snort3:catch_benchmark to master Squashed commit of the following: commit fad1033e09ad17dd0d52c9f351770cc066e763b3 Author: Oleksandr Serhiienko Date: Tue Sep 7 21:53:26 2021 +0300 utils: add reference and description for ClamAV test cases commit 4e4d7fe2d3f4e97128331b0319d97b3a3d72bef1 Author: Oleksandr Serhiienko Date: Wed Sep 1 16:41:19 2021 +0300 utils: add benchmark tests for JSNormalizer commit 2a9ad15365ed73732ff0777886e1fbc239efbf72 Author: Oleksandr Serhiienko Date: Mon Aug 30 12:59:31 2021 +0300 catch: enable benchmarking --- diff --git a/CMakeLists.txt b/CMakeLists.txt index 7fa0d7050..d4673fad1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -57,9 +57,12 @@ include_directories (${PROJECT_SOURCE_DIR}) if (ENABLE_UNIT_TESTS) include(CTest) - add_custom_target (check COMMAND ${CMAKE_CTEST_COMMAND}) endif (ENABLE_UNIT_TESTS) +if (ENABLE_UNIT_TESTS OR ENABLE_BENCHMARK_TESTS) + add_custom_target (check COMMAND ${CMAKE_CTEST_COMMAND}) +endif (ENABLE_UNIT_TESTS OR ENABLE_BENCHMARK_TESTS) + add_subdirectory (src) add_subdirectory (tools) add_subdirectory (lua) diff --git a/cmake/configure_options.cmake b/cmake/configure_options.cmake index bff6cd806..8a7a3aebd 100644 --- a/cmake/configure_options.cmake +++ b/cmake/configure_options.cmake @@ -7,6 +7,11 @@ set ( SHELL ${ENABLE_SHELL} ) set ( UNIT_TEST ${ENABLE_UNIT_TESTS} ) set ( PIGLET ${ENABLE_PIGLET} ) +if ( ENABLE_BENCHMARK_TESTS ) + add_definitions( -DBENCHMARK_TEST ) + add_definitions( -DCATCH_CONFIG_ENABLE_BENCHMARKING ) +endif ( ENABLE_BENCHMARK_TESTS ) + if ( NOT ENABLE_COREFILES ) set ( NOCOREFILE ON ) endif ( NOT ENABLE_COREFILES ) diff --git a/cmake/create_options.cmake b/cmake/create_options.cmake index d1f7e9f47..090a76240 100644 --- a/cmake/create_options.cmake +++ b/cmake/create_options.cmake @@ -17,6 +17,7 @@ option ( ENABLE_STATIC_DAQ "link static DAQ modules" ON ) # features option ( ENABLE_SHELL "enable shell support" OFF ) option ( ENABLE_UNIT_TESTS "enable unit tests" OFF ) +option ( ENABLE_BENCHMARK_TESTS "enable benchmark tests" OFF ) option ( ENABLE_PIGLET "enable piglet test harness" OFF ) option ( ENABLE_COREFILES "Prevent Snort from generating core files" ON ) diff --git a/cmake/macros.cmake b/cmake/macros.cmake index 2d601d36f..9333e0169 100644 --- a/cmake/macros.cmake +++ b/cmake/macros.cmake @@ -70,7 +70,7 @@ endfunction (add_cpputest) function (add_catch_test testname) - if ( ENABLE_UNIT_TESTS ) + if ( ENABLE_UNIT_TESTS OR ENABLE_BENCHMARK_TESTS ) set(options NO_TEST_SOURCE) set(multiValueArgs SOURCES LIBS) cmake_parse_arguments(Catch "${options}" "" "${multiValueArgs}" ${ARGN}) @@ -83,9 +83,11 @@ function (add_catch_test testname) ${Catch_SOURCES} $ ) - target_compile_options(${testname} PRIVATE "-DCATCH_TEST_BUILD") + if ( ENABLE_UNIT_TESTS ) + target_compile_options(${testname} PRIVATE "-DCATCH_TEST_BUILD") + endif ( ENABLE_UNIT_TESTS ) target_link_libraries(${testname} PRIVATE ${Catch_LIBS}) add_test(${testname} ${testname}) add_dependencies(check ${testname}) - endif ( ENABLE_UNIT_TESTS ) + endif ( ENABLE_UNIT_TESTS OR ENABLE_BENCHMARK_TESTS ) endfunction (add_catch_test) diff --git a/configure_cmake.sh b/configure_cmake.sh index b5c6dc392..e2275cfcf 100755 --- a/configure_cmake.sh +++ b/configure_cmake.sh @@ -325,6 +325,12 @@ while [ $# -ne 0 ]; do --disable-unit-tests) append_cache_entry ENABLE_UNIT_TESTS BOOL false ;; + --enable-benchmark-tests) + append_cache_entry ENABLE_BENCHMARK_TESTS BOOL true + ;; + --disable-benchmark-tests) + append_cache_entry ENABLE_BENCHMARK_TESTS BOOL false + ;; --enable-piglet) append_cache_entry ENABLE_PIGLET BOOL true ;; diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index cf5887bee..b8d26c71e 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -74,11 +74,14 @@ include_directories(AFTER ${EXTERNAL_INCLUDES}) include_directories(BEFORE ${CMAKE_CURRENT_SOURCE_DIR}) include_directories(BEFORE network_inspectors) -if (ENABLE_UNIT_TESTS) +if ( ENABLE_UNIT_TESTS ) enable_testing() +endif ( ENABLE_UNIT_TESTS ) + +if ( ENABLE_UNIT_TESTS OR ENABLE_BENCHMARK_TESTS ) set( UNIT_TESTS_LIBRARIES $) add_subdirectory(catch) -endif (ENABLE_UNIT_TESTS) +endif ( ENABLE_UNIT_TESTS OR ENABLE_BENCHMARK_TESTS ) if ( ENABLE_PIGLET ) set ( PIGLET_LIBRARIES $ $ ) diff --git a/src/catch/CMakeLists.txt b/src/catch/CMakeLists.txt index 02e3d3653..62a8ecd2c 100644 --- a/src/catch/CMakeLists.txt +++ b/src/catch/CMakeLists.txt @@ -1,5 +1,5 @@ -if ( ENABLE_UNIT_TESTS ) +if ( ENABLE_UNIT_TESTS OR ENABLE_BENCHMARK_TESTS ) set (CATCH_INCLUDES catch.hpp @@ -21,4 +21,4 @@ if ( ENABLE_UNIT_TESTS ) catch_main.cc ) -endif() +endif( ENABLE_UNIT_TESTS OR ENABLE_BENCHMARK_TESTS ) diff --git a/src/catch/dev_notes.txt b/src/catch/dev_notes.txt index 14ec16c1d..db226203d 100644 --- a/src/catch/dev_notes.txt +++ b/src/catch/dev_notes.txt @@ -1,6 +1,12 @@ -==== Unit Test +==== Unit and Benchmark Tests -This directory contains the unit-test interface. +This directory contains unit-test and benchmark-test interfaces. + +In order to enable unit tests, configure snort with --enable-unit-tests. + +To enable benchmark tests, configure snort with --enable-benchmark-tests. +For benchmarking is also preferred to configure a non-debug build with +optimizations. catch.hpp is from https://github.com/philsquared/Catch. diff --git a/src/dev_notes.txt b/src/dev_notes.txt index a1a78ea07..9f60733d2 100644 --- a/src/dev_notes.txt +++ b/src/dev_notes.txt @@ -16,8 +16,8 @@ The shell has to be explicitly enabled at build time to be available and then must be configured at run time to be activated. Multiple simultaneous remote shells are supported. -Unit test and piglet test harness build options also impact actual -execution. +Unit test, benchmark test, and piglet test harness build options also +impact actual execution. Reload is implemented by swapping a thread local config pointer by each running Pig. The inspector manager is called to empty trash if the main diff --git a/src/main.cc b/src/main.cc index ec66560c1..389a0edce 100644 --- a/src/main.cc +++ b/src/main.cc @@ -56,7 +56,7 @@ #include "utils/util.h" #include "utils/safec.h" -#ifdef UNIT_TEST +#if defined(UNIT_TEST) || defined(BENCHMARK_TEST) #include "catch/unit_test.h" #endif @@ -851,7 +851,7 @@ static bool set_mode() return false; } #endif -#ifdef UNIT_TEST +#if defined(UNIT_TEST) || defined(BENCHMARK_TEST) // FIXIT-M X we should move this out of set_mode and not do Snort bring up/teardown at all if ( catch_enabled() ) { diff --git a/src/main/snort_module.cc b/src/main/snort_module.cc index a07fc1be1..247cfa1d5 100644 --- a/src/main/snort_module.cc +++ b/src/main/snort_module.cc @@ -40,7 +40,7 @@ #include "parser/vars.h" #include "trace/trace_config.h" -#ifdef UNIT_TEST +#if defined(UNIT_TEST) || defined(BENCHMARK_TEST) #include "catch/unit_test.h" #endif @@ -567,9 +567,9 @@ static const Parameter s_params[] = { "--tweaks", Parameter::PT_STRING, nullptr, nullptr, "tune configuration" }, -#ifdef UNIT_TEST +#if defined(UNIT_TEST) || defined(BENCHMARK_TEST) { "--catch-test", Parameter::PT_STRING, nullptr, nullptr, - "comma separated list of cat unit test tags or 'all'" }, + "comma separated list of Catch test tags or 'all'" }, #endif { "--version", Parameter::PT_IMPLIED, nullptr, nullptr, "show version number (same as -V)" }, @@ -1104,7 +1104,7 @@ bool SnortModule::set(const char*, Value& v, SnortConfig* sc) else if ( v.is("--tweaks") ) sc->set_tweaks(v.get_string()); -#ifdef UNIT_TEST +#if defined(UNIT_TEST) || defined(BENCHMARK_TEST) else if ( v.is("--catch-test") ) catch_set_filter(v.get_string()); #endif diff --git a/src/utils/js_identifier_ctx.cc b/src/utils/js_identifier_ctx.cc index 308c7d7fb..3b682b490 100644 --- a/src/utils/js_identifier_ctx.cc +++ b/src/utils/js_identifier_ctx.cc @@ -23,7 +23,7 @@ #include "js_identifier_ctx.h" -#ifndef CATCH_TEST_BUILD +#if !defined(CATCH_TEST_BUILD) && !defined(BENCHMARK_TEST) #include "service_inspectors/http_inspect/http_enum.h" #include "service_inspectors/http_inspect/http_module.h" #else diff --git a/src/utils/test/js_normalizer_test.cc b/src/utils/test/js_normalizer_test.cc index 65322a88f..85935fdf0 100644 --- a/src/utils/test/js_normalizer_test.cc +++ b/src/utils/test/js_normalizer_test.cc @@ -58,6 +58,11 @@ using namespace snort; #define DEPTH 65535 #define MAX_TEMPLATE_NESTNIG 4 + +// Unit tests + +#ifdef CATCH_TEST_BUILD + #define DST_SIZE 512 #define NORMALIZE(src, expected) \ @@ -204,7 +209,7 @@ using namespace snort; CLOSE(); \ } -// ClamAV test cases +// ClamAV test vectors from: https://github.com/Cisco-Talos/clamav/blob/main/unit_tests/check_jsnorm.c static const char clamav_buf0[] = "function foo(a, b) {\n" "var x = 1.9e2*2*a/ 4.;\n" @@ -324,77 +329,78 @@ static const char clamav_expected14[] = TEST_CASE("clamav tests", "[JSNormalizer]") { - SECTION("test_case_0") + SECTION("test_case_0 - mixed identifiers and comments") { NORMALIZE(clamav_buf0, clamav_expected0); VALIDATE(clamav_buf0, clamav_expected0); } - SECTION("test_case_1") + SECTION("test_case_1 - escaped unicode in identifier") { NORMALIZE(clamav_buf1, clamav_expected1); VALIDATE(clamav_buf1, clamav_expected1); } - SECTION("test_case_2") + SECTION("test_case_2 - accumulated string assignment") { NORMALIZE(clamav_buf2, clamav_expected2); VALIDATE(clamav_buf2, clamav_expected2); } - SECTION("test_case_3") + SECTION("test_case_3 - percent-encoded string") { NORMALIZE(clamav_buf3, clamav_expected3); VALIDATE(clamav_buf3, clamav_expected3); } - SECTION("test_case_4") + SECTION("test_case_4 - percent-encoded string") { NORMALIZE(clamav_buf4, clamav_expected4); VALIDATE(clamav_buf4, clamav_expected4); } - SECTION("test_case_5") + SECTION("test_case_5 - obfuscated script") { NORMALIZE(clamav_buf5, clamav_expected5); VALIDATE(clamav_buf5, clamav_expected5); } - SECTION("test_case_6") + SECTION("test_case_6 - obfuscated script") { NORMALIZE(clamav_buf6, clamav_expected6); VALIDATE(clamav_buf6, clamav_expected6); } - SECTION("test_case_7") + SECTION("test_case_7 - single quotes string") { NORMALIZE(clamav_buf7, clamav_expected7); VALIDATE(clamav_buf7, clamav_expected7); } - SECTION("test_case_8") + SECTION("test_case_8 - double quotes string") { NORMALIZE(clamav_buf8, clamav_expected8); VALIDATE(clamav_buf8, clamav_expected8); } - SECTION("test_case_9") + SECTION("test_case_9 - obfuscated script") { NORMALIZE(clamav_buf9, clamav_expected9); VALIDATE(clamav_buf9, clamav_expected9); } - SECTION("test_case_10") + SECTION("test_case_10 - obfuscated script") { NORMALIZE(clamav_buf10, clamav_expected10); VALIDATE(clamav_buf10, clamav_expected10); } - SECTION("test_case_11") + SECTION("test_case_11 - integer literal") { NORMALIZE(clamav_buf11, clamav_expected11); VALIDATE(clamav_buf11, clamav_expected11); } - SECTION("test_case_12") + SECTION("test_case_12 - escaped unicode in string literal") { NORMALIZE(clamav_buf12, clamav_expected12); VALIDATE(clamav_buf12, clamav_expected12); } - SECTION("test_case_13") + // FIXIT-L this should be revisited + SECTION("test_case_13 - invalid escape sequence") { NORMALIZE(clamav_buf13, clamav_expected13); VALIDATE(clamav_buf13, clamav_expected13); } - SECTION("test_case_14") + SECTION("test_case_14 - EOF in the middle of string literal") { NORMALIZE(clamav_buf14, clamav_expected14); // trailing \0 is included as a part of the string @@ -404,7 +410,7 @@ TEST_CASE("clamav tests", "[JSNormalizer]") } } -// Test cases for all match patterns +// Test vectors for all match patterns static const char all_patterns_buf0[] = "var \x9\xB\xC\x20\xA0\x8\xA\xD\xEF\xBB\xBF\xE2\x80\xA8\xE2\x80\xA9\n" " \n\t\r\v a; \0"; @@ -469,7 +475,6 @@ static const char all_patterns_buf6[] = static const char all_patterns_expected6[] = "tag ` template\n ${a+b} template`"; - TEST_CASE("all patterns", "[JSNormalizer]") { SECTION("whitespaces and special characters") @@ -562,7 +567,7 @@ TEST_CASE("all patterns", "[JSNormalizer]") } } -// Tests for different syntax cases +// Test vectors for different syntax cases static const char syntax_cases_buf0[] = "var a;\n" "var b = \"init this stuff\";\n" @@ -879,7 +884,6 @@ static const char syntax_cases_buf23[] = static const char syntax_cases_expected23[] = "`${`${`${`${`"; - TEST_CASE("syntax cases", "[JSNormalizer]") { SECTION("variables") @@ -1870,3 +1874,106 @@ TEST_CASE("memcap", "[JSNormalizer]") NORM_LIMITED(5, dat1, dat2, exp1, exp2); } } + +#endif // CATCH_TEST_BUILD + +// Benchmark tests + +#ifdef BENCHMARK_TEST + +#define UNLIM_DEPTH -1 + +static constexpr const char* s_closing_tag = ""; + +#define MAKE_INPUT(src, src_len, start, mid, end, depth) \ + std::string input_##src(start); \ + input_##src.append(depth - strlen(start) - strlen(end) - strlen(s_closing_tag), mid); \ + input_##src.append(end, strlen(end)); \ + input_##src.append(s_closing_tag, strlen(s_closing_tag)); \ + const char* src = input_##src.c_str(); \ + size_t src_len = input_##src.size() + +TEST_CASE("benchmarking - ::normalize() - literals", "[JSNormalizer]") +{ + JSIdentifierCtxTest ident_ctx; + JSNormalizer normalizer(ident_ctx, UNLIM_DEPTH, MAX_TEMPLATE_NESTNIG); + char dst[DEPTH]; + + MAKE_INPUT(src_ws, src_ws_len, "", ' ', "", DEPTH); + MAKE_INPUT(src_bcomm, src_bcomm_len, "/*", ' ', "*/", DEPTH); + MAKE_INPUT(src_dqstr, src_dqstr_len, "\"", ' ', "\"", DEPTH); + + BENCHMARK("memcpy - whitespaces - 65535 bytes") + { + return memcpy(dst, src_ws, src_ws_len); + }; + BENCHMARK("whitespaces - 65535 bytes") + { + return normalizer.normalize(src_ws, src_ws_len, dst, DEPTH); + }; + BENCHMARK("block comment - 65535 bytes") + { + return normalizer.normalize(src_bcomm, src_bcomm_len, dst, DEPTH); + }; + BENCHMARK("double quotes string - 65535 bytes") + { + return normalizer.normalize(src_dqstr, src_dqstr_len, dst, DEPTH); + }; + + constexpr size_t dpeth_8k = 8192; + + MAKE_INPUT(src_ws_8k, src_ws_len_8k, "", ' ', "", dpeth_8k); + MAKE_INPUT(src_bcomm_8k, src_bcomm_len_8k, "/*", ' ', "*/", dpeth_8k); + MAKE_INPUT(src_dqstr_8k, src_dqstr_len_8k, "\"", ' ', "\"", dpeth_8k); + + BENCHMARK("memcpy - whitespaces - 8192 bytes") + { + return memcpy(dst, src_ws_8k, src_ws_len_8k); + }; + BENCHMARK("whitespaces - 8192 bytes") + { + return normalizer.normalize(src_ws_8k, src_ws_len_8k, dst, DEPTH); + }; + BENCHMARK("block comment - 8192 bytes") + { + return normalizer.normalize(src_bcomm_8k, src_bcomm_len_8k, dst, DEPTH); + }; + BENCHMARK("double quotes string - 8192 bytes") + { + return normalizer.normalize(src_dqstr_8k, src_dqstr_len_8k, dst, DEPTH); + }; +} + +TEST_CASE("benchmarking - ::normalize() - identifiers") +{ + // around 11 000 identifiers + std::string input; + for (int it = 0; it < DEPTH; ++it) + input.append("n" + std::to_string(it) + " "); + + input.resize(DEPTH - strlen(s_closing_tag)); + input.append(s_closing_tag, strlen(s_closing_tag)); + const char* src = input.c_str(); + size_t src_len = input.size(); + + char dst[DEPTH]; + + JSIdentifierCtxTest ident_ctx_mock; + JSNormalizer normalizer_wo_ident(ident_ctx_mock, UNLIM_DEPTH, MAX_TEMPLATE_NESTNIG); + + BENCHMARK("without substitution") + { + return normalizer_wo_ident.normalize(src, src_len, dst, DEPTH); + }; + + JSIdentifierCtx ident_ctx(DEPTH); + JSNormalizer normalizer_w_ident(ident_ctx, UNLIM_DEPTH, MAX_TEMPLATE_NESTNIG); + + BENCHMARK("with substitution") + { + return normalizer_w_ident.normalize(src, src_len, dst, DEPTH); + }; +} + +#endif // BENCHMARK_TEST +