From: Oleksii Shumeiko -X (oshumeik - SOFTSERVE INC at Cisco) Date: Tue, 6 Sep 2022 09:16:15 +0000 (+0000) Subject: Pull request #3572: utils: Rewrite normalizer unit tests X-Git-Tag: 3.1.41.0~2 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=80d2369f6b0d44c9d644c60cc7b1cfe7feb17895;p=thirdparty%2Fsnort3.git Pull request #3572: utils: Rewrite normalizer unit tests Merge in SNORT/snort3 from ~DKYRYLOV/snort3:js_unit_demacro to master Squashed commit of the following: commit 5aec814cf4c7bf82ddc5458dd2d807f8414137f3 Author: dkyrylov Date: Mon Jun 6 13:14:46 2022 +0300 utils: refactor JS normalizer unit tests --- diff --git a/src/utils/js_tokenizer.h b/src/utils/js_tokenizer.h index e69636055..e4910a744 100644 --- a/src/utils/js_tokenizer.h +++ b/src/utils/js_tokenizer.h @@ -45,6 +45,7 @@ enum JSProgramScopeType : unsigned int; class JSIdentifierCtxBase; #if defined(CATCH_TEST_BUILD) || defined(BENCHMARK_TEST) class JSTokenizerTester; +class JSTestConfig; #endif // CATCH_TEST_BUILD || BENCHMARK_TEST class JSTokenizer : public yyFlexLexer @@ -398,6 +399,7 @@ private: #if defined(CATCH_TEST_BUILD) || defined(BENCHMARK_TEST) friend JSTokenizerTester; + friend JSTestConfig; #endif // CATCH_TEST_BUILD || BENCHMARK_TEST }; diff --git a/src/utils/test/CMakeLists.txt b/src/utils/test/CMakeLists.txt index c382eadc6..b9be76201 100644 --- a/src/utils/test/CMakeLists.txt +++ b/src/utils/test/CMakeLists.txt @@ -17,9 +17,23 @@ add_catch_test( js_normalizer_test ../js_normalizer.cc ../streambuf.cc ../util_cstring.cc + js_test_options.cc js_test_utils.cc ) +if (ENABLE_BENCHMARK_TESTS) + add_catch_test( js_norm_benchmark + SOURCES + ${FLEX_js_tokenizer_OUTPUTS} + ../js_identifier_ctx.cc + ../js_normalizer.cc + ../streambuf.cc + ../util_cstring.cc + js_test_options.cc + js_test_utils.cc + ) +endif(ENABLE_BENCHMARK_TESTS) + add_catch_test( js_dealias_test SOURCES ${FLEX_js_tokenizer_OUTPUTS} @@ -27,6 +41,7 @@ add_catch_test( js_dealias_test ../js_normalizer.cc ../streambuf.cc ../util_cstring.cc + js_test_options.cc js_test_utils.cc ) @@ -37,6 +52,7 @@ add_catch_test( js_unescape_test ../js_normalizer.cc ../streambuf.cc ../util_cstring.cc + js_test_options.cc js_test_utils.cc ) @@ -49,3 +65,4 @@ add_catch_test( streambuf_test SOURCES ../streambuf.cc ) + diff --git a/src/utils/test/dev_notes.txt b/src/utils/test/dev_notes.txt new file mode 100644 index 000000000..bde1f61f2 --- /dev/null +++ b/src/utils/test/dev_notes.txt @@ -0,0 +1,33 @@ +This folder contains unit tests for utils folder and testing fixtures/utilities for these tests. + +js_test_utils: +Provides constants and testing functions for JavaScript normalizer tests. +Test functions to check normalization and scope, both simple and multi-PDU, +are made configurable through derivable configs and overrides. + +Use examples: + +[source,cpp] +----------------------- + test_normalization("abc", "var_0000"); // will use params from default_config + + test_normalization("abc", "var_0000", {return_code(JSTokenizer::SCRIPT_CONTINUE)}); + // add check for the tokenizer return code, use default_config for everything else + + auto config = default_config.derive({norm_depth(10), check_mixed_encoding(true)}); + config.test_normalization("abc", "var_0000", {return_code(JSTokenizer::SCRIPT_CONTINUE)}); + // create derived config to make a reusable setup +----------------------- + +js_test_option: +Provides a mechanism to make configurable testing setup with a number of optional, parametrized +settings. Any option is passed in params as a variant and is stored as a ConfigSet field. +Checklist to add a new option with an already present type: +1. Add the option to ConfigType enum; +2. Add the option as a Field to ConfigSet; +3. Add the option to a matching type case in Config copy constructor; +4. If the option type has a destructor (e.g. unordered_set), add the option to the matching type case in the + Config destructor; +5. Add field assignment to the set_overrides test case; +6. Add a named constuctor that returns Config with your option as the type. + diff --git a/src/utils/test/js_dealias_test.cc b/src/utils/test/js_dealias_test.cc index 1b67b9744..138132e03 100644 --- a/src/utils/test/js_dealias_test.cc +++ b/src/utils/test/js_dealias_test.cc @@ -23,7 +23,7 @@ #include "catch/catch.hpp" -#include "utils/test/js_test_utils.h" +#include "js_test_utils.h" using namespace snort; diff --git a/src/utils/test/js_norm_benchmark.cc b/src/utils/test/js_norm_benchmark.cc new file mode 100644 index 000000000..bc18c66ce --- /dev/null +++ b/src/utils/test/js_norm_benchmark.cc @@ -0,0 +1,300 @@ +//-------------------------------------------------------------------------- +// Copyright (C) 2022-2022 Cisco and/or its affiliates. All rights reserved. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License Version 2 as published +// by the Free Software Foundation. You may not use, modify or distribute +// this program under any other version of the GNU General Public License. +// +// This program is distributed in the hope that it will be useful, but +// WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with this program; if not, write to the Free Software Foundation, Inc., +// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +//-------------------------------------------------------------------------- +// js_norm_benchmark.cc authors Danylo Kyrylov , Oleksandr Serhiienko + +#ifdef BENCHMARK_TEST + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include + +#include "catch/catch.hpp" + +#include "utils/js_identifier_ctx.h" +#include "utils/js_normalizer.h" + +#include "js_test_utils.h" + +using namespace snort; + +static constexpr const char* s_closing_tag = ""; + +const int max_depth = default_config.norm_depth; + +static const std::string make_input(const char* begin, const char* mid, const char* end, size_t len) +{ + std::string str(begin); + int fill = (len - strlen(begin) - strlen(end) - strlen(s_closing_tag)) / strlen(mid); + for (int i = 0; i < fill; ++i) + str.append(mid); + str.append(end); + str.append(s_closing_tag); + return str; +} + +static const std::string make_input_repeat(const char* pattern, size_t depth) +{ + std::string str; + size_t fill = (depth - strlen(s_closing_tag))/strlen(pattern); + for (size_t it = 0; it < fill; ++it) + str.append(pattern); + + str.append(s_closing_tag); + return str; +} + +static JSTokenizer::JSRet norm_ret(JSNormalizer& normalizer, const std::string& input) +{ + normalizer.rewind_output(); + return normalizer.normalize(input.c_str(), input.size()); +} + +TEST_CASE("JS Normalizer, literals by 8 K", "[JSNormalizer]") +{ + auto conf = default_config.derive({norm_depth(unlim_depth), normalize_identifiers(false)}); + JSTokenizerTester tester(conf); + JSNormalizer& normalizer = tester.normalizer; + char dst[max_depth]; + + constexpr size_t size = 1 << 13; + + auto data_pl = make_input("", ".", "", size); + auto data_ws = make_input("", " ", "", size); + auto data_bc = make_input("/*", " ", "*/", size); + auto data_dq = make_input("\"", " ", "\"", size); + + BENCHMARK("memcpy()") + { + return memcpy(dst, data_pl.c_str(), data_pl.size()); + }; + + REQUIRE(norm_ret(normalizer, data_ws) == JSTokenizer::SCRIPT_ENDED); + BENCHMARK("whitespaces") + { + normalizer.rewind_output(); + return normalizer.normalize(data_ws.c_str(), data_ws.size()); + }; + + REQUIRE(norm_ret(normalizer, data_bc) == JSTokenizer::SCRIPT_ENDED); + BENCHMARK("block comment") + { + normalizer.rewind_output(); + return normalizer.normalize(data_bc.c_str(), data_bc.size()); + }; + + REQUIRE(norm_ret(normalizer, data_dq) == JSTokenizer::SCRIPT_ENDED); + BENCHMARK("double quotes string") + { + normalizer.rewind_output(); + return normalizer.normalize(data_dq.c_str(), data_dq.size()); + }; +} + +TEST_CASE("JS Normalizer, literals by 64 K", "[JSNormalizer]") +{ + auto conf = default_config.derive({norm_depth(unlim_depth), normalize_identifiers(false)}); + JSTokenizerTester tester(conf); + JSNormalizer& normalizer = tester.normalizer; + char dst[max_depth]; + + constexpr size_t size = 1 << 16; + + auto data_pl = make_input("", ".", "", size); + auto data_ws = make_input("", " ", "", size); + auto data_bc = make_input("/*", " ", "*/", size); + auto data_dq = make_input("\"", " ", "\"", size); + + BENCHMARK("memcpy()") + { + return memcpy(dst, data_pl.c_str(), data_pl.size()); + }; + + REQUIRE(norm_ret(normalizer, data_ws) == JSTokenizer::SCRIPT_ENDED); + BENCHMARK("whitespaces") + { + normalizer.rewind_output(); + return normalizer.normalize(data_ws.c_str(), data_ws.size()); + }; + + REQUIRE(norm_ret(normalizer, data_bc) == JSTokenizer::SCRIPT_ENDED); + BENCHMARK("block comment") + { + normalizer.rewind_output(); + return normalizer.normalize(data_bc.c_str(), data_bc.size()); + }; + + REQUIRE(norm_ret(normalizer, data_dq) == JSTokenizer::SCRIPT_ENDED); + BENCHMARK("double quotes string") + { + normalizer.rewind_output(); + return normalizer.normalize(data_dq.c_str(), data_dq.size()); + }; +} + +TEST_CASE("JS Normalizer, id normalization", "[JSNormalizer]") +{ + // around 11 000 identifiers + std::string input; + for (int it = 0; it < max_depth; ++it) + input.append("n" + std::to_string(it) + " "); + + input.resize(max_depth - strlen(s_closing_tag)); + input.append(s_closing_tag, strlen(s_closing_tag)); + + auto bench_config = default_config.derive({norm_depth(unlim_depth)}); + + { + auto conf = bench_config.derive({normalize_identifiers(false)}); + JSTokenizerTester tester(conf); + JSNormalizer& normalizer_wo_ident = tester.normalizer; + + REQUIRE(norm_ret(normalizer_wo_ident, input) == JSTokenizer::SCRIPT_ENDED); + BENCHMARK("without substitution") + { + normalizer_wo_ident.rewind_output(); + return normalizer_wo_ident.normalize(input.c_str(), input.size()); + }; + } + + { + auto conf = bench_config.derive({ignored_ids_list({}), ignored_properties_list({})}); + JSTokenizerTester tester(conf); + JSNormalizer& normalizer_w_ident = tester.normalizer; + + REQUIRE(norm_ret(normalizer_w_ident, input) == JSTokenizer::SCRIPT_ENDED); + BENCHMARK("with substitution") + { + normalizer_w_ident.rewind_output(); + return normalizer_w_ident.normalize(input.c_str(), input.size()); + }; + } + + { + auto conf = bench_config.derive({ignored_ids_list({"n"}), ignored_properties_list({"n"})}); + JSTokenizerTester tester(conf); + JSNormalizer& normalizer_iids = tester.normalizer; + + REQUIRE(norm_ret(normalizer_iids, input) == JSTokenizer::SCRIPT_ENDED); + BENCHMARK("with ignored identifiers") + { + normalizer_iids.rewind_output(); + return normalizer_iids.normalize(input.c_str(), input.size()); + }; + } +} + +TEST_CASE("JS Normalizer, scope tracking", "[JSNormalizer]") +{ + constexpr uint32_t depth = 65535; + auto conf = default_config.derive({norm_depth(unlim_depth),normalize_identifiers(false),max_bracket_depth(depth)}); + JSTokenizerTester tester(conf); + JSNormalizer& normalizer = tester.normalizer; + + auto src_ws = make_input("", " ", "", depth); + auto src_brace_rep = make_input_repeat("{}", depth); + auto src_paren_rep = make_input_repeat("()", depth); + auto src_bracket_rep = make_input_repeat("[]", depth); + + REQUIRE(norm_ret(normalizer, src_ws) == JSTokenizer::SCRIPT_ENDED); + BENCHMARK("whitespaces") + { + normalizer.rewind_output(); + return normalizer.normalize(src_ws.c_str(), src_ws.size()); + }; + + REQUIRE(norm_ret(normalizer, src_brace_rep) == JSTokenizer::SCRIPT_ENDED); + BENCHMARK("...{}{}{}...") + { + normalizer.rewind_output(); + return normalizer.normalize(src_brace_rep.c_str(), src_brace_rep.size()); + }; + + REQUIRE(norm_ret(normalizer, src_paren_rep) == JSTokenizer::SCRIPT_ENDED); + BENCHMARK("...()()()...") + { + normalizer.rewind_output(); + return normalizer.normalize(src_paren_rep.c_str(), src_paren_rep.size()); + }; + + REQUIRE(norm_ret(normalizer, src_bracket_rep) == JSTokenizer::SCRIPT_ENDED); + BENCHMARK("...[][][]...") + { + normalizer.rewind_output(); + return normalizer.normalize(src_bracket_rep.c_str(), src_bracket_rep.size()); + }; +} + +TEST_CASE("JS Normalizer, automatic semicolon", "[JSNormalizer]") +{ + auto w_semicolons = make_input("", "a;\n", "", max_depth); + auto wo_semicolons = make_input("", "a \n", "", max_depth); + const char* src_w_semicolons = w_semicolons.c_str(); + const char* src_wo_semicolons = wo_semicolons.c_str(); + size_t src_len = w_semicolons.size(); + + auto conf = default_config.derive({norm_depth(unlim_depth),normalize_identifiers(false)}); + JSTokenizerTester tester(conf); + JSNormalizer& normalizer_wo_ident = tester.normalizer; + + REQUIRE(norm_ret(normalizer_wo_ident, w_semicolons) == JSTokenizer::SCRIPT_ENDED); + BENCHMARK("without semicolon insertion") + { + normalizer_wo_ident.rewind_output(); + return normalizer_wo_ident.normalize(src_w_semicolons, src_len); + }; + + REQUIRE(norm_ret(normalizer_wo_ident, wo_semicolons) == JSTokenizer::SCRIPT_ENDED); + BENCHMARK("with semicolon insertion") + { + normalizer_wo_ident.rewind_output(); + return normalizer_wo_ident.normalize(src_wo_semicolons, src_len); + }; +} + +TEST_CASE("JS Normalizer, unescape", "[JSNormalizer]") +{ + auto str_unescape = make_input("'", "\\u0061", "'", max_depth); + auto f_unescape = make_input_repeat("unescape('')", max_depth); + const char* src_str_unescape = str_unescape.c_str(); + const char* src_f_unescape = f_unescape.c_str(); + size_t src_len = max_depth; + + auto conf = default_config.derive({norm_depth(unlim_depth)}); + JSTokenizerTester tester(conf); + JSNormalizer& norm = tester.normalizer; + + REQUIRE(norm_ret(norm, str_unescape) == JSTokenizer::SCRIPT_ENDED); + BENCHMARK("unescape sequence") + { + norm.rewind_output(); + return norm.normalize(src_str_unescape, src_len); + }; + + REQUIRE(norm_ret(norm, f_unescape) == JSTokenizer::SCRIPT_ENDED); + BENCHMARK("unescape function tracking") + { + norm.rewind_output(); + return norm.normalize(src_f_unescape, src_len); + }; +} + +#endif + diff --git a/src/utils/test/js_normalizer_test.cc b/src/utils/test/js_normalizer_test.cc index 3171828d8..5b947c56c 100644 --- a/src/utils/test/js_normalizer_test.cc +++ b/src/utils/test/js_normalizer_test.cc @@ -21,572 +21,159 @@ #include "config.h" #endif -#include +#include #include "catch/catch.hpp" #include "utils/js_identifier_ctx.h" #include "utils/js_normalizer.h" -#include "utils/test/js_test_utils.h" -using namespace snort; +#include "js_test_utils.h" -// Unit tests +using namespace snort; +using namespace std::string_literals; #ifdef CATCH_TEST_BUILD -#define DST_SIZE 512 - -#define NORMALIZE(src) \ - JSIdentifierCtxStub ident_ctx; \ - JSNormalizer norm(ident_ctx, norm_depth, max_template_nesting, max_bracket_depth); \ - auto ret = norm.normalize(src, sizeof(src)); \ - const char* ptr = norm.get_src_next(); \ - int act_len = norm.script_size(); \ - const char* dst = norm.take_script(); - -#define NORMALIZE_EXT(src) \ - JSIdentifierCtxStub ident_ctx; \ - JSNormalizer norm(ident_ctx, norm_depth, max_template_nesting, max_bracket_depth); \ - auto ret = norm.normalize(src, sizeof(src), true); \ - const char* ptr = norm.get_src_next(); \ - int act_len = norm.script_size(); \ - const char* dst = norm.take_script(); - -#define VALIDATE(src, expected) \ - CHECK(ret == JSTokenizer::SCRIPT_CONTINUE); \ - CHECK((ptr - src) == sizeof(src)); \ - CHECK(act_len == sizeof(expected) - 1); \ - CHECK(!memcmp(dst, expected, act_len)); \ - delete[] dst; - -#define VALIDATE_FAIL(src, expected, ret_code, ptr_offset) \ - CHECK(ret == ret_code); \ - CHECK((ptr - src) == ptr_offset); \ - CHECK(act_len == sizeof(expected) - 1); \ - CHECK(!memcmp(dst, expected, act_len)); \ - delete[] dst; - -#define CHECK_OTAG(is_set) \ - if (is_set) \ - CHECK(norm.is_opening_tag_seen()); \ - else \ - CHECK(!norm.is_opening_tag_seen()); - - -#define NORMALIZE_L(src, src_len, dst, dst_len, depth, ret, ptr, len) \ - { \ - JSIdentifierCtxStub ident_ctx; \ - JSNormalizer norm(ident_ctx, depth, max_template_nesting, max_bracket_depth); \ - ret = norm.normalize(src, src_len); \ - ptr = norm.get_src_next(); \ - len = norm.script_size(); \ - const char* dptr = norm.get_script(); \ - REQUIRE(len == dst_len); \ - memcpy(dst, dptr, dst_len); \ - } - -#define DO(src, slen, dst, dlen) \ - { \ - auto ret = norm.normalize(src, slen); \ - CHECK(ret == JSTokenizer::SCRIPT_CONTINUE); \ - auto nsrc = norm.get_src_next(); \ - int act_len = norm.script_size(); \ - const char* ptr = norm.take_script(); \ - REQUIRE(nsrc - src == slen); \ - REQUIRE(act_len == dlen); \ - memcpy(dst, ptr, dlen); \ - delete[] ptr; \ - } - -#define TRY(src, slen, dst, dlen, rexp) \ - { \ - auto ret = norm.normalize(src, slen); \ - CHECK(ret == rexp); \ - int act_len = norm.script_size(); \ - const char* ptr = norm.get_script(); \ - REQUIRE(act_len == dlen); \ - memcpy(dst, ptr, dlen); \ - } - -#define CLOSE() \ - { \ - const char end[] = ""; \ - auto ret = norm.normalize(end, sizeof(end) - 1); \ - CHECK(ret == JSTokenizer::SCRIPT_ENDED); \ - } - -#define NORMALIZE_S(src1, exp1) \ - { \ - char dst1[sizeof(exp1)]; \ - \ - JSIdentifierCtx ident_ctx(norm_depth, max_scope_depth, s_ignored_ids, s_ignored_props); \ - JSNormalizer norm(ident_ctx, norm_depth, max_template_nesting, max_bracket_depth); \ - \ - DO(src1, sizeof(src1) - 1, dst1, sizeof(dst1) - 1); \ - CHECK(!memcmp(exp1, dst1, sizeof(exp1) - 1)); \ - \ - CLOSE(); \ - } - -#define NORMALIZE_T(src1, src2, exp1, exp2) \ - { \ - char dst1[sizeof(exp1)]; \ - char dst2[sizeof(exp2)]; \ - \ - JSIdentifierCtx ident_ctx(norm_depth, max_scope_depth, s_ignored_ids, s_ignored_props); \ - JSNormalizer norm(ident_ctx, norm_depth, max_template_nesting, max_bracket_depth); \ - \ - DO(src1, sizeof(src1) - 1, dst1, sizeof(dst1) - 1); \ - CHECK(!memcmp(exp1, dst1, sizeof(exp1) - 1)); \ - \ - DO(src2, sizeof(src2) - 1, dst2, sizeof(dst2) - 1); \ - CHECK(!memcmp(exp2, dst2, sizeof(exp2) - 1)); \ - \ - CLOSE(); \ - } - -#define NORMALIZE_1(src1, exp1) \ - { \ - char dst1[sizeof(exp1)]; \ - \ - JSIdentifierCtxStub ident_ctx; \ - JSNormalizer norm(ident_ctx, norm_depth, max_template_nesting, max_bracket_depth); \ - \ - DO(src1, sizeof(src1) - 1, dst1, sizeof(dst1) - 1); \ - CHECK(!memcmp(exp1, dst1, sizeof(exp1) - 1)); \ - \ - CLOSE(); \ - } - -#define NORMALIZE_2(src1, src2, exp1, exp2) \ - { \ - char dst1[sizeof(exp1)]; \ - char dst2[sizeof(exp2)]; \ - \ - JSIdentifierCtxStub ident_ctx; \ - JSNormalizer norm(ident_ctx, norm_depth, max_template_nesting, max_bracket_depth); \ - \ - DO(src1, sizeof(src1) - 1, dst1, sizeof(dst1) - 1); \ - CHECK(!memcmp(exp1, dst1, sizeof(exp1) - 1)); \ - \ - DO(src2, sizeof(src2) - 1, dst2, sizeof(dst2) - 1); \ - CHECK(!memcmp(exp2, dst2, sizeof(exp2) - 1)); \ - \ - CLOSE(); \ - } - -#define NORMALIZE_3(src1, src2, src3, exp1, exp2, exp3) \ - { \ - char dst1[sizeof(exp1)]; \ - char dst2[sizeof(exp2)]; \ - char dst3[sizeof(exp3)]; \ - \ - JSIdentifierCtxStub ident_ctx; \ - JSNormalizer norm(ident_ctx, norm_depth, max_template_nesting, max_bracket_depth); \ - \ - DO(src1, sizeof(src1) - 1, dst1, sizeof(dst1) - 1); \ - CHECK(!memcmp(exp1, dst1, sizeof(exp1) - 1)); \ - \ - DO(src2, sizeof(src2) - 1, dst2, sizeof(dst2) - 1); \ - CHECK(!memcmp(exp2, dst2, sizeof(exp2) - 1)); \ - \ - DO(src3, sizeof(src3) - 1, dst3, sizeof(dst3) - 1); \ - CHECK(!memcmp(exp3, dst3, sizeof(exp3) - 1)); \ - \ - CLOSE(); \ - } - -#define NORM_BAD_1(src1, exp1, code) \ - { \ - char dst1[sizeof(exp1)]; \ - \ - JSIdentifierCtxStub ident_ctx; \ - JSNormalizer norm(ident_ctx, norm_depth, max_template_nesting, max_bracket_depth); \ - \ - TRY(src1, sizeof(src1) - 1, dst1, sizeof(dst1) - 1, code); \ - CHECK(!memcmp(exp1, dst1, sizeof(exp1) - 1)); \ - } - -#define NORM_BAD_2(src1, src2, exp1, exp2, code) \ - { \ - char dst1[sizeof(exp1)]; \ - char dst2[sizeof(exp2)]; \ - \ - JSIdentifierCtxStub ident_ctx; \ - JSNormalizer norm(ident_ctx, norm_depth, max_template_nesting, max_bracket_depth); \ - \ - DO(src1, sizeof(src1) - 1, dst1, sizeof(dst1) - 1); \ - CHECK(!memcmp(exp1, dst1, sizeof(exp1) - 1)); \ - \ - TRY(src2, sizeof(src2) - 1, dst2, sizeof(dst2) - 1, code); \ - CHECK(!memcmp(exp2, dst2, sizeof(exp2) - 1)); \ - } - -#define NORM_BAD_3(src1, src2, src3, exp1, exp2, exp3, code) \ - { \ - char dst1[sizeof(exp1)]; \ - char dst2[sizeof(exp2)]; \ - char dst3[sizeof(exp3)]; \ - \ - JSIdentifierCtxStub ident_ctx; \ - JSNormalizer norm(ident_ctx, norm_depth, max_template_nesting, max_bracket_depth); \ - \ - DO(src1, sizeof(src1) - 1, dst1, sizeof(dst1) - 1); \ - CHECK(!memcmp(exp1, dst1, sizeof(exp1) - 1)); \ - \ - DO(src2, sizeof(src2) - 1, dst2, sizeof(dst2) - 1); \ - CHECK(!memcmp(exp2, dst2, sizeof(exp2) - 1)); \ - \ - TRY(src3, sizeof(src3) - 1, dst3, sizeof(dst3) - 1, code); \ - CHECK(!memcmp(exp3, dst3, sizeof(exp3) - 1)); \ - } - -#define NORM_LIMITED(limit, src1, src2, exp1, exp2) \ - { \ - char dst1[sizeof(exp1)]; \ - char dst2[sizeof(exp2)]; \ - \ - JSIdentifierCtxStub ident_ctx; \ - JSNormalizer norm(ident_ctx, norm_depth, max_template_nesting, max_bracket_depth, limit); \ - \ - DO(src1, sizeof(src1) - 1, dst1, sizeof(dst1) - 1); \ - CHECK(!memcmp(exp1, dst1, sizeof(exp1) - 1)); \ - \ - DO(src2, sizeof(src2) - 1, dst2, sizeof(dst2) - 1); \ - CHECK(!memcmp(exp2, dst2, sizeof(exp2) - 1)); \ - \ - CLOSE(); \ - } - -#define NORM_COMBINED_2(src1, src2, exp) \ - { \ - JSIdentifierCtxStub ident_ctx; \ - JSNormalizer norm(ident_ctx, norm_depth, max_template_nesting, max_bracket_depth); \ - \ - auto ret = norm.normalize(src1, sizeof(src1) - 1); \ - REQUIRE(ret == JSTokenizer::SCRIPT_CONTINUE); \ - \ - ret = norm.normalize(src2, sizeof(src2) - 1); \ - REQUIRE(ret == JSTokenizer::SCRIPT_CONTINUE); \ - \ - const char end[] = ""; \ - ret = norm.normalize(end, sizeof(end) - 1); \ - REQUIRE(ret == JSTokenizer::SCRIPT_ENDED); \ - \ - size_t act_len = norm.script_size(); \ - REQUIRE(act_len == sizeof(exp) - 1); \ - \ - const char* dst = norm.get_script(); \ - CHECK(!memcmp(exp, dst, sizeof(exp) - 1)); \ - } - -#define NORM_COMBINED_3(src1, src2, src3, exp) \ - { \ - JSIdentifierCtxStub ident_ctx; \ - JSNormalizer norm(ident_ctx, norm_depth, max_template_nesting, max_bracket_depth); \ - \ - auto ret = norm.normalize(src1, sizeof(src1) - 1); \ - REQUIRE(ret == JSTokenizer::SCRIPT_CONTINUE); \ - \ - ret = norm.normalize(src2, sizeof(src2) - 1); \ - REQUIRE(ret == JSTokenizer::SCRIPT_CONTINUE); \ - \ - ret = norm.normalize(src3, sizeof(src3) - 1); \ - REQUIRE(ret == JSTokenizer::SCRIPT_CONTINUE); \ - \ - const char end[] = ""; \ - ret = norm.normalize(end, sizeof(end) - 1); \ - REQUIRE(ret == JSTokenizer::SCRIPT_ENDED); \ - \ - size_t act_len = norm.script_size(); \ - REQUIRE(act_len == sizeof(exp) - 1); \ - \ - const char* dst = norm.get_script(); \ - CHECK(!memcmp(exp, dst, sizeof(exp) - 1)); \ - } - -#define NORM_COMBINED_BAD_2(src1, src2, exp, eret) \ - { \ - JSIdentifierCtxStub ident_ctx; \ - JSNormalizer norm(ident_ctx, norm_depth, max_template_nesting, max_bracket_depth); \ - \ - auto ret = norm.normalize(src1, sizeof(src1) - 1); \ - REQUIRE(ret == JSTokenizer::SCRIPT_CONTINUE); \ - \ - ret = norm.normalize(src2, sizeof(src2) - 1); \ - REQUIRE(ret == eret); \ - \ - size_t act_len = norm.script_size(); \ - REQUIRE(act_len == sizeof(exp) - 1); \ - \ - const char* dst = norm.get_script(); \ - CHECK(!memcmp(exp, dst, sizeof(exp) - 1)); \ - } - -#define NORM_COMBINED_BAD_3(src1, src2, src3, exp, eret) \ - { \ - JSIdentifierCtxStub ident_ctx; \ - JSNormalizer norm(ident_ctx, norm_depth, max_template_nesting, max_bracket_depth); \ - \ - auto ret = norm.normalize(src1, sizeof(src1) - 1); \ - REQUIRE(ret == JSTokenizer::SCRIPT_CONTINUE); \ - \ - ret = norm.normalize(src2, sizeof(src2) - 1); \ - REQUIRE(ret == JSTokenizer::SCRIPT_CONTINUE); \ - \ - ret = norm.normalize(src3, sizeof(src3) - 1); \ - REQUIRE(ret == eret); \ - \ - size_t act_len = norm.script_size(); \ - REQUIRE(act_len == sizeof(exp) - 1); \ - \ - const char* dst = norm.get_script(); \ - CHECK(!memcmp(exp, dst, sizeof(exp) - 1)); \ - } - -#define NORM_COMBINED_LIMITED_2(limit, src1, src2, exp) \ - { \ - JSIdentifierCtxStub ident_ctx; \ - JSNormalizer norm(ident_ctx, norm_depth, max_template_nesting, max_bracket_depth, limit); \ - \ - auto ret = norm.normalize(src1, sizeof(src1) - 1); \ - REQUIRE(ret == JSTokenizer::SCRIPT_CONTINUE); \ - \ - ret = norm.normalize(src2, sizeof(src2) - 1); \ - REQUIRE(ret == JSTokenizer::SCRIPT_CONTINUE); \ - \ - const char end[] = ""; \ - ret = norm.normalize(end, sizeof(end) - 1); \ - REQUIRE(ret == JSTokenizer::SCRIPT_ENDED); \ - \ - size_t act_len = norm.script_size(); \ - REQUIRE(act_len == sizeof(exp) - 1); \ - \ - const char* dst = norm.get_script(); \ - CHECK(!memcmp(exp, dst, sizeof(exp) - 1)); \ - } - -#define NORM_COMBINED_S_2(src1, src2, exp) \ - { \ - JSIdentifierCtx ident_ctx(norm_depth, max_scope_depth, s_ignored_ids, s_ignored_props); \ - JSNormalizer norm(ident_ctx, norm_depth, max_template_nesting, max_bracket_depth); \ - \ - auto ret = norm.normalize(src1, sizeof(src1) - 1); \ - REQUIRE(ret == JSTokenizer::SCRIPT_CONTINUE); \ - \ - ret = norm.normalize(src2, sizeof(src2) - 1); \ - REQUIRE(ret == JSTokenizer::SCRIPT_CONTINUE); \ - \ - const char end[] = ""; \ - ret = norm.normalize(end, sizeof(end) - 1); \ - REQUIRE(ret == JSTokenizer::SCRIPT_ENDED); \ - \ - size_t act_len = norm.script_size(); \ - REQUIRE(act_len == sizeof(exp) - 1); \ - \ - const char* dst = norm.get_script(); \ - CHECK(!memcmp(exp, dst, sizeof(exp) - 1)); \ - } - // ClamAV test vectors from: https://github.com/Cisco-Talos/clamav/blob/main/unit_tests/check_jsnorm.c -static const char clamav_buf0[] = - "function foo(a, b) {\n" - "var x = 1.9e2*2*a/ 4.;\n" - "var y = 'test\\'tst';//var\n" - "x=b[5],/* multiline\nvar z=6;\nsome*some/other**/" - "z=x/y;/* multiline oneline */var t=z/a;\n" - "z=[test,testi];" - "document.writeln('something\\n');}"; - -static const char clamav_expected0[] = - "function foo(a,b){var x=1.9e2*2*a/4.;var y='test\\'tst';x=b[5],z=x/y;var t=z/a;" - "z=[test,testi];document.writeln('something\\n');}"; - -static const char clamav_buf1[] = - "function () { var id\\u1234tx;}"; - -static const char clamav_expected1[] = - "function(){var id\u1234tx;}"; - -static const char clamav_buf2[] = - "function () { var tst=\"a\"+'bc'+ 'd'; }"; - -static const char clamav_expected2[] = - "function(){var tst=\"abcd';}"; - -static const char clamav_buf3[] = - "dF('bmfsu%2639%2638x11u%2638%263%3A%264C1');"; - -static const char clamav_expected3[] = - "dF('bmfsu%2639%2638x11u%2638%263%3A%264C1');"; - -#define B64 "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" - -static char clamav_buf4[] = - "qbphzrag.jevgr(harfpncr('%3P%73%63%72%69%70%74%20%6P%61%6R%67%75%61%67%65%3Q%22%6N%61%76%61" - "%73%63%72%69%70%74%22%3R%66%75%6R%63%74%69%6S%6R%20%64%46%28%73%29%7O%76%61%72%20%73%31" - "%3Q%75%6R%65%73%63%61%70%65%28%73%2R%73%75%62%73%74%72%28%30%2P%73%2R%6P%65%6R%67%74%68" - "%2Q%31%29%29%3O%20%76%61%72%20%74%3Q%27%27%3O%66%6S%72%28%69%3Q%30%3O%69%3P%73%31%2R%6P" - "%65%6R%67%74%68%3O%69%2O%2O%29%74%2O%3Q%53%74%72%69%6R%67%2R%66%72%6S%6Q%43%68%61%72%43" - "%6S%64%65%28%73%31%2R%63%68%61%72%43%6S%64%65%41%74%28%69%29%2Q%73%2R%73%75%62%73%74%72" - "%28%73%2R%6P%65%6R%67%74%68%2Q%31%2P%31%29%29%3O%64%6S%63%75%6Q%65%6R%74%2R%77%72%69%74" - "%65%28%75%6R%65%73%63%61%70%65%28%74%29%29%3O%7Q%3P%2S%73%63%72%69%70%74%3R'));" - "riny(qS('tV%285%3O%285%3Nsdwjl%28585%3N7%28586Q%28585%3N7%3P%7P55l%28585%3N7%3P%28585%3N7" - "%28586R%28585%3N8T5%285%3N%285%3P%286R3'));"; - -static char clamav_expected4[] = - "qbphzrag.jevgr(harfpncr('%3P%73%63%72%69%70%74%20%6P%61%6R%67%75%61%67%65%3Q%22%6N%61%76%61" - "%73%63%72%69%70%74%22%3R%66%75%6R%63%74%69%6S%6R%20%64%46%28%73%29%7O%76%61%72%20%73%31" - "%3Q%75%6R%65%73%63%61%70%65%28%73%2R%73%75%62%73%74%72%28%30%2P%73%2R%6P%65%6R%67%74%68" - "%2Q%31%29%29%3O%20%76%61%72%20%74%3Q%27%27%3O%66%6S%72%28%69%3Q%30%3O%69%3P%73%31%2R%6P" - "%65%6R%67%74%68%3O%69%2O%2O%29%74%2O%3Q%53%74%72%69%6R%67%2R%66%72%6S%6Q%43%68%61%72%43" - "%6S%64%65%28%73%31%2R%63%68%61%72%43%6S%64%65%41%74%28%69%29%2Q%73%2R%73%75%62%73%74%72" - "%28%73%2R%6P%65%6R%67%74%68%2Q%31%2P%31%29%29%3O%64%6S%63%75%6Q%65%6R%74%2R%77%72%69%74" - "%65%28%75%6R%65%73%63%61%70%65%28%74%29%29%3O%7Q%3P%2S%73%63%72%69%70%74%3R'));" - "riny(qS('tV%285%3O%285%3Nsdwjl%28585%3N7%28586Q%28585%3N7%3P%7P55l%28585%3N7%3P%28585%3N7" - "%28586R%28585%3N8T5%285%3N%285%3P%286R3'));"; - -static char clamav_buf5[] = - "shapgvba (c,n,p,x,r,e){}('0(\\'1\\');',2,2,'nyreg|j00g'.fcyvg('|'),0,{});"; - -static const char clamav_expected5[] = - "shapgvba(c,n,p,x,r,e){}('0(\\'1\\');',2,2,'nyreg|j00g'.fcyvg('|'),0,{});"; - -static const char clamav_buf6[] = - "function $(p,a,c,k,e,d){} something(); $('0(\\'1\\');',2,2,'alert|w00t'.split('|'),0,{});"; - -static const char clamav_expected6[] = - "function $(p,a,c,k,e,d){}something();$('0(\\'1\\');',2,2,'alert|w00t'.split('|'),0,{});"; - -static const char clamav_buf7[] = - "var z=\"tst" B64 "tst\";"; - -static const char clamav_expected7[] = - "var z=\"tst" B64 "tst\";"; - -static const char clamav_buf8[] = - "var z=\'tst" B64 "tst\';"; - -static const char clamav_expected8[] = - "var z=\'tst" B64 "tst\';"; - -static char clamav_buf9[] = - "riny(harfpncr('%61%6p%65%72%74%28%27%74%65%73%74%27%29%3o'));"; - -static const char clamav_expected9[] = - "riny(harfpncr('%61%6p%65%72%74%28%27%74%65%73%74%27%29%3o'));"; - -static const char clamav_buf10[] = - "function $ $() dF(x); function (p,a,c,k,e,r){function $(){}"; - -static const char clamav_expected10[] = - "function $ $()dF(x);function(p,a,c,k,e,r){function $(){}"; - -static const char clamav_buf11[] = - "var x=123456789 ;"; - -static const char clamav_expected11[] = - "var x=123456789;"; - -static const char clamav_buf12[] = - "var x='test\\u0000test';"; - -static const char clamav_expected12[] = - "var x='test\u0000test';"; - -static const char clamav_buf13[] = - "var x\\s12345"; - -static const char clamav_expected13[] = - "var x\\s12345"; - -static const char clamav_buf14[] = - "document.write(unescape('test%20test"; - -static const char clamav_expected14[] = - "document.write('test test"; TEST_CASE("clamav tests", "[JSNormalizer]") { SECTION("test_case_0 - mixed identifiers and comments") { - NORMALIZE(clamav_buf0); - VALIDATE(clamav_buf0, clamav_expected0); + test_normalization_noident( + "function foo(a, b) {\n" + "var x = 1.9e2*2*a/ 4.;\n" + "var y = 'test\\'tst';//var\n" + "x=b[5],/* multiline\nvar z=6;\nsome*some/other**/" + "z=x/y;/* multiline oneline */var t=z/a;\n" + "z=[test,testi];" + "document.writeln('something\\n');}", + + "function foo(a,b){var x=1.9e2*2*a/4.;var y='test\\'tst';x=b[5],z=x/y;var t=z/a;" + "z=[test,testi];document.writeln('something\\n');}" + ); } SECTION("test_case_1 - escaped unicode in identifier") { - NORMALIZE(clamav_buf1); - VALIDATE(clamav_buf1, clamav_expected1); + test_normalization_noident( + "function () { var id\\u1234tx;}", + "function(){var id\u1234tx;}" + ); } SECTION("test_case_2 - accumulated string assignment") { - NORMALIZE(clamav_buf2); - VALIDATE(clamav_buf2, clamav_expected2); + test_normalization_noident( + "function () { var tst=\"a\"+'bc'+ 'd'; }", + "function(){var tst=\"abcd';}" + ); } SECTION("test_case_3 - percent-encoded string") { - NORMALIZE(clamav_buf3); - VALIDATE(clamav_buf3, clamav_expected3); + test_normalization_noident( + "dF('bmfsu%2639%2638x11u%2638%263%3A%264C1');", + "dF('bmfsu%2639%2638x11u%2638%263%3A%264C1');" + ); } SECTION("test_case_4 - percent-encoded string") { - NORMALIZE(clamav_buf4); - VALIDATE(clamav_buf4, clamav_expected4); + test_normalization_noident( + "qbphzrag.jevgr(harfpncr('%3P%73%63%72%69%70%74%20%6P%61%6R%67%75%61%67%65%3Q%22%6N%61%76%61" + "%73%63%72%69%70%74%22%3R%66%75%6R%63%74%69%6S%6R%20%64%46%28%73%29%7O%76%61%72%20%73%31" + "%3Q%75%6R%65%73%63%61%70%65%28%73%2R%73%75%62%73%74%72%28%30%2P%73%2R%6P%65%6R%67%74%68" + "%2Q%31%29%29%3O%20%76%61%72%20%74%3Q%27%27%3O%66%6S%72%28%69%3Q%30%3O%69%3P%73%31%2R%6P" + "%65%6R%67%74%68%3O%69%2O%2O%29%74%2O%3Q%53%74%72%69%6R%67%2R%66%72%6S%6Q%43%68%61%72%43" + "%6S%64%65%28%73%31%2R%63%68%61%72%43%6S%64%65%41%74%28%69%29%2Q%73%2R%73%75%62%73%74%72" + "%28%73%2R%6P%65%6R%67%74%68%2Q%31%2P%31%29%29%3O%64%6S%63%75%6Q%65%6R%74%2R%77%72%69%74" + "%65%28%75%6R%65%73%63%61%70%65%28%74%29%29%3O%7Q%3P%2S%73%63%72%69%70%74%3R'));" + "riny(qS('tV%285%3O%285%3Nsdwjl%28585%3N7%28586Q%28585%3N7%3P%7P55l%28585%3N7%3P%28585%3N7" + "%28586R%28585%3N8T5%285%3N%285%3P%286R3'));", + + "qbphzrag.jevgr(harfpncr('%3P%73%63%72%69%70%74%20%6P%61%6R%67%75%61%67%65%3Q%22%6N%61%76%61" + "%73%63%72%69%70%74%22%3R%66%75%6R%63%74%69%6S%6R%20%64%46%28%73%29%7O%76%61%72%20%73%31" + "%3Q%75%6R%65%73%63%61%70%65%28%73%2R%73%75%62%73%74%72%28%30%2P%73%2R%6P%65%6R%67%74%68" + "%2Q%31%29%29%3O%20%76%61%72%20%74%3Q%27%27%3O%66%6S%72%28%69%3Q%30%3O%69%3P%73%31%2R%6P" + "%65%6R%67%74%68%3O%69%2O%2O%29%74%2O%3Q%53%74%72%69%6R%67%2R%66%72%6S%6Q%43%68%61%72%43" + "%6S%64%65%28%73%31%2R%63%68%61%72%43%6S%64%65%41%74%28%69%29%2Q%73%2R%73%75%62%73%74%72" + "%28%73%2R%6P%65%6R%67%74%68%2Q%31%2P%31%29%29%3O%64%6S%63%75%6Q%65%6R%74%2R%77%72%69%74" + "%65%28%75%6R%65%73%63%61%70%65%28%74%29%29%3O%7Q%3P%2S%73%63%72%69%70%74%3R'));" + "riny(qS('tV%285%3O%285%3Nsdwjl%28585%3N7%28586Q%28585%3N7%3P%7P55l%28585%3N7%3P%28585%3N7" + "%28586R%28585%3N8T5%285%3N%285%3P%286R3'));" + ); } SECTION("test_case_5 - obfuscated script") { - NORMALIZE(clamav_buf5); - VALIDATE(clamav_buf5, clamav_expected5); + test_normalization_noident( + "shapgvba (c,n,p,x,r,e){}('0(\\'1\\');',2,2,'nyreg|j00g'.fcyvg('|'),0,{});", + "shapgvba(c,n,p,x,r,e){}('0(\\'1\\');',2,2,'nyreg|j00g'.fcyvg('|'),0,{});" + ); } SECTION("test_case_6 - obfuscated script") { - NORMALIZE(clamav_buf6); - VALIDATE(clamav_buf6, clamav_expected6); + test_normalization_noident( + "function $(p,a,c,k,e,d){} something(); $('0(\\'1\\');',2,2,'alert|w00t'.split('|'),0,{});", + "function $(p,a,c,k,e,d){}something();$('0(\\'1\\');',2,2,'alert|w00t'.split('|'),0,{});" + ); } + +#define B64 "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + SECTION("test_case_7 - single quotes string") { - NORMALIZE(clamav_buf7); - VALIDATE(clamav_buf7, clamav_expected7); + test_normalization_noident( + "var z=\"tst" B64 "tst\";", + "var z=\"tst" B64 "tst\";" + ); } SECTION("test_case_8 - double quotes string") { - NORMALIZE(clamav_buf8); - VALIDATE(clamav_buf8, clamav_expected8); + test_normalization_noident( + "var z=\'tst" B64 "tst\';", + "var z=\'tst" B64 "tst\';" + ); } SECTION("test_case_9 - obfuscated script") { - NORMALIZE(clamav_buf9); - VALIDATE(clamav_buf9, clamav_expected9); + test_normalization_noident( + "riny(harfpncr('%61%6p%65%72%74%28%27%74%65%73%74%27%29%3o'));", + "riny(harfpncr('%61%6p%65%72%74%28%27%74%65%73%74%27%29%3o'));" + ); } SECTION("test_case_10 - obfuscated script") { - NORMALIZE(clamav_buf10); - VALIDATE(clamav_buf10, clamav_expected10); + test_normalization_noident( + "function $ $() dF(x); function (p,a,c,k,e,r){function $(){}", + "function $ $()dF(x);function(p,a,c,k,e,r){function $(){}" + ); } SECTION("test_case_11 - integer literal") { - NORMALIZE(clamav_buf11); - VALIDATE(clamav_buf11, clamav_expected11); + test_normalization_noident( + "var x=123456789 ;", + "var x=123456789;" + ); } SECTION("test_case_12 - escaped unicode in string literal") { - NORMALIZE(clamav_buf12); - VALIDATE(clamav_buf12, clamav_expected12); + test_normalization_noident( + "var x='test\\u0000test';"s, + "var x='test\u0000test';"s + ); } // FIXIT-L this should be revisited SECTION("test_case_13 - invalid escape sequence") { - NORMALIZE(clamav_buf13); - VALIDATE(clamav_buf13, clamav_expected13); + test_normalization_noident( + "var x\\s12345", + "var x\\s12345" + ); } SECTION("test_case_14 - EOF in the middle of string literal") { - NORMALIZE(clamav_buf14); - // trailing \0 is included as a part of the string - // to utilize available macros we alter the read length - act_len -= 1; - VALIDATE(clamav_buf14, clamav_expected14); + test_normalization_noident( + "document.write(unescape('test%20test", + "document.write('test test" + ); } } @@ -705,13 +292,17 @@ TEST_CASE("all patterns", "[JSNormalizer]") { SECTION("whitespaces and special characters") { - NORMALIZE(all_patterns_buf0); - VALIDATE(all_patterns_buf0, all_patterns_expected0); + test_normalization_noident( + all_patterns_buf0, + all_patterns_expected0 + ); } SECTION("comments") { - NORMALIZE(all_patterns_buf1); - VALIDATE(all_patterns_buf1, all_patterns_expected1); + test_normalization_noident( + all_patterns_buf1, + all_patterns_expected1 + ); } SECTION("directives") { @@ -725,91 +316,74 @@ TEST_CASE("all patterns", "[JSNormalizer]") const char expected1[] = "\"use strict\";var a=1;"; const char expected2[] = "var a=1 'use strict';"; - char dst0[sizeof(expected0) - 1]; - char dst1[sizeof(expected1) - 1]; - char dst2[sizeof(expected0) - 1]; - char dst3[sizeof(expected1) - 1]; - char dst4[sizeof(expected2) - 1]; - - int ret0, ret1, ret2, ret3, ret4; - const char *ptr0, *ptr1, *ptr2, *ptr3, *ptr4; - int act_len0, act_len1, act_len2, act_len3, act_len4; - - NORMALIZE_L(src0, sizeof(src0), dst0, sizeof(dst0), norm_depth, ret0, ptr0, act_len0); - NORMALIZE_L(src1, sizeof(src1), dst1, sizeof(dst1), norm_depth, ret1, ptr1, act_len1); - NORMALIZE_L(src2, sizeof(src2), dst2, sizeof(dst2), norm_depth, ret2, ptr2, act_len2); - NORMALIZE_L(src3, sizeof(src3), dst3, sizeof(dst3), norm_depth, ret3, ptr3, act_len3); - NORMALIZE_L(src4, sizeof(src4), dst4, sizeof(dst4), norm_depth, ret4, ptr4, act_len4); - - CHECK(ret0 == JSTokenizer::SCRIPT_CONTINUE); - CHECK((ptr0 - src0) == sizeof(src0)); - CHECK(act_len0 == sizeof(expected0) - 1); - CHECK(!memcmp(dst0, expected0, act_len0)); - - CHECK(ret1 == JSTokenizer::SCRIPT_CONTINUE); - CHECK((ptr1 - src1) == sizeof(src1)); - CHECK(act_len1 == sizeof(expected1) - 1); - CHECK(!memcmp(dst1, expected1, act_len1)); - - CHECK(ret2 == JSTokenizer::SCRIPT_CONTINUE); - CHECK((ptr2 - src2) == sizeof(src2)); - CHECK(act_len2 == sizeof(expected0) - 1); - CHECK(!memcmp(dst2, expected0, act_len2)); - - CHECK(ret3 == JSTokenizer::SCRIPT_CONTINUE); - CHECK((ptr3 - src3) == sizeof(src3)); - CHECK(act_len3 == sizeof(expected1) - 1); - CHECK(!memcmp(dst3, expected1, act_len3)); - - CHECK(ret4 == JSTokenizer::SCRIPT_CONTINUE); - CHECK((ptr4 - src4) == sizeof(src4)); - CHECK(act_len4 == sizeof(expected2) - 1); - CHECK(!memcmp(dst4, expected2, act_len4)); + test_normalization_noident(src0, expected0); + test_normalization_noident(src1, expected1); + test_normalization_noident(src2, expected0); + test_normalization_noident(src3, expected1); + test_normalization_noident(src4, expected2); } SECTION("punctuators") { - NORMALIZE(all_patterns_buf2); - VALIDATE(all_patterns_buf2, all_patterns_expected2); + test_normalization_noident( + all_patterns_buf2, + all_patterns_expected2 + ); } SECTION("keywords") { - NORMALIZE(all_patterns_buf3); - VALIDATE(all_patterns_buf3, all_patterns_expected3); + test_normalization_noident( + all_patterns_buf3, + all_patterns_expected3 + ); } SECTION("literals") { - NORMALIZE(all_patterns_buf4); - VALIDATE(all_patterns_buf4, all_patterns_expected4); + test_normalization_noident( + all_patterns_buf4, + all_patterns_expected4 + ); } SECTION("identifiers") { - NORMALIZE(all_patterns_buf5); - VALIDATE(all_patterns_buf5, all_patterns_expected5); + test_normalization_noident( + all_patterns_buf5, + all_patterns_expected5 + ); } SECTION("template literals") { - NORMALIZE(all_patterns_buf6); - VALIDATE(all_patterns_buf6, all_patterns_expected6); + test_normalization_noident( + all_patterns_buf6, + all_patterns_expected6 + ); } SECTION("regex literal with slashes") { - NORMALIZE(all_patterns_buf7); - VALIDATE(all_patterns_buf7, all_patterns_expected7); + test_normalization_noident( + all_patterns_buf7, + all_patterns_expected7 + ); } SECTION("multiple escaped slashes") { - NORMALIZE(all_patterns_buf8); - VALIDATE(all_patterns_buf8, all_patterns_expected8); + test_normalization_noident( + all_patterns_buf8, + all_patterns_expected8 + ); } SECTION("slashes and braces") { - NORMALIZE(all_patterns_buf9); - VALIDATE(all_patterns_buf9, all_patterns_expected9); + test_normalization_noident( + all_patterns_buf9, + all_patterns_expected9 + ); } SECTION("regex literal with curly brace") { - NORMALIZE(all_patterns_buf10); - VALIDATE(all_patterns_buf10, all_patterns_expected10); + test_normalization_noident( + all_patterns_buf10, + all_patterns_expected10 + ); } } @@ -1156,148 +730,200 @@ TEST_CASE("syntax cases", "[JSNormalizer]") { SECTION("variables") { - NORMALIZE(syntax_cases_buf0); - VALIDATE(syntax_cases_buf0, syntax_cases_expected0); + test_normalization_noident( + syntax_cases_buf0, + syntax_cases_expected0 + ); } SECTION("operators") { - NORMALIZE(syntax_cases_buf1); - VALIDATE(syntax_cases_buf1, syntax_cases_expected1); + test_normalization_noident( + syntax_cases_buf1, + syntax_cases_expected1 + ); } SECTION("arithmetic and logical operators") { - NORMALIZE(syntax_cases_buf2); - VALIDATE(syntax_cases_buf2, syntax_cases_expected2); + test_normalization_noident( + syntax_cases_buf2, + syntax_cases_expected2 + ); } SECTION("complex object") { - NORMALIZE(syntax_cases_buf3); - VALIDATE(syntax_cases_buf3, syntax_cases_expected3); + test_normalization_noident( + syntax_cases_buf3, + syntax_cases_expected3 + ); } SECTION("arrays") { - NORMALIZE(syntax_cases_buf4); - VALIDATE(syntax_cases_buf4, syntax_cases_expected4); + test_normalization_noident( + syntax_cases_buf4, + syntax_cases_expected4 + ); } SECTION("loops") { - NORMALIZE(syntax_cases_buf5); - VALIDATE(syntax_cases_buf5, syntax_cases_expected5); + test_normalization_noident( + syntax_cases_buf5, + syntax_cases_expected5 + ); } SECTION("if-else and switch statements") { - NORMALIZE(syntax_cases_buf6); - VALIDATE(syntax_cases_buf6, syntax_cases_expected6); + test_normalization_noident( + syntax_cases_buf6, + syntax_cases_expected6 + ); } SECTION("try-catch statements") { - NORMALIZE(syntax_cases_buf7); - VALIDATE(syntax_cases_buf7, syntax_cases_expected7); + test_normalization_noident( + syntax_cases_buf7, + syntax_cases_expected7 + ); } SECTION("functions and promises") { - NORMALIZE(syntax_cases_buf8); - VALIDATE(syntax_cases_buf8, syntax_cases_expected8); + test_normalization_noident( + syntax_cases_buf8, + syntax_cases_expected8 + ); } SECTION("regex-division ambiguity") { - NORMALIZE(syntax_cases_buf9); - VALIDATE(syntax_cases_buf9, syntax_cases_expected9); + test_normalization_noident( + syntax_cases_buf9, + syntax_cases_expected9 + ); } SECTION("regex on a new line") { - NORMALIZE(syntax_cases_buf10); - VALIDATE(syntax_cases_buf10, syntax_cases_expected10); + test_normalization_noident( + syntax_cases_buf10, + syntax_cases_expected10 + ); } SECTION("string and regex literals ambiguity with escaped sentinel chars") { - NORMALIZE(syntax_cases_buf11); - VALIDATE(syntax_cases_buf11, syntax_cases_expected11); + test_normalization_noident( + syntax_cases_buf11, + syntax_cases_expected11 + ); } SECTION("escaped LF and CR chars in literals") { - NORMALIZE(syntax_cases_buf12); - VALIDATE(syntax_cases_buf12, syntax_cases_expected12); + test_normalization_noident( + syntax_cases_buf12, + syntax_cases_expected12 + ); } SECTION("regex after keyword") { - NORMALIZE(syntax_cases_buf13); - VALIDATE(syntax_cases_buf13, syntax_cases_expected13); + test_normalization_noident( + syntax_cases_buf13, + syntax_cases_expected13 + ); } SECTION("white space between '+'<-->'++' and '-'<-->'--'") { - NORMALIZE(syntax_cases_buf14); - VALIDATE(syntax_cases_buf14, syntax_cases_expected14); + test_normalization_noident( + syntax_cases_buf14, + syntax_cases_expected14 + ); } SECTION("LS within literal") { - NORMALIZE(syntax_cases_buf15); - VALIDATE(syntax_cases_buf15, syntax_cases_expected15); + test_normalization_noident(syntax_cases_buf15, syntax_cases_expected15); } SECTION("PS within literal") { - NORMALIZE(syntax_cases_buf21); - VALIDATE(syntax_cases_buf21, syntax_cases_expected21); + test_normalization_noident(syntax_cases_buf21, syntax_cases_expected21); } SECTION("template literals") { - NORMALIZE(syntax_cases_buf22); - VALIDATE(syntax_cases_buf22, syntax_cases_expected22); + test_normalization_noident( + syntax_cases_buf22, + syntax_cases_expected22 + ); } SECTION("regex after operator") { - NORMALIZE_S(syntax_cases_buf25, syntax_cases_expected25); + test_normalization(syntax_cases_buf25, syntax_cases_expected25); } } TEST_CASE("bad tokens", "[JSNormalizer]") { SECTION("explicit LF within literal") - { - NORMALIZE(syntax_cases_buf16); - VALIDATE_FAIL(syntax_cases_buf16, syntax_cases_expected16, JSTokenizer::BAD_TOKEN, 23); - } + test_normalization_noident( + syntax_cases_buf16, syntax_cases_expected16, + { + return_code(JSTokenizer::BAD_TOKEN), + expected_cursor_pos(23) + }); + SECTION("explicit CR within literal") - { - NORMALIZE(syntax_cases_buf17); - VALIDATE_FAIL(syntax_cases_buf17, syntax_cases_expected17, JSTokenizer::BAD_TOKEN, 23); - } + test_normalization_noident( + syntax_cases_buf17, syntax_cases_expected17, + { + return_code(JSTokenizer::BAD_TOKEN), + expected_cursor_pos(23) + }); + SECTION("escaped LF-CR sequence within literal") - { - NORMALIZE(syntax_cases_buf18); - VALIDATE_FAIL(syntax_cases_buf18, syntax_cases_expected18, JSTokenizer::BAD_TOKEN, 25); - } + test_normalization_noident( + syntax_cases_buf18, syntax_cases_expected18, + { + return_code(JSTokenizer::BAD_TOKEN), + expected_cursor_pos(25) + }); + SECTION("escaped LF within regex literal") - { - NORMALIZE(syntax_cases_buf19); - VALIDATE_FAIL(syntax_cases_buf19, syntax_cases_expected19, JSTokenizer::BAD_TOKEN, 23); - } + test_normalization_noident( + syntax_cases_buf19, syntax_cases_expected19, + { + return_code(JSTokenizer::BAD_TOKEN), + expected_cursor_pos(23) + }); + SECTION("escaped CR-LF within regex literal") { - NORMALIZE(syntax_cases_buf20); - VALIDATE_FAIL(syntax_cases_buf20, syntax_cases_expected20, JSTokenizer::BAD_TOKEN, 23); + test_normalization_noident(syntax_cases_buf20, syntax_cases_expected20, + { + return_code(JSTokenizer::BAD_TOKEN), + expected_cursor_pos(23), + }); } SECTION("explicit LF within regex literal") { - NORMALIZE(syntax_cases_buf26); - VALIDATE_FAIL(syntax_cases_buf26, syntax_cases_expected26, JSTokenizer::BAD_TOKEN, 23); + test_normalization_noident(syntax_cases_buf26, syntax_cases_expected26, + { + return_code(JSTokenizer::BAD_TOKEN), + expected_cursor_pos(23), + }); } } TEST_CASE("braces overflow", "[JSNormalizer]") { SECTION("exceeding template literal limit") - { - NORMALIZE(syntax_cases_buf23); - VALIDATE_FAIL(syntax_cases_buf23, syntax_cases_expected23, - JSTokenizer::TEMPLATE_NESTING_OVERFLOW, 15); - } + test_normalization_noident( + syntax_cases_buf23, syntax_cases_expected23, + { + return_code(JSTokenizer::TEMPLATE_NESTING_OVERFLOW), + expected_cursor_pos(15) + }); + SECTION("exceeding regex literal limit") - { - NORMALIZE(syntax_cases_buf24); - VALIDATE_FAIL(syntax_cases_buf24, syntax_cases_expected24, - JSTokenizer::TEMPLATE_NESTING_OVERFLOW, 30); - } + test_normalization_noident( + syntax_cases_buf24, syntax_cases_expected24, + { + return_code(JSTokenizer::TEMPLATE_NESTING_OVERFLOW), + expected_cursor_pos(30) + }); + } static const char asi_cases_buf0[] = @@ -1466,164 +1092,218 @@ TEST_CASE("automatic semicolon insertion", "[JSNormalizer]") { SECTION("group_4 to group_1") { - NORMALIZE(asi_cases_buf0); - VALIDATE(asi_cases_buf0, asi_cases_expected0); + test_normalization_noident( + asi_cases_buf0, + asi_cases_expected0 + ); } SECTION("group_4 to group_7") { - NORMALIZE(asi_cases_buf1); - VALIDATE(asi_cases_buf1, asi_cases_expected1); + test_normalization_noident( + asi_cases_buf1, + asi_cases_expected1 + ); } SECTION("group_4 to group_8") { - NORMALIZE(asi_cases_buf2); - VALIDATE(asi_cases_buf2, asi_cases_expected2); + test_normalization_noident( + asi_cases_buf2, + asi_cases_expected2 + ); } SECTION("group_4 to group_9") { - NORMALIZE(asi_cases_buf3); - VALIDATE(asi_cases_buf3, asi_cases_expected3); + test_normalization_noident( + asi_cases_buf3, + asi_cases_expected3 + ); } SECTION("group_4 to group_10") { - NORMALIZE(asi_cases_buf4); - VALIDATE(asi_cases_buf4, asi_cases_expected4); + test_normalization_noident( + asi_cases_buf4, + asi_cases_expected4 + ); } SECTION("group_5 to group_7") { - NORMALIZE(asi_cases_buf5); - VALIDATE(asi_cases_buf5, asi_cases_expected5); + test_normalization_noident( + asi_cases_buf5, + asi_cases_expected5 + ); } SECTION("group_5 to group_8") { - NORMALIZE(asi_cases_buf6); - VALIDATE(asi_cases_buf6, asi_cases_expected6); + test_normalization_noident( + asi_cases_buf6, + asi_cases_expected6 + ); } SECTION("group_5 to group_9") { - NORMALIZE(asi_cases_buf7); - VALIDATE(asi_cases_buf7, asi_cases_expected7); + test_normalization_noident( + asi_cases_buf7, + asi_cases_expected7 + ); } SECTION("group_5 to group_10") { - NORMALIZE(asi_cases_buf8); - VALIDATE(asi_cases_buf8, asi_cases_expected8); + test_normalization_noident( + asi_cases_buf8, + asi_cases_expected8 + ); } SECTION("group_7 to group_1") { - NORMALIZE(asi_cases_buf9); - VALIDATE(asi_cases_buf9, asi_cases_expected9); + test_normalization_noident( + asi_cases_buf9, + asi_cases_expected9 + ); } SECTION("group_7 to group_7") { - NORMALIZE(asi_cases_buf10); - VALIDATE(asi_cases_buf10, asi_cases_expected10); + test_normalization_noident( + asi_cases_buf10, + asi_cases_expected10 + ); } SECTION("group_7 to group_8") { - NORMALIZE(asi_cases_buf11); - VALIDATE(asi_cases_buf11, asi_cases_expected11); + test_normalization_noident( + asi_cases_buf11, + asi_cases_expected11 + ); } SECTION("group_7 to group_9") { - NORMALIZE(asi_cases_buf12); - VALIDATE(asi_cases_buf12, asi_cases_expected12); + test_normalization_noident( + asi_cases_buf12, + asi_cases_expected12 + ); } SECTION("group_7 to group_10") { - NORMALIZE(asi_cases_buf13); - VALIDATE(asi_cases_buf13, asi_cases_expected13); + test_normalization_noident( + asi_cases_buf13, + asi_cases_expected13 + ); } SECTION("group_8 to group_1") { - NORMALIZE(asi_cases_buf14); - VALIDATE(asi_cases_buf14, asi_cases_expected14); + test_normalization_noident( + asi_cases_buf14, + asi_cases_expected14 + ); } SECTION("group_8 to group_3") { - NORMALIZE(asi_cases_buf15); - VALIDATE(asi_cases_buf15, asi_cases_expected15); + test_normalization_noident( + asi_cases_buf15, + asi_cases_expected15 + ); } SECTION("group_8 to group_7") { - NORMALIZE(asi_cases_buf16); - VALIDATE(asi_cases_buf16, asi_cases_expected16); + test_normalization_noident( + asi_cases_buf16, + asi_cases_expected16 + ); } SECTION("group_8 to group_8") { - NORMALIZE(asi_cases_buf17); - VALIDATE(asi_cases_buf17, asi_cases_expected17); + test_normalization_noident( + asi_cases_buf17, + asi_cases_expected17 + ); } SECTION("group_8 to group_9") { - NORMALIZE(asi_cases_buf18); - VALIDATE(asi_cases_buf18, asi_cases_expected18); + test_normalization_noident( + asi_cases_buf18, + asi_cases_expected18 + ); } SECTION("group_8 to group_10") { - NORMALIZE(asi_cases_buf19); - VALIDATE(asi_cases_buf19, asi_cases_expected19); + test_normalization_noident( + asi_cases_buf19, + asi_cases_expected19 + ); } SECTION("group_9 to group_1") { - NORMALIZE(asi_cases_buf20); - VALIDATE(asi_cases_buf20, asi_cases_expected20); + test_normalization_noident( + asi_cases_buf20, + asi_cases_expected20 + ); } SECTION("group_9 to group_3") { - NORMALIZE(asi_cases_buf21); - VALIDATE(asi_cases_buf21, asi_cases_expected21); + test_normalization_noident( + asi_cases_buf21, + asi_cases_expected21 + ); } SECTION("group_9 to group_6") { - NORMALIZE(asi_cases_buf22); - VALIDATE(asi_cases_buf22, asi_cases_expected22); + test_normalization_noident( + asi_cases_buf22, + asi_cases_expected22 + ); } SECTION("group_9 to group_7") { - NORMALIZE(asi_cases_buf23); - VALIDATE(asi_cases_buf23, asi_cases_expected23); + test_normalization_noident( + asi_cases_buf23, + asi_cases_expected23 + ); } SECTION("group_9 to group_8") { - NORMALIZE(asi_cases_buf24); - VALIDATE(asi_cases_buf24, asi_cases_expected24); + test_normalization_noident( + asi_cases_buf24, + asi_cases_expected24 + ); } SECTION("group_9 to group_9") { - NORMALIZE(asi_cases_buf25); - VALIDATE(asi_cases_buf25, asi_cases_expected25); + test_normalization_noident( + asi_cases_buf25, + asi_cases_expected25 + ); } SECTION("group_9 to group_10") { - NORMALIZE(asi_cases_buf26); - VALIDATE(asi_cases_buf26, asi_cases_expected26); + test_normalization_noident( + asi_cases_buf26, + asi_cases_expected26 + ); } } @@ -1638,48 +1318,28 @@ TEST_CASE("endings", "[JSNormalizer]") "var c = 3 ;\n"; const int ptr_offset = 33; const char expected[] = "var a=1;var b=2;"; - char dst[sizeof(expected) - 1]; - int act_len; - const char* ptr; - int ret; - NORMALIZE_L(src, sizeof(src), dst, sizeof(dst), norm_depth, ret, ptr, act_len); - - CHECK(ret == JSTokenizer::SCRIPT_ENDED); - CHECK(act_len == sizeof(expected) - 1); - CHECK((ptr - src) == ptr_offset); - CHECK(!memcmp(dst, expected, act_len)); + test_normalization_noident(src, expected, + {return_code(JSTokenizer::SCRIPT_ENDED), expected_cursor_pos(ptr_offset)}); } SECTION("depth reached", "[JSNormalizer]") { - const char src[] = "var abc = 123;\n\r"; - const char src2[] = "var foo = 321;\n\r"; - const char expected[] = "var abc"; - const char* ptr; - int ret; - - JSIdentifierCtxStub ident_ctx; - JSNormalizer norm(ident_ctx, 7, max_template_nesting, max_bracket_depth); - ret = norm.normalize(src, sizeof(src)); - ptr = norm.get_src_next(); - int act_len1 = norm.script_size(); - const char* dst1 = norm.take_script(); - - CHECK(ret == JSTokenizer::EOS); - CHECK(ptr == src + 7); - CHECK(act_len1 == sizeof(expected) - 1); - CHECK(!memcmp(dst1, expected, act_len1)); - delete[] dst1; - - ret = norm.normalize(src2, sizeof(src2)); - ptr = norm.get_src_next(); - int act_len2 = norm.script_size(); - const char* dst2 = norm.take_script(); - - CHECK(ret == JSTokenizer::EOS); - CHECK(ptr == src2 + sizeof(src2)); - CHECK(act_len2 == 0); - delete[] dst2; + std::string source = "var abc = 123;\n\r"; + std::string expected = "var abc"; + int depth = 7; + + auto norm = default_config.derive({norm_depth(depth), normalize_identifiers(false), + use_expected_for_last_pdu(true)}); + + JSTokenizerTester tester(norm); + + tester.do_pdu(source); + tester.check_output(expected); + tester.run_checks({return_code(JSTokenizer::EOS), expected_cursor_pos(depth)}); + + tester.do_pdu(source); + tester.check_output(""); + tester.run_checks({return_code(JSTokenizer::EOS), expected_cursor_pos(source.length())}); } } @@ -1960,169 +1620,210 @@ TEST_CASE("nested script tags", "[JSNormalizer]") { SECTION("explicit open tag - simple") { - NORMALIZE(unexpected_tag_buf0); - VALIDATE(unexpected_tag_buf0, unexpected_tag_expected0); - CHECK_OTAG(true); + test_normalization_noident( + unexpected_tag_buf0, unexpected_tag_expected0, + {check_open_tag(true)}); } SECTION("explicit open tag - complex") { - NORMALIZE(unexpected_tag_buf1); - VALIDATE(unexpected_tag_buf1, unexpected_tag_expected1); - CHECK_OTAG(true); + test_normalization_noident( + unexpected_tag_buf1, unexpected_tag_expected1, + {check_open_tag(true)}); } SECTION("open tag within literal - start") { - NORMALIZE(unexpected_tag_buf2); - VALIDATE(unexpected_tag_buf2, unexpected_tag_expected2); - CHECK_OTAG(true); + test_normalization_noident( + unexpected_tag_buf2, unexpected_tag_expected2, + {check_open_tag(true)}); } SECTION("open tag within literal - mid") { - NORMALIZE(unexpected_tag_buf3); - VALIDATE(unexpected_tag_buf3, unexpected_tag_expected3); - CHECK_OTAG(true); + test_normalization_noident( + unexpected_tag_buf3, unexpected_tag_expected3, + {check_open_tag(true)}); } SECTION("open tag within literal - end") { - NORMALIZE(unexpected_tag_buf4); - VALIDATE(unexpected_tag_buf4, unexpected_tag_expected4); - CHECK_OTAG(true); + test_normalization_noident( + unexpected_tag_buf4, unexpected_tag_expected4, + {check_open_tag(true)}); } SECTION("close tag within literal - start") - { - NORMALIZE(unexpected_tag_buf5); - VALIDATE_FAIL(unexpected_tag_buf5, unexpected_tag_expected5, JSTokenizer::CLOSING_TAG, 31); - } + test_normalization_noident( + unexpected_tag_buf5, unexpected_tag_expected5, + { + return_code(JSTokenizer::CLOSING_TAG), + expected_cursor_pos(31), + normalize_identifiers(false) + }); + SECTION("close tag within literal - mid") - { - NORMALIZE(unexpected_tag_buf6); - VALIDATE_FAIL(unexpected_tag_buf6, unexpected_tag_expected6, JSTokenizer::CLOSING_TAG, 41); - } + test_normalization_noident( + unexpected_tag_buf6, unexpected_tag_expected6, + { + return_code(JSTokenizer::CLOSING_TAG), + expected_cursor_pos(41) + }); + SECTION("close tag within literal - end") - { - NORMALIZE(unexpected_tag_buf7); - VALIDATE_FAIL(unexpected_tag_buf7, unexpected_tag_expected7, JSTokenizer::CLOSING_TAG, 41); - } + test_normalization_noident( + unexpected_tag_buf7, unexpected_tag_expected7, + { + return_code(JSTokenizer::CLOSING_TAG), + expected_cursor_pos(41) + }); + SECTION("open tag within literal - escaped") { - NORMALIZE(unexpected_tag_buf8); - VALIDATE(unexpected_tag_buf8, unexpected_tag_expected8); - CHECK_OTAG(true); + test_normalization_noident( + unexpected_tag_buf8, unexpected_tag_expected8, + {check_open_tag(true)}); } SECTION("close tag within literal - escaped") { - NORMALIZE(unexpected_tag_buf9); - VALIDATE(unexpected_tag_buf9, unexpected_tag_expected9); + test_normalization_noident( + unexpected_tag_buf9, + unexpected_tag_expected9 + ); } SECTION("open tag within single-line comment - start") { - NORMALIZE(unexpected_tag_buf10); - VALIDATE(unexpected_tag_buf10, unexpected_tag_expected10); - CHECK_OTAG(true); + test_normalization_noident( + unexpected_tag_buf10, unexpected_tag_expected10, + {check_open_tag(true)}); } SECTION("open tag within single-line comment - mid") { - NORMALIZE(unexpected_tag_buf11); - VALIDATE(unexpected_tag_buf11, unexpected_tag_expected11); - CHECK_OTAG(true); + test_normalization_noident( + unexpected_tag_buf11, unexpected_tag_expected11, + {check_open_tag(true)}); } SECTION("open tag within single-line comment - end") { - NORMALIZE(unexpected_tag_buf12); - VALIDATE(unexpected_tag_buf12, unexpected_tag_expected12); - CHECK_OTAG(true); + test_normalization_noident( + unexpected_tag_buf12, unexpected_tag_expected12, + {check_open_tag(true)}); } SECTION("open tag within multi-line comment - start") { - NORMALIZE(unexpected_tag_buf13); - VALIDATE(unexpected_tag_buf13, unexpected_tag_expected13); - CHECK_OTAG(true); + test_normalization_noident( + unexpected_tag_buf13, unexpected_tag_expected13, + {check_open_tag(true)}); } SECTION("open tag within multi-line comment - mid") { - NORMALIZE(unexpected_tag_buf14); - VALIDATE(unexpected_tag_buf14, unexpected_tag_expected14); - CHECK_OTAG(true); + test_normalization_noident( + unexpected_tag_buf14, unexpected_tag_expected14, + {check_open_tag(true)}); } SECTION("open tag within multi-line comment - end") { - NORMALIZE(unexpected_tag_buf15); - VALIDATE(unexpected_tag_buf15, unexpected_tag_expected15); - CHECK_OTAG(true); + test_normalization_noident( + unexpected_tag_buf15, unexpected_tag_expected15, + {check_open_tag(true)}); } SECTION("close tag within single-line comment - start") - { - NORMALIZE(unexpected_tag_buf16); - VALIDATE_FAIL(unexpected_tag_buf16, unexpected_tag_expected16, JSTokenizer::SCRIPT_ENDED, 22); - } + test_normalization_noident( + unexpected_tag_buf16, unexpected_tag_expected16, + { + return_code(JSTokenizer::SCRIPT_ENDED), + expected_cursor_pos(22) + }); + SECTION("close tag within single-line comment - mid") - { - NORMALIZE(unexpected_tag_buf17); - VALIDATE_FAIL(unexpected_tag_buf17, unexpected_tag_expected17, JSTokenizer::SCRIPT_ENDED, 34); - } + test_normalization_noident( + unexpected_tag_buf17, unexpected_tag_expected17, + { + return_code(JSTokenizer::SCRIPT_ENDED), + expected_cursor_pos(34) + }); + SECTION("close tag within single-line comment - end") - { - NORMALIZE(unexpected_tag_buf18); - VALIDATE_FAIL(unexpected_tag_buf18, unexpected_tag_expected18, JSTokenizer::SCRIPT_ENDED, 32); - } + test_normalization_noident( + unexpected_tag_buf18, unexpected_tag_expected18, + { + return_code(JSTokenizer::SCRIPT_ENDED), + expected_cursor_pos(32) + }); + SECTION("close tag within multi-line comment - start") - { - NORMALIZE(unexpected_tag_buf19); - VALIDATE_FAIL(unexpected_tag_buf19, unexpected_tag_expected19, JSTokenizer::CLOSING_TAG, 22); - } + test_normalization_noident( + unexpected_tag_buf19, unexpected_tag_expected19, + { + return_code(JSTokenizer::CLOSING_TAG), + expected_cursor_pos(22) + }); + SECTION("close tag within multi-line comment - mid") - { - NORMALIZE(unexpected_tag_buf20); - VALIDATE_FAIL(unexpected_tag_buf20, unexpected_tag_expected20, JSTokenizer::CLOSING_TAG, 32); - } + test_normalization_noident( + unexpected_tag_buf20, unexpected_tag_expected20, + { + return_code(JSTokenizer::CLOSING_TAG), + expected_cursor_pos(32) + }); + SECTION("close tag within multi-line comment - end") - { - NORMALIZE(unexpected_tag_buf21); - VALIDATE_FAIL(unexpected_tag_buf21, unexpected_tag_expected21, JSTokenizer::CLOSING_TAG, 32); - } + test_normalization_noident( + unexpected_tag_buf21, unexpected_tag_expected21, + { + return_code(JSTokenizer::CLOSING_TAG), + expected_cursor_pos(32) + }); + SECTION("multiple patterns - not matched") { - NORMALIZE(unexpected_tag_buf22); - VALIDATE(unexpected_tag_buf22, unexpected_tag_expected22); + test_normalization_noident( + unexpected_tag_buf22, + unexpected_tag_expected22 + ); } SECTION("multiple patterns - matched") { - NORMALIZE(unexpected_tag_buf23); - VALIDATE(unexpected_tag_buf23, unexpected_tag_expected23); - CHECK_OTAG(true); + test_normalization_noident( + unexpected_tag_buf23, unexpected_tag_expected23, + {check_open_tag(true)}); } SECTION("mixed lower and upper case") { - NORMALIZE(unexpected_tag_buf24); - VALIDATE(unexpected_tag_buf24, unexpected_tag_expected24); - CHECK_OTAG(true); + test_normalization_noident( + unexpected_tag_buf24, unexpected_tag_expected24, + {check_open_tag(true)}); } SECTION("opening tag within template literal") { - NORMALIZE(unexpected_tag_buf25); - VALIDATE(unexpected_tag_buf25, unexpected_tag_expected25); - CHECK_OTAG(true); + test_normalization_noident( + unexpected_tag_buf25, unexpected_tag_expected25, + {check_open_tag(true)}); } SECTION("opening tag within regex literal") { - NORMALIZE(unexpected_tag_buf26); - VALIDATE(unexpected_tag_buf26, unexpected_tag_expected26); - CHECK_OTAG(true); + test_normalization_noident( + unexpected_tag_buf26, unexpected_tag_expected26, + {check_open_tag(true)}); } SECTION("closing tag within template literal") { - NORMALIZE(unexpected_tag_buf27); - VALIDATE_FAIL(unexpected_tag_buf27, unexpected_tag_expected27, JSTokenizer::CLOSING_TAG, 26); + test_normalization_noident(unexpected_tag_buf27, unexpected_tag_expected27, + { + return_code(JSTokenizer::CLOSING_TAG), + expected_cursor_pos(26), + }); } SECTION("closing tag within regex literal") { - NORMALIZE(unexpected_tag_buf28); - VALIDATE_FAIL(unexpected_tag_buf28, unexpected_tag_expected28, JSTokenizer::CLOSING_TAG, 23); + test_normalization_noident(unexpected_tag_buf28, unexpected_tag_expected28, + { + return_code(JSTokenizer::CLOSING_TAG), + expected_cursor_pos(23), + }); } SECTION("closing tag from regex literal expression") { - NORMALIZE(unexpected_tag_buf29); - VALIDATE_FAIL(unexpected_tag_buf29, unexpected_tag_expected29, JSTokenizer::SCRIPT_ENDED, 19); + test_normalization_noident(unexpected_tag_buf29, unexpected_tag_expected29, + { + return_code(JSTokenizer::SCRIPT_ENDED), + expected_cursor_pos(19), + }); } } @@ -2133,81 +1834,81 @@ TEST_CASE("opening tag sequence", "[JSNormalizer]") const char src[] = "\";"; - NORMALIZE_3(dat1, dat2, dat3, exp1, exp2, exp3); - NORM_COMBINED_3(dat1, dat2, dat3, exp); + norm.test_normalization( + { + {dat1, exp1}, + {dat2, exp2}, + {dat3, exp3}, + }); + + norm.test_normalization_combined({dat1, dat2, dat3}, exp); } } TEST_CASE("split in closing tag", "[JSNormalizer]") { + auto norm = default_config.derive({normalize_identifiers(false), use_expected_for_last_pdu(true)}); + SECTION("< /script>") { const char dat1[] = "<"; @@ -2768,8 +2700,15 @@ TEST_CASE("split in closing tag", "[JSNormalizer]") const char exp2[] = ""; const char exp[] = ""; - NORM_BAD_2(dat1, dat2, exp1, exp2, JSTokenizer::SCRIPT_ENDED); - NORM_COMBINED_BAD_2(dat1, dat2, exp, JSTokenizer::SCRIPT_ENDED); + norm.test_normalization( + { + {dat1, exp1}, + {dat2, exp2}, + }, + {return_code(JSTokenizer::SCRIPT_ENDED)}); + + norm.test_normalization_combined({dat1, dat2}, exp, + {return_code(JSTokenizer::SCRIPT_ENDED)}); } SECTION("") { @@ -2779,8 +2718,15 @@ TEST_CASE("split in closing tag", "[JSNormalizer]") const char exp2[] = ""; const char exp[] = ""; - NORM_BAD_2(dat1, dat2, exp1, exp2, JSTokenizer::SCRIPT_ENDED); - NORM_COMBINED_BAD_2(dat1, dat2, exp, JSTokenizer::SCRIPT_ENDED); + norm.test_normalization( + { + {dat1, exp1}, + {dat2, exp2}, + }, + {return_code(JSTokenizer::SCRIPT_ENDED)}); + + norm.test_normalization_combined({dat1, dat2}, exp, + {return_code(JSTokenizer::SCRIPT_ENDED)}); } SECTION("str=''") { @@ -2790,8 +2736,15 @@ TEST_CASE("split in closing tag", "[JSNormalizer]") const char exp2[] = ""; const char exp[] = "var str='"; - NORM_BAD_2(dat1, dat2, exp1, exp2, JSTokenizer::CLOSING_TAG); - NORM_COMBINED_BAD_2(dat1, dat2, exp, JSTokenizer::CLOSING_TAG); + norm.test_normalization( + { + {dat1, exp1}, + {dat2, exp2}, + }, + {return_code(JSTokenizer::CLOSING_TAG)}); + + norm.test_normalization_combined({dat1, dat2}, exp, + {return_code(JSTokenizer::CLOSING_TAG)}); } SECTION("str=''") { @@ -2801,8 +2754,15 @@ TEST_CASE("split in closing tag", "[JSNormalizer]") const char exp2[] = ""; const char exp[] = "var str='"; - NORM_BAD_2(dat1, dat2, exp1, exp2, JSTokenizer::CLOSING_TAG); - NORM_COMBINED_BAD_2(dat1, dat2, exp, JSTokenizer::CLOSING_TAG); + norm.test_normalization( + { + {dat1, exp1}, + {dat2, exp2}, + }, + {return_code(JSTokenizer::CLOSING_TAG)}); + + norm.test_normalization_combined({dat1, dat2}, exp, + {return_code(JSTokenizer::CLOSING_TAG)}); } SECTION("") { @@ -2814,8 +2774,16 @@ TEST_CASE("split in closing tag", "[JSNormalizer]") const char exp3[] = ""; const char exp[] = ""; - NORM_BAD_3(dat1, dat2, dat3, exp1, exp2, exp3, JSTokenizer::SCRIPT_ENDED); - NORM_COMBINED_BAD_3(dat1, dat2, dat3, exp, JSTokenizer::SCRIPT_ENDED); + norm.test_normalization( + { + {dat1, exp1}, + {dat2, exp2}, + {dat3, exp3}, + }, + {return_code(JSTokenizer::SCRIPT_ENDED)}); + + norm.test_normalization_combined({dat1, dat2, dat3}, exp, + {return_code(JSTokenizer::SCRIPT_ENDED)}); } SECTION("str=''") { @@ -2827,8 +2795,16 @@ TEST_CASE("split in closing tag", "[JSNormalizer]") const char exp3[] = ""; const char exp[] = "var str=\""; - NORM_BAD_3(dat1, dat2, dat3, exp1, exp2, exp3, JSTokenizer::CLOSING_TAG); - NORM_COMBINED_BAD_3(dat1, dat2, dat3, exp, JSTokenizer::CLOSING_TAG); + norm.test_normalization( + { + {dat1, exp1}, + {dat2, exp2}, + {dat3, exp3}, + }, + {return_code(JSTokenizer::CLOSING_TAG)}); + + norm.test_normalization_combined({dat1, dat2, dat3}, exp, + {return_code(JSTokenizer::CLOSING_TAG)}); } SECTION("::::") { @@ -2840,13 +2816,23 @@ TEST_CASE("split in closing tag", "[JSNormalizer]") const char exp3[] = ""; const char exp[] = "::::"; - NORM_BAD_3(dat1, dat2, dat3, exp1, exp2, exp3, JSTokenizer::SCRIPT_ENDED); - NORM_COMBINED_BAD_3(dat1, dat2, dat3, exp, JSTokenizer::SCRIPT_ENDED); + norm.test_normalization( + { + {dat1, exp1}, + {dat2, exp2}, + {dat3, exp3}, + }, + {return_code(JSTokenizer::SCRIPT_ENDED)}); + + norm.test_normalization_combined({dat1, dat2, dat3}, exp, + {return_code(JSTokenizer::SCRIPT_ENDED)}); } } TEST_CASE("split in string literal", "[JSNormalizer]") { + auto norm = default_config.derive({normalize_identifiers(false), use_expected_for_last_pdu(true)}); + SECTION("\\ LF") { const char dat1[] = "var str =\"any\\"; @@ -2855,8 +2841,13 @@ TEST_CASE("split in string literal", "[JSNormalizer]") const char exp2[] = "one\";"; const char exp[] = "var str=\"anyone\";"; - NORMALIZE_2(dat1, dat2, exp1, exp2); - NORM_COMBINED_2(dat1, dat2, exp); + norm.test_normalization( + { + {dat1, exp1}, + {dat2, exp2}, + }); + + norm.test_normalization_combined({dat1, dat2}, exp); } SECTION("\\ CR") { @@ -2866,8 +2857,13 @@ TEST_CASE("split in string literal", "[JSNormalizer]") const char exp2[] = "one\";"; const char exp[] = "var str=\"anyone\";"; - NORMALIZE_2(dat1, dat2, exp1, exp2); - NORM_COMBINED_2(dat1, dat2, exp); + norm.test_normalization( + { + {dat1, exp1}, + {dat2, exp2}, + }); + + norm.test_normalization_combined({dat1, dat2}, exp); } SECTION("\\CR LF") { @@ -2877,8 +2873,13 @@ TEST_CASE("split in string literal", "[JSNormalizer]") const char exp2[] = "one\";"; const char exp[] = "var str=\"anyone\";"; - NORMALIZE_2(dat1, dat2, exp1, exp2); - NORM_COMBINED_2(dat1, dat2, exp); + norm.test_normalization( + { + {dat1, exp1}, + {dat2, exp2}, + }); + + norm.test_normalization_combined({dat1, dat2}, exp); } SECTION("\\ CRLF") { @@ -2888,8 +2889,13 @@ TEST_CASE("split in string literal", "[JSNormalizer]") const char exp2[] = "one\";"; const char exp[] = "var str=\"anyone\";"; - NORMALIZE_2(dat1, dat2, exp1, exp2); - NORM_COMBINED_2(dat1, dat2, exp); + norm.test_normalization( + { + {dat1, exp1}, + {dat2, exp2}, + }); + + norm.test_normalization_combined({dat1, dat2}, exp); } SECTION("\\ \"") { @@ -2899,8 +2905,13 @@ TEST_CASE("split in string literal", "[JSNormalizer]") const char exp2[] = "\\\"one\";"; const char exp[] = "var str=\"any\\\"one\";"; - NORMALIZE_2(dat1, dat2, exp1, exp2); - NORM_COMBINED_2(dat1, dat2, exp); + norm.test_normalization( + { + {dat1, exp1}, + {dat2, exp2}, + }); + + norm.test_normalization_combined({dat1, dat2}, exp); } SECTION("\\ \'") { @@ -2910,8 +2921,13 @@ TEST_CASE("split in string literal", "[JSNormalizer]") const char exp2[] = "\'one\";"; const char exp[] = "var str=\"any\\\'one\";"; - NORMALIZE_2(dat1, dat2, exp1, exp2); - NORM_COMBINED_2(dat1, dat2, exp); + norm.test_normalization( + { + {dat1, exp1}, + {dat2, exp2}, + }); + + norm.test_normalization_combined({dat1, dat2}, exp); } SECTION("\\ u1234tx") { @@ -2921,8 +2937,13 @@ TEST_CASE("split in string literal", "[JSNormalizer]") const char exp2[] = "\u1234tx\";"; const char exp[] = "var str=\"any\u1234tx\";"; - NORMALIZE_2(dat1, dat2, exp1, exp2); - NORM_COMBINED_2(dat1, dat2, exp); + norm.test_normalization( + { + {dat1, exp1}, + {dat2, exp2}, + }); + + norm.test_normalization_combined({dat1, dat2}, exp); } SECTION("\\u 1234tx") { @@ -2932,13 +2953,20 @@ TEST_CASE("split in string literal", "[JSNormalizer]") const char exp2[] = "\u1234tx\";"; const char exp[] = "var str=\"any\u1234tx\";"; - NORMALIZE_2(dat1, dat2, exp1, exp2); - NORM_COMBINED_2(dat1, dat2, exp); + norm.test_normalization( + { + {dat1, exp1}, + {dat2, exp2}, + }); + + norm.test_normalization_combined({dat1, dat2}, exp); } } TEST_CASE("split in identifier", "[JSNormalizer]") { + auto norm = default_config.derive({normalize_identifiers(false), use_expected_for_last_pdu(true)}); + SECTION("abc def") { const char dat1[] = "var abc"; @@ -2947,8 +2975,13 @@ TEST_CASE("split in identifier", "[JSNormalizer]") const char exp2[] = " abcdef=5"; const char exp[] = "var abcdef=5"; - NORMALIZE_2(dat1, dat2, exp1, exp2); - NORM_COMBINED_2(dat1, dat2, exp); + norm.test_normalization( + { + {dat1, exp1}, + {dat2, exp2}, + }); + + norm.test_normalization_combined({dat1, dat2}, exp); } SECTION("long identifier") { @@ -2958,13 +2991,20 @@ TEST_CASE("split in identifier", "[JSNormalizer]") const char exp2[] = "long_id_starts_here_long_id_ends_here;"; const char exp[] = "var res=something+long_id_starts_here_long_id_ends_here;"; - NORMALIZE_2(dat1, dat2, exp1, exp2); - NORM_COMBINED_2(dat1, dat2, exp); + norm.test_normalization( + { + {dat1, exp1}, + {dat2, exp2}, + }); + + norm.test_normalization_combined({dat1, dat2}, exp); } } TEST_CASE("split in keyword", "[JSNormalizer]") { + auto norm = default_config.derive({normalize_identifiers(false), use_expected_for_last_pdu(true)}); + SECTION("finally") { const char dat1[] = "\nfin"; @@ -2973,8 +3013,13 @@ TEST_CASE("split in keyword", "[JSNormalizer]") const char exp2[] = "finally;"; const char exp[] = "finally;"; - NORMALIZE_2(dat1, dat2, exp1, exp2); - NORM_COMBINED_2(dat1, dat2, exp); + norm.test_normalization( + { + {dat1, exp1}, + {dat2, exp2}, + }); + + norm.test_normalization_combined({dat1, dat2}, exp); } SECTION("in") { @@ -2984,8 +3029,13 @@ TEST_CASE("split in keyword", "[JSNormalizer]") const char exp2[] = "in"; const char exp[] = "in"; - NORMALIZE_2(dat1, dat2, exp1, exp2); - NORM_COMBINED_2(dat1, dat2, exp); + norm.test_normalization( + { + {dat1, exp1}, + {dat2, exp2}, + }); + + norm.test_normalization_combined({dat1, dat2}, exp); } SECTION("instanceof") { @@ -2997,13 +3047,21 @@ TEST_CASE("split in keyword", "[JSNormalizer]") const char exp3[] = "instanceof"; const char exp[] = "instanceof"; - NORMALIZE_3(dat1, dat2, dat3, exp1, exp2, exp3); - NORM_COMBINED_3(dat1, dat2, dat3, exp); + norm.test_normalization( + { + {dat1, exp1}, + {dat2, exp2}, + {dat3, exp3}, + }); + + norm.test_normalization_combined({dat1, dat2, dat3}, exp); } } TEST_CASE("split in integer literal", "[JSNormalizer]") { + auto norm = default_config.derive({normalize_identifiers(false), use_expected_for_last_pdu(true)}); + SECTION("1 2;") { const char dat1[] = "1"; @@ -3012,8 +3070,13 @@ TEST_CASE("split in integer literal", "[JSNormalizer]") const char exp2[] = "12;"; const char exp[] = "12;"; - NORMALIZE_2(dat1, dat2, exp1, exp2); - NORM_COMBINED_2(dat1, dat2, exp); + norm.test_normalization( + { + {dat1, exp1}, + {dat2, exp2}, + }); + + norm.test_normalization_combined({dat1, dat2}, exp); } SECTION("0 b01;") { @@ -3023,8 +3086,13 @@ TEST_CASE("split in integer literal", "[JSNormalizer]") const char exp2[] = "0b01;"; const char exp[] = "0b01;"; - NORMALIZE_2(dat1, dat2, exp1, exp2); - NORM_COMBINED_2(dat1, dat2, exp); + norm.test_normalization( + { + {dat1, exp1}, + {dat2, exp2}, + }); + + norm.test_normalization_combined({dat1, dat2}, exp); } SECTION("0o 12;") { @@ -3034,8 +3102,13 @@ TEST_CASE("split in integer literal", "[JSNormalizer]") const char exp2[] = "0o12;"; const char exp[] = "0o12;"; - NORMALIZE_2(dat1, dat2, exp1, exp2); - NORM_COMBINED_2(dat1, dat2, exp); + norm.test_normalization( + { + {dat1, exp1}, + {dat2, exp2}, + }); + + norm.test_normalization_combined({dat1, dat2}, exp); } SECTION("0 12;") { @@ -3045,8 +3118,13 @@ TEST_CASE("split in integer literal", "[JSNormalizer]") const char exp2[] = "012;"; const char exp[] = "012;"; - NORMALIZE_2(dat1, dat2, exp1, exp2); - NORM_COMBINED_2(dat1, dat2, exp); + norm.test_normalization( + { + {dat1, exp1}, + {dat2, exp2}, + }); + + norm.test_normalization_combined({dat1, dat2}, exp); } SECTION("01 9;") { @@ -3056,8 +3134,13 @@ TEST_CASE("split in integer literal", "[JSNormalizer]") const char exp2[] = "019;"; const char exp[] = "019;"; - NORMALIZE_2(dat1, dat2, exp1, exp2); - NORM_COMBINED_2(dat1, dat2, exp); + norm.test_normalization( + { + {dat1, exp1}, + {dat2, exp2}, + }); + + norm.test_normalization_combined({dat1, dat2}, exp); } SECTION(". 12;") { @@ -3067,8 +3150,13 @@ TEST_CASE("split in integer literal", "[JSNormalizer]") const char exp2[] = ".12;"; const char exp[] = ".12;"; - NORMALIZE_2(dat1, dat2, exp1, exp2); - NORM_COMBINED_2(dat1, dat2, exp); + norm.test_normalization( + { + {dat1, exp1}, + {dat2, exp2}, + }); + + norm.test_normalization_combined({dat1, dat2}, exp); } SECTION("0 x 12;") { @@ -3080,8 +3168,14 @@ TEST_CASE("split in integer literal", "[JSNormalizer]") const char exp3[] = "0x12;"; const char exp[] = "0x12;"; - NORMALIZE_3(dat1, dat2, dat3, exp1, exp2, exp3); - NORM_COMBINED_3(dat1, dat2, dat3, exp); + norm.test_normalization( + { + {dat1, exp1}, + {dat2, exp2}, + {dat3, exp3}, + }); + + norm.test_normalization_combined({dat1, dat2, dat3}, exp); } SECTION("1 _ 2;") { @@ -3093,8 +3187,14 @@ TEST_CASE("split in integer literal", "[JSNormalizer]") const char exp3[] = "1_2;"; const char exp[] = "1_2;"; - NORMALIZE_3(dat1, dat2, dat3, exp1, exp2, exp3); - NORM_COMBINED_3(dat1, dat2, dat3, exp); + norm.test_normalization( + { + {dat1, exp1}, + {dat2, exp2}, + {dat3, exp3}, + }); + + norm.test_normalization_combined({dat1, dat2, dat3}, exp); } SECTION("1 E 2;") { @@ -3106,8 +3206,14 @@ TEST_CASE("split in integer literal", "[JSNormalizer]") const char exp3[] = "1E2;"; const char exp[] = "1E2;"; - NORMALIZE_3(dat1, dat2, dat3, exp1, exp2, exp3); - NORM_COMBINED_3(dat1, dat2, dat3, exp); + norm.test_normalization( + { + {dat1, exp1}, + {dat2, exp2}, + {dat3, exp3}, + }); + + norm.test_normalization_combined({dat1, dat2, dat3}, exp); } } @@ -3121,32 +3227,30 @@ TEST_CASE("split and continuation combined", "[JSNormalizer]") const char src4[] = "\n" ""; const char exp1[] = "var_0000 var_0001"; - const char exp2[] = " var_0002 var_0003"; - const char exp3[] = " var_0002 var_0003"; - const char exp4[] = " var_0002 var_0003"; + const char exp2[] = "var_0000 var_0002 var_0003"; + const char exp3[] = "var_0000 var_0002 var_0003"; + const char exp4[] = "var_0000 var_0002 var_0003"; - char dst1[sizeof(exp1)]; - char dst2[sizeof(exp2)]; - char dst3[sizeof(exp3)]; - char dst4[sizeof(exp4)]; + JSTokenizerTester tester(default_config); - JSIdentifierCtx ident_ctx(norm_depth, max_scope_depth, s_ignored_ids, s_ignored_props); - JSNormalizer norm(ident_ctx, norm_depth, max_template_nesting, max_bracket_depth); + tester.do_pdu(src1); + tester.check_output(exp1); - DO(src1, sizeof(src1) - 1, dst1, sizeof(dst1) - 1); - CHECK(!memcmp(exp1, dst1, sizeof(exp1) - 1)); + tester.do_pdu(src2); + tester.check_output(exp2); + tester.run_checks({return_code(JSTokenizer::SCRIPT_ENDED)}); - TRY(src2, sizeof(src2) - 1, dst2, sizeof(dst2) - 1, JSTokenizer::SCRIPT_ENDED); - CHECK(!memcmp(exp2, dst2, sizeof(exp2) - 1)); + tester.do_pdu(src3); + tester.check_output(exp3); + tester.run_checks({return_code(JSTokenizer::SCRIPT_ENDED)}); - TRY(src3, sizeof(src3) - 1, dst3, sizeof(dst3) - 1, JSTokenizer::SCRIPT_ENDED); - CHECK(!memcmp(exp3, dst3, sizeof(exp3) - 1)); + tester.do_pdu(src4); + tester.check_output(exp4); - DO(src4, sizeof(src4) - 1, dst4, sizeof(dst4) - 1); - CHECK(!memcmp(exp4, dst4, sizeof(exp4) - 1)); - - CLOSE(); + tester.do_pdu(""); + tester.run_checks({return_code(JSTokenizer::SCRIPT_ENDED)}); } + SECTION("PDU 1 [cont] PDU 2 [cont] PDU 3 [end]") { const char src1[] = "<"; @@ -3154,41 +3258,49 @@ TEST_CASE("split and continuation combined", "[JSNormalizer]") const char src3[] = "-comment\n"; const char exp1[] = "<"; - const char exp2[] = ""); + tester.run_checks({return_code(JSTokenizer::SCRIPT_ENDED)}); } } TEST_CASE("memcap", "[JSNormalizer]") { + auto norm = default_config.derive({normalize_identifiers(false), use_expected_for_last_pdu(true)}); + SECTION("3 tokens") { const char dat1[] = "var abc=in"; @@ -3197,8 +3309,15 @@ TEST_CASE("memcap", "[JSNormalizer]") const char exp2[] = "input;"; const char exp[] = "var abc=input;"; - NORM_LIMITED(6, dat1, dat2, exp1, exp2); - NORM_COMBINED_LIMITED_2(6, dat1, dat2, exp); + norm.test_normalization( + { + {dat1, exp1}, + {dat2, exp2}, + }, + {max_token_buf_size(6)}); + + norm.test_normalization_combined({dat1, dat2}, exp, + {max_token_buf_size(6)}); } SECTION("2 tokens and a half") { @@ -3208,8 +3327,15 @@ TEST_CASE("memcap", "[JSNormalizer]") const char exp2[] = "input;"; const char exp[] = "var abc=input;"; - NORM_LIMITED(4, dat1, dat2, exp1, exp2); - NORM_COMBINED_LIMITED_2(4, dat1, dat2, exp); + norm.test_normalization( + { + {dat1, exp1}, + {dat2, exp2}, + }, + {max_token_buf_size(4)}); + + norm.test_normalization_combined({dat1, dat2}, exp, + {max_token_buf_size(4)}); } SECTION("1 token") { @@ -3219,8 +3345,15 @@ TEST_CASE("memcap", "[JSNormalizer]") const char exp2[] = "input;"; const char exp[] = "var abc=input;"; - NORM_LIMITED(2, dat1, dat2, exp1, exp2); - NORM_COMBINED_LIMITED_2(2, dat1, dat2, exp); + norm.test_normalization( + { + {dat1, exp1}, + {dat2, exp2}, + }, + {max_token_buf_size(2)}); + + norm.test_normalization_combined({dat1, dat2}, exp, + {max_token_buf_size(2)}); } SECTION("a half") { @@ -3230,13 +3363,22 @@ TEST_CASE("memcap", "[JSNormalizer]") const char exp2[] = ""; const char exp[] = "var abc=extract"; - NORM_LIMITED(5, dat1, dat2, exp1, exp2); - NORM_COMBINED_LIMITED_2(5, dat1, dat2, exp); + norm.test_normalization( + { + {dat1, exp1}, + {dat2, exp2}, + }, + {max_token_buf_size(5)}); + + norm.test_normalization_combined({dat1, dat2}, exp, + {max_token_buf_size(5)}); } } TEST_CASE("scope tracking", "[JSNormalizer]") { + auto norm = default_config.derive({normalize_identifiers(false), use_expected_for_last_pdu(true)}); + SECTION("parentheses") { const char dat1[] = "()"; @@ -3246,9 +3388,9 @@ TEST_CASE("scope tracking", "[JSNormalizer]") const char exp2[] = "()()()"; const char exp3[] = "((()))"; - NORMALIZE_1(dat1, exp1); - NORMALIZE_1(dat2, exp2); - NORMALIZE_1(dat3, exp3); + test_normalization(dat1, exp1); + test_normalization(dat2, exp2); + test_normalization(dat3, exp3); } SECTION("curly braces") { @@ -3259,9 +3401,9 @@ TEST_CASE("scope tracking", "[JSNormalizer]") const char exp2[] = "{}{}{}"; const char exp3[] = "{{{}}}"; - NORMALIZE_1(dat1, exp1); - NORMALIZE_1(dat2, exp2); - NORMALIZE_1(dat3, exp3); + test_normalization(dat1, exp1); + test_normalization(dat2, exp2); + test_normalization(dat3, exp3); } SECTION("square brackets") { @@ -3272,9 +3414,9 @@ TEST_CASE("scope tracking", "[JSNormalizer]") const char exp2[] = "[][][]"; const char exp3[] = "[[[]]]"; - NORMALIZE_1(dat1, exp1); - NORMALIZE_1(dat2, exp2); - NORMALIZE_1(dat3, exp3); + test_normalization(dat1, exp1); + test_normalization(dat2, exp2); + test_normalization(dat3, exp3); } SECTION("mix of brackets") { @@ -3285,9 +3427,9 @@ TEST_CASE("scope tracking", "[JSNormalizer]") const char exp2[] = "({})[]"; const char exp3[] = "(){[]}"; - NORMALIZE_1(dat1, exp1); - NORMALIZE_1(dat2, exp2); - NORMALIZE_1(dat3, exp3); + test_normalization(dat1, exp1); + test_normalization(dat2, exp2); + test_normalization(dat3, exp3); } SECTION("parentheses - wrong closing symbol") { @@ -3296,8 +3438,8 @@ TEST_CASE("scope tracking", "[JSNormalizer]") const char exp1[] = "({[("; const char exp2[] = "({[("; - NORM_BAD_1(dat1, exp1, JSTokenizer::WRONG_CLOSING_SYMBOL); - NORM_BAD_1(dat2, exp2, JSTokenizer::WRONG_CLOSING_SYMBOL); + test_normalization(dat1, exp1, {return_code(JSTokenizer::WRONG_CLOSING_SYMBOL)}); + test_normalization(dat2, exp2, {return_code(JSTokenizer::WRONG_CLOSING_SYMBOL)}); } SECTION("curly braces - wrong closing symbol") { @@ -3306,8 +3448,8 @@ TEST_CASE("scope tracking", "[JSNormalizer]") const char exp1[] = "({[{"; const char exp2[] = "({[{"; - NORM_BAD_1(dat1, exp1, JSTokenizer::WRONG_CLOSING_SYMBOL); - NORM_BAD_1(dat2, exp2, JSTokenizer::WRONG_CLOSING_SYMBOL); + test_normalization(dat1, exp1, {return_code(JSTokenizer::WRONG_CLOSING_SYMBOL)}); + test_normalization(dat2, exp2, {return_code(JSTokenizer::WRONG_CLOSING_SYMBOL)}); } SECTION("square brackets - wrong closing symbol") { @@ -3316,8 +3458,8 @@ TEST_CASE("scope tracking", "[JSNormalizer]") const char exp1[] = "([{["; const char exp2[] = "([{["; - NORM_BAD_1(dat1, exp1, JSTokenizer::WRONG_CLOSING_SYMBOL); - NORM_BAD_1(dat2, exp2, JSTokenizer::WRONG_CLOSING_SYMBOL); + test_normalization(dat1, exp1, {return_code(JSTokenizer::WRONG_CLOSING_SYMBOL)}); + test_normalization(dat2, exp2, {return_code(JSTokenizer::WRONG_CLOSING_SYMBOL)}); } SECTION("parentheses - mismatch") { @@ -3332,11 +3474,11 @@ TEST_CASE("scope tracking", "[JSNormalizer]") const char exp4[] = "("; const char exp5[] = "(()"; - NORM_BAD_1(dat1, exp1, JSTokenizer::WRONG_CLOSING_SYMBOL); - NORM_BAD_1(dat2, exp2, JSTokenizer::WRONG_CLOSING_SYMBOL); - NORM_BAD_1(dat3, exp3, JSTokenizer::WRONG_CLOSING_SYMBOL); - NORM_BAD_1(dat4, exp4, JSTokenizer::ENDED_IN_INNER_SCOPE); - NORM_BAD_1(dat5, exp5, JSTokenizer::ENDED_IN_INNER_SCOPE); + test_normalization(dat1, exp1, {return_code(JSTokenizer::WRONG_CLOSING_SYMBOL)}); + test_normalization(dat2, exp2, {return_code(JSTokenizer::WRONG_CLOSING_SYMBOL)}); + test_normalization(dat3, exp3, {return_code(JSTokenizer::WRONG_CLOSING_SYMBOL)}); + test_normalization(dat4, exp4, {return_code(JSTokenizer::ENDED_IN_INNER_SCOPE)}); + test_normalization(dat5, exp5, {return_code(JSTokenizer::ENDED_IN_INNER_SCOPE)}); } SECTION("curly braces - mismatch") { @@ -3351,11 +3493,11 @@ TEST_CASE("scope tracking", "[JSNormalizer]") const char exp4[] = "{"; const char exp5[] = "{{}"; - NORM_BAD_1(dat1, exp1, JSTokenizer::WRONG_CLOSING_SYMBOL); - NORM_BAD_1(dat2, exp2, JSTokenizer::WRONG_CLOSING_SYMBOL); - NORM_BAD_1(dat3, exp3, JSTokenizer::WRONG_CLOSING_SYMBOL); - NORM_BAD_1(dat4, exp4, JSTokenizer::ENDED_IN_INNER_SCOPE); - NORM_BAD_1(dat5, exp5, JSTokenizer::ENDED_IN_INNER_SCOPE); + test_normalization(dat1, exp1, {return_code(JSTokenizer::WRONG_CLOSING_SYMBOL)}); + test_normalization(dat2, exp2, {return_code(JSTokenizer::WRONG_CLOSING_SYMBOL)}); + test_normalization(dat3, exp3, {return_code(JSTokenizer::WRONG_CLOSING_SYMBOL)}); + test_normalization(dat4, exp4, {return_code(JSTokenizer::ENDED_IN_INNER_SCOPE)}); + test_normalization(dat5, exp5, {return_code(JSTokenizer::ENDED_IN_INNER_SCOPE)}); } SECTION("square brackets - mismatch") { @@ -3370,11 +3512,11 @@ TEST_CASE("scope tracking", "[JSNormalizer]") const char exp4[] = "["; const char exp5[] = "[[]"; - NORM_BAD_1(dat1, exp1, JSTokenizer::WRONG_CLOSING_SYMBOL); - NORM_BAD_1(dat2, exp2, JSTokenizer::WRONG_CLOSING_SYMBOL); - NORM_BAD_1(dat3, exp3, JSTokenizer::WRONG_CLOSING_SYMBOL); - NORM_BAD_1(dat4, exp4, JSTokenizer::ENDED_IN_INNER_SCOPE); - NORM_BAD_1(dat5, exp5, JSTokenizer::ENDED_IN_INNER_SCOPE); + test_normalization(dat1, exp1, {return_code(JSTokenizer::WRONG_CLOSING_SYMBOL)}); + test_normalization(dat2, exp2, {return_code(JSTokenizer::WRONG_CLOSING_SYMBOL)}); + test_normalization(dat3, exp3, {return_code(JSTokenizer::WRONG_CLOSING_SYMBOL)}); + test_normalization(dat4, exp4, {return_code(JSTokenizer::ENDED_IN_INNER_SCOPE)}); + test_normalization(dat5, exp5, {return_code(JSTokenizer::ENDED_IN_INNER_SCOPE)}); } SECTION("parentheses - continuation") { @@ -3384,8 +3526,13 @@ TEST_CASE("scope tracking", "[JSNormalizer]") const char exp2[] = "))"; const char exp[] = "(())"; - NORMALIZE_2(dat1, dat2, exp1, exp2); - NORM_COMBINED_2(dat1, dat2, exp); + norm.test_normalization( + { + {dat1, exp1}, + {dat2, exp2}, + }); + + norm.test_normalization_combined({dat1, dat2}, exp); } SECTION("curly braces - continuation") { @@ -3395,8 +3542,13 @@ TEST_CASE("scope tracking", "[JSNormalizer]") const char exp2[] = "}}"; const char exp[] = "{{}}"; - NORMALIZE_2(dat1, dat2, exp1, exp2); - NORM_COMBINED_2(dat1, dat2, exp); + norm.test_normalization( + { + {dat1, exp1}, + {dat2, exp2}, + }); + + norm.test_normalization_combined({dat1, dat2}, exp); } SECTION("square brackets - continuation") { @@ -3406,8 +3558,13 @@ TEST_CASE("scope tracking", "[JSNormalizer]") const char exp2[] = "]]"; const char exp[] = "[[]]"; - NORMALIZE_2(dat1, dat2, exp1, exp2); - NORM_COMBINED_2(dat1, dat2, exp); + norm.test_normalization( + { + {dat1, exp1}, + {dat2, exp2}, + }); + + norm.test_normalization_combined({dat1, dat2}, exp); } SECTION("parentheses - mismatch in continuation") { @@ -3423,11 +3580,25 @@ TEST_CASE("scope tracking", "[JSNormalizer]") const char exp5[] = "()"; const char exp6[] = "("; - NORM_BAD_2(dat1, dat2, exp1, exp2, JSTokenizer::WRONG_CLOSING_SYMBOL); - NORM_BAD_2(dat3, dat4, exp3, exp4, JSTokenizer::ENDED_IN_INNER_SCOPE); + norm.test_normalization( + { + {dat1, exp1}, + {dat2, exp2}, + }, + {return_code(JSTokenizer::WRONG_CLOSING_SYMBOL)}); + + norm.test_normalization( + { + {dat3, exp3}, + {dat4, exp4}, + }, + {return_code(JSTokenizer::ENDED_IN_INNER_SCOPE)}); - NORM_COMBINED_BAD_2(dat1, dat2, exp5, JSTokenizer::WRONG_CLOSING_SYMBOL); - NORM_COMBINED_BAD_2(dat3, dat4, exp6, JSTokenizer::ENDED_IN_INNER_SCOPE); + + norm.test_normalization_combined({dat1, dat2}, exp5, + {return_code(JSTokenizer::WRONG_CLOSING_SYMBOL)}); + norm.test_normalization_combined({dat3, dat4}, exp6, + {return_code(JSTokenizer::ENDED_IN_INNER_SCOPE)}); } SECTION("curly braces - mismatch in continuation") { @@ -3443,11 +3614,25 @@ TEST_CASE("scope tracking", "[JSNormalizer]") const char exp5[] = "{}"; const char exp6[] = "{"; - NORM_BAD_2(dat1, dat2, exp1, exp2, JSTokenizer::WRONG_CLOSING_SYMBOL); - NORM_BAD_2(dat3, dat4, exp3, exp4, JSTokenizer::ENDED_IN_INNER_SCOPE); + norm.test_normalization( + { + {dat1, exp1}, + {dat2, exp2}, + }, + {return_code(JSTokenizer::WRONG_CLOSING_SYMBOL)}); + + norm.test_normalization( + { + {dat3, exp3}, + {dat4, exp4}, + }, + {return_code(JSTokenizer::ENDED_IN_INNER_SCOPE)}); - NORM_COMBINED_BAD_2(dat1, dat2, exp5, JSTokenizer::WRONG_CLOSING_SYMBOL); - NORM_COMBINED_BAD_2(dat3, dat4, exp6, JSTokenizer::ENDED_IN_INNER_SCOPE); + + norm.test_normalization_combined({dat1, dat2}, exp5, + {return_code(JSTokenizer::WRONG_CLOSING_SYMBOL)}); + norm.test_normalization_combined({dat3, dat4}, exp6, + {return_code(JSTokenizer::ENDED_IN_INNER_SCOPE)}); } SECTION("square brackets - mismatch in continuation") { @@ -3463,11 +3648,25 @@ TEST_CASE("scope tracking", "[JSNormalizer]") const char exp5[] = "[]"; const char exp6[] = "["; - NORM_BAD_2(dat1, dat2, exp1, exp2, JSTokenizer::WRONG_CLOSING_SYMBOL); - NORM_BAD_2(dat3, dat4, exp3, exp4, JSTokenizer::ENDED_IN_INNER_SCOPE); + norm.test_normalization( + { + {dat1, exp1}, + {dat2, exp2}, + }, + {return_code(JSTokenizer::WRONG_CLOSING_SYMBOL)}); + + norm.test_normalization( + { + {dat3, exp3}, + {dat4, exp4}, + }, + {return_code(JSTokenizer::ENDED_IN_INNER_SCOPE)}); - NORM_COMBINED_BAD_2(dat1, dat2, exp5, JSTokenizer::WRONG_CLOSING_SYMBOL); - NORM_COMBINED_BAD_2(dat3, dat4, exp6, JSTokenizer::ENDED_IN_INNER_SCOPE); + + norm.test_normalization_combined({dat1, dat2}, exp5, + {return_code(JSTokenizer::WRONG_CLOSING_SYMBOL)}); + norm.test_normalization_combined({dat3, dat4}, exp6, + {return_code(JSTokenizer::ENDED_IN_INNER_SCOPE)}); } } @@ -3477,7 +3676,7 @@ TEST_CASE("scope misc", "[JSNormalizer]") const char* open = "1+("; const char* close = "-1)"; - SECTION("max stack") + SECTION("one below max stack") { std::string scr; @@ -3486,24 +3685,10 @@ TEST_CASE("scope misc", "[JSNormalizer]") for (int i = 0; i < stack_limit - 1; ++i) scr += close; - const char* dat = scr.c_str(); - int dat_len = strlen(dat); - const char* exp = scr.c_str(); - int exp_len = strlen(exp); - char* act = new char[exp_len]; - - JSIdentifierCtxStub ident_ctx; - JSNormalizer norm(ident_ctx, norm_depth, max_template_nesting, max_bracket_depth); - - DO(dat, dat_len, act, exp_len); - CHECK(!memcmp(exp, act, exp_len)); - - delete[] act; - - CLOSE(); + test_normalization(scr, scr, {return_code(JSTokenizer::SCRIPT_CONTINUE)}); } - SECTION("max stack") + SECTION("over max stack") { std::string scr; std::string nsc; @@ -3516,24 +3701,13 @@ TEST_CASE("scope misc", "[JSNormalizer]") nsc += open; nsc += "1+"; - const char* dat = scr.c_str(); - int dat_len = strlen(dat); - const char* exp = nsc.c_str(); - int exp_len = strlen(exp); - char* act = new char[exp_len]; - - JSIdentifierCtxStub ident_ctx; - JSNormalizer norm(ident_ctx, norm_depth, max_template_nesting, max_bracket_depth); - - TRY(dat, dat_len, act, exp_len, JSTokenizer::BRACKET_NESTING_OVERFLOW); - CHECK(!memcmp(exp, act, exp_len)); - - delete[] act; + test_normalization(scr, nsc, {return_code(JSTokenizer::BRACKET_NESTING_OVERFLOW)}); } } TEST_CASE("scope tail handling", "[JSNormalizer]") { + auto norm = default_config.derive({normalize_identifiers(false), use_expected_for_last_pdu(true)}); // Padding ':' symbol has been chosen, since it: // * forms a single state for Parser // * doesn't insert white spaces @@ -3565,13 +3739,28 @@ TEST_CASE("scope tail handling", "[JSNormalizer]") const char exp8[] = "{}{{::::::::::{}}}::"; const char exp9[] = "[][[::::::::::::]][]"; - NORMALIZE_2(dat1, dat2, exp1, exp2); - NORMALIZE_2(dat3, dat4, exp3, exp4); - NORMALIZE_2(dat5, dat6, exp5, exp6); + norm.test_normalization( + { + {dat1, exp1}, + {dat2, exp2}, + }); + + norm.test_normalization( + { + {dat3, exp3}, + {dat4, exp4}, + }); + + norm.test_normalization( + { + {dat5, exp5}, + {dat6, exp6}, + }); + - NORM_COMBINED_2(dat1, dat2, exp7); - NORM_COMBINED_2(dat3, dat4, exp8); - NORM_COMBINED_2(dat5, dat6, exp9); + norm.test_normalization_combined({dat1, dat2}, exp7); + norm.test_normalization_combined({dat3, dat4}, exp8); + norm.test_normalization_combined({dat5, dat6}, exp9); } SECTION("opening scope-symbols in the tail") @@ -3593,13 +3782,28 @@ TEST_CASE("scope tail handling", "[JSNormalizer]") const char exp8[] = ":::::::::::{:::::::}"; const char exp9[] = "::::[:::::::::::]:::"; - NORMALIZE_2(dat1, dat2, exp1, exp2); - NORMALIZE_2(dat3, dat4, exp3, exp4); - NORMALIZE_2(dat5, dat6, exp5, exp6); + norm.test_normalization( + { + {dat1, exp1}, + {dat2, exp2}, + }); + + norm.test_normalization( + { + {dat3, exp3}, + {dat4, exp4}, + }); + + norm.test_normalization( + { + {dat5, exp5}, + {dat6, exp6}, + }); + - NORM_COMBINED_2(dat1, dat2, exp7); - NORM_COMBINED_2(dat3, dat4, exp8); - NORM_COMBINED_2(dat5, dat6, exp9); + norm.test_normalization_combined({dat1, dat2}, exp7); + norm.test_normalization_combined({dat3, dat4}, exp8); + norm.test_normalization_combined({dat5, dat6}, exp9); } SECTION("closing scope-symbols in the tail") @@ -3621,13 +3825,28 @@ TEST_CASE("scope tail handling", "[JSNormalizer]") const char exp8[] = "{{{{:::::::}:::::}}}"; const char exp9[] = "[::::::::]::::::::::"; - NORMALIZE_2(dat1, dat2, exp1, exp2); - NORMALIZE_2(dat3, dat4, exp3, exp4); - NORMALIZE_2(dat5, dat6, exp5, exp6); + norm.test_normalization( + { + {dat1, exp1}, + {dat2, exp2}, + }); + + norm.test_normalization( + { + {dat3, exp3}, + {dat4, exp4}, + }); - NORM_COMBINED_2(dat1, dat2, exp7); - NORM_COMBINED_2(dat3, dat4, exp8); - NORM_COMBINED_2(dat5, dat6, exp9); + norm.test_normalization( + { + {dat5, exp5}, + {dat6, exp6}, + }); + + + norm.test_normalization_combined({dat1, dat2}, exp7); + norm.test_normalization_combined({dat3, dat4}, exp8); + norm.test_normalization_combined({dat5, dat6}, exp9); } SECTION("newly opening scope-symbols in the tail") @@ -3649,13 +3868,28 @@ TEST_CASE("scope tail handling", "[JSNormalizer]") const char exp8[] = "{:{:{:{:{:{:::}}}}}}"; const char exp9[] = "[:[:[:::[:::::::]]]]"; - NORMALIZE_2(dat1, dat2, exp1, exp2); - NORMALIZE_2(dat3, dat4, exp3, exp4); - NORMALIZE_2(dat5, dat6, exp5, exp6); + norm.test_normalization( + { + {dat1, exp1}, + {dat2, exp2}, + }); + + norm.test_normalization( + { + {dat3, exp3}, + {dat4, exp4}, + }); + + norm.test_normalization( + { + {dat5, exp5}, + {dat6, exp6}, + }); + - NORM_COMBINED_2(dat1, dat2, exp7); - NORM_COMBINED_2(dat3, dat4, exp8); - NORM_COMBINED_2(dat5, dat6, exp9); + norm.test_normalization_combined({dat1, dat2}, exp7); + norm.test_normalization_combined({dat3, dat4}, exp8); + norm.test_normalization_combined({dat5, dat6}, exp9); } SECTION("fully closing scope-symbols in the tail") @@ -3677,13 +3911,28 @@ TEST_CASE("scope tail handling", "[JSNormalizer]") const char exp8[] = "{{{{}:}:}:}:::::{}{}"; const char exp9[] = "[[:::::::]:][::::::]"; - NORMALIZE_2(dat1, dat2, exp1, exp2); - NORMALIZE_2(dat3, dat4, exp3, exp4); - NORMALIZE_2(dat5, dat6, exp5, exp6); + norm.test_normalization( + { + {dat1, exp1}, + {dat2, exp2}, + }); + + norm.test_normalization( + { + {dat3, exp3}, + {dat4, exp4}, + }); - NORM_COMBINED_2(dat1, dat2, exp7); - NORM_COMBINED_2(dat3, dat4, exp8); - NORM_COMBINED_2(dat5, dat6, exp9); + norm.test_normalization( + { + {dat5, exp5}, + {dat6, exp6}, + }); + + + norm.test_normalization_combined({dat1, dat2}, exp7); + norm.test_normalization_combined({dat3, dat4}, exp8); + norm.test_normalization_combined({dat5, dat6}, exp9); } SECTION("extra scope-symbols in the tail") @@ -3705,13 +3954,34 @@ TEST_CASE("scope tail handling", "[JSNormalizer]") const char exp8[] = "{{{{{{{{}}}}}}"; const char exp9[] = "[[[[[[[[]]]]]]]"; - NORM_BAD_2(dat1, dat2, exp1, exp2, JSTokenizer::WRONG_CLOSING_SYMBOL); - NORM_BAD_2(dat3, dat4, exp3, exp4, JSTokenizer::WRONG_CLOSING_SYMBOL); - NORM_BAD_2(dat5, dat6, exp5, exp6, JSTokenizer::ENDED_IN_INNER_SCOPE); + norm.test_normalization( + { + {dat1, exp1}, + {dat2, exp2}, + }, + {return_code(JSTokenizer::WRONG_CLOSING_SYMBOL)}); + + norm.test_normalization( + { + {dat3, exp3}, + {dat4, exp4}, + }, + {return_code(JSTokenizer::WRONG_CLOSING_SYMBOL)}); - NORM_COMBINED_BAD_2(dat1, dat2, exp7, JSTokenizer::WRONG_CLOSING_SYMBOL); - NORM_COMBINED_BAD_2(dat3, dat4, exp8, JSTokenizer::WRONG_CLOSING_SYMBOL); - NORM_COMBINED_BAD_2(dat5, dat6, exp9, JSTokenizer::ENDED_IN_INNER_SCOPE); + norm.test_normalization( + { + {dat5, exp5}, + {dat6, exp6}, + }, + {return_code(JSTokenizer::ENDED_IN_INNER_SCOPE)}); + + + norm.test_normalization_combined({dat1, dat2}, exp7, + {return_code(JSTokenizer::WRONG_CLOSING_SYMBOL)}); + norm.test_normalization_combined({dat3, dat4}, exp8, + {return_code(JSTokenizer::WRONG_CLOSING_SYMBOL)}); + norm.test_normalization_combined({dat5, dat6}, exp9, + {return_code(JSTokenizer::ENDED_IN_INNER_SCOPE)}); } SECTION("overwriting scope-symbols in the tail") @@ -3733,18 +4003,35 @@ TEST_CASE("scope tail handling", "[JSNormalizer]") const char exp8[] = "({[(:):]{}{}}[]())"; const char exp9[] = "{{{{}[[]]((()))}}}"; - NORMALIZE_2(dat1, dat2, exp1, exp2); - NORMALIZE_2(dat3, dat4, exp3, exp4); - NORMALIZE_2(dat5, dat6, exp5, exp6); + norm.test_normalization( + { + {dat1, exp1}, + {dat2, exp2}, + }); + + norm.test_normalization( + { + {dat3, exp3}, + {dat4, exp4}, + }); + + norm.test_normalization( + { + {dat5, exp5}, + {dat6, exp6}, + }); + - NORM_COMBINED_2(dat1, dat2, exp7); - NORM_COMBINED_2(dat3, dat4, exp8); - NORM_COMBINED_2(dat5, dat6, exp9); + norm.test_normalization_combined({dat1, dat2}, exp7); + norm.test_normalization_combined({dat3, dat4}, exp8); + norm.test_normalization_combined({dat5, dat6}, exp9); } } TEST_CASE("scope regex groups", "[JSNormalizer]") { + auto norm = default_config.derive({normalize_identifiers(false), use_expected_for_last_pdu(true)}); + SECTION("parentheses") { const char dat1[] = "a=/()/;"; @@ -3754,9 +4041,9 @@ TEST_CASE("scope regex groups", "[JSNormalizer]") const char exp2[] = "b=/()()()/;"; const char exp3[] = "c=/((()))/;"; - NORMALIZE_1(dat1, exp1); - NORMALIZE_1(dat2, exp2); - NORMALIZE_1(dat3, exp3); + test_normalization_noident(dat1, exp1); + test_normalization_noident(dat2, exp2); + test_normalization_noident(dat3, exp3); } SECTION("curly braces") { @@ -3767,9 +4054,9 @@ TEST_CASE("scope regex groups", "[JSNormalizer]") const char exp2[] = "b=/{}{}{}/;"; const char exp3[] = "c=/{{{}}}/;"; - NORMALIZE_1(dat1, exp1); - NORMALIZE_1(dat2, exp2); - NORMALIZE_1(dat3, exp3); + test_normalization_noident(dat1, exp1); + test_normalization_noident(dat2, exp2); + test_normalization_noident(dat3, exp3); } SECTION("square brackets") { @@ -3782,10 +4069,10 @@ TEST_CASE("scope regex groups", "[JSNormalizer]") const char exp3[] = "c=/[[[[[]/;"; const char exp4[] = "d=/[/]/;"; - NORMALIZE_1(dat1, exp1); - NORMALIZE_1(dat2, exp2); - NORMALIZE_1(dat3, exp3); - NORMALIZE_1(dat4, exp4); + test_normalization_noident(dat1, exp1); + test_normalization_noident(dat2, exp2); + test_normalization_noident(dat3, exp3); + test_normalization_noident(dat4, exp4); } SECTION("mix of brackets") { @@ -3796,9 +4083,9 @@ TEST_CASE("scope regex groups", "[JSNormalizer]") const char exp2[] = "b=/({})[]/"; const char exp3[] = "c=/(){[]}/"; - NORMALIZE_1(dat1, exp1); - NORMALIZE_1(dat2, exp2); - NORMALIZE_1(dat3, exp3); + test_normalization_noident(dat1, exp1); + test_normalization_noident(dat2, exp2); + test_normalization_noident(dat3, exp3); } SECTION("square brackets - raw bracket") { @@ -3809,9 +4096,9 @@ TEST_CASE("scope regex groups", "[JSNormalizer]") const char exp2[] = "/[]]/"; const char exp3[] = "/][]]/g"; - NORMALIZE_1(dat1, exp1); - NORMALIZE_1(dat2, exp2); - NORMALIZE_1(dat3, exp3); + test_normalization_noident(dat1, exp1); + test_normalization_noident(dat2, exp2); + test_normalization_noident(dat3, exp3); } SECTION("parentheses - mismatch") { @@ -3822,9 +4109,9 @@ TEST_CASE("scope regex groups", "[JSNormalizer]") const char exp2[] = "/()"; const char exp3[] = "/( ()) "; - NORM_BAD_1(dat1, exp1, JSTokenizer::BAD_TOKEN); - NORM_BAD_1(dat2, exp2, JSTokenizer::BAD_TOKEN); - NORM_BAD_1(dat3, exp3, JSTokenizer::BAD_TOKEN); + test_normalization(dat1, exp1, {return_code(JSTokenizer::BAD_TOKEN)}); + test_normalization(dat2, exp2, {return_code(JSTokenizer::BAD_TOKEN)}); + test_normalization(dat3, exp3, {return_code(JSTokenizer::BAD_TOKEN)}); } SECTION("parentheses - continuation") { @@ -3834,8 +4121,13 @@ TEST_CASE("scope regex groups", "[JSNormalizer]") const char exp2[] = "))/"; const char exp[] = "/(())/"; - NORMALIZE_2(dat1, dat2, exp1, exp2); - NORM_COMBINED_2(dat1, dat2, exp); + norm.test_normalization( + { + {dat1, exp1}, + {dat2, exp2}, + }); + + norm.test_normalization_combined({dat1, dat2}, exp); } SECTION("square brackets - continuation") { @@ -3845,8 +4137,13 @@ TEST_CASE("scope regex groups", "[JSNormalizer]") const char exp2[] = "[]/"; const char exp[] = "/[[[]/"; - NORMALIZE_2(dat1, dat2, exp1, exp2); - NORM_COMBINED_2(dat1, dat2, exp); + norm.test_normalization( + { + {dat1, exp1}, + {dat2, exp2}, + }); + + norm.test_normalization_combined({dat1, dat2}, exp); } SECTION("parentheses - mismatch in continuation") { @@ -3856,8 +4153,15 @@ TEST_CASE("scope regex groups", "[JSNormalizer]") const char exp2[] = ")"; const char exp[] = "/()"; - NORM_BAD_2(dat1, dat2, exp1, exp2, JSTokenizer::BAD_TOKEN); - NORM_COMBINED_BAD_2(dat1, dat2, exp, JSTokenizer::BAD_TOKEN); + norm.test_normalization( + { + {dat1, exp1}, + {dat2, exp2}, + }, + {return_code(JSTokenizer::BAD_TOKEN)}); + + norm.test_normalization_combined({dat1, dat2}, exp, + {return_code(JSTokenizer::BAD_TOKEN)}); } } @@ -3878,11 +4182,11 @@ TEST_CASE("ignored identifiers", "[JSNormalizer]") const char exp4[] = "var_0000(0);console(1);var_0001(2);var_0002(3);var_0003(4);"; const char exp5[] = "var_0000(0);eval(1);var_0001(2);var_0002(3);var_0003(4);"; - NORMALIZE_S(dat1, exp1); - NORMALIZE_S(dat2, exp2); - NORMALIZE_S(dat3, exp3); - NORMALIZE_S(dat4, exp4); - NORMALIZE_S(dat5, exp5); + test_normalization(dat1, exp1); + test_normalization(dat2, exp2); + test_normalization(dat3, exp3); + test_normalization(dat4, exp4); + test_normalization(dat5, exp5); } SECTION("inner objects") @@ -3896,10 +4200,10 @@ TEST_CASE("ignored identifiers", "[JSNormalizer]") const char exp3[] = "eval.alpha.bravo.charlie.delta"; const char exp4[] = "eval.alpha.bravo.console.delta"; - NORMALIZE_S(dat1, exp1); - NORMALIZE_S(dat2, exp2); - NORMALIZE_S(dat3, exp3); - NORMALIZE_S(dat4, exp4); + test_normalization(dat1, exp1); + test_normalization(dat2, exp2); + test_normalization(dat3, exp3); + test_normalization(dat4, exp4); } SECTION("function calls") @@ -3929,18 +4233,18 @@ TEST_CASE("ignored identifiers", "[JSNormalizer]") const char exp11[] = "var_0000.var_0001().var_0002"; const char exp12[] = "var_0000().var_0001.var_0002"; - NORMALIZE_S(dat1, exp1); - NORMALIZE_S(dat2, exp2); - NORMALIZE_S(dat3, exp3); - NORMALIZE_S(dat4, exp4); - NORMALIZE_S(dat5, exp5); - NORMALIZE_S(dat6, exp6); - NORMALIZE_S(dat7, exp7); - NORMALIZE_S(dat8, exp8); - NORMALIZE_S(dat9, exp9); - NORMALIZE_S(dat10, exp10); - NORMALIZE_S(dat11, exp11); - NORMALIZE_S(dat12, exp12); + test_normalization(dat1, exp1); + test_normalization(dat2, exp2); + test_normalization(dat3, exp3); + test_normalization(dat4, exp4); + test_normalization(dat5, exp5); + test_normalization(dat6, exp6); + test_normalization(dat7, exp7); + test_normalization(dat8, exp8); + test_normalization(dat9, exp9); + test_normalization(dat10, exp10); + test_normalization(dat11, exp11); + test_normalization(dat12, exp12); } } @@ -4034,47 +4338,47 @@ TEST_CASE("ignored identifier chain tracking", "[JSNormalizer]") const char exp40[] = "eval.var.foo"; const char exp41[] = "eval.break().foo"; - NORMALIZE_S(dat1, exp1); - NORMALIZE_S(dat2, exp2); - NORMALIZE_S(dat3, exp3); - NORMALIZE_S(dat4, exp4); - NORMALIZE_S(dat5, exp5); - NORMALIZE_S(dat6, exp6); - NORMALIZE_S(dat7, exp7); - NORMALIZE_S(dat8, exp8); - NORMALIZE_S(dat9, exp9); - NORMALIZE_S(dat10, exp10); - NORMALIZE_S(dat11, exp11); - NORMALIZE_S(dat12, exp12); - NORMALIZE_S(dat13, exp13); - NORMALIZE_S(dat14, exp14); - NORMALIZE_S(dat15, exp15); - NORMALIZE_S(dat16, exp16); - NORMALIZE_S(dat17, exp17); - NORMALIZE_S(dat18, exp18); - NORMALIZE_S(dat19, exp19); - NORMALIZE_S(dat20, exp20); - NORMALIZE_S(dat21, exp21); - NORMALIZE_S(dat22, exp22); - NORMALIZE_S(dat23, exp23); - NORMALIZE_S(dat24, exp24); - NORMALIZE_S(dat25, exp25); - NORMALIZE_S(dat26, exp26); - NORMALIZE_S(dat27, exp27); - NORMALIZE_S(dat28, exp28); - NORMALIZE_S(dat29, exp29); - NORMALIZE_S(dat30, exp30); - NORMALIZE_S(dat31, exp31); - NORMALIZE_S(dat32, exp32); - NORMALIZE_S(dat33, exp33); - NORMALIZE_S(dat34, exp34); - NORMALIZE_S(dat35, exp35); - NORMALIZE_S(dat36, exp36); - NORMALIZE_S(dat37, exp37); - NORMALIZE_S(dat38, exp38); - NORMALIZE_S(dat39, exp39); - NORMALIZE_S(dat40, exp40); - NORMALIZE_S(dat41, exp41); + test_normalization(dat1, exp1); + test_normalization(dat2, exp2); + test_normalization(dat3, exp3); + test_normalization(dat4, exp4); + test_normalization(dat5, exp5); + test_normalization(dat6, exp6); + test_normalization(dat7, exp7); + test_normalization(dat8, exp8); + test_normalization(dat9, exp9); + test_normalization(dat10, exp10); + test_normalization(dat11, exp11); + test_normalization(dat12, exp12); + test_normalization(dat13, exp13); + test_normalization(dat14, exp14); + test_normalization(dat15, exp15); + test_normalization(dat16, exp16); + test_normalization(dat17, exp17); + test_normalization(dat18, exp18); + test_normalization(dat19, exp19); + test_normalization(dat20, exp20); + test_normalization(dat21, exp21); + test_normalization(dat22, exp22); + test_normalization(dat23, exp23); + test_normalization(dat24, exp24); + test_normalization(dat25, exp25); + test_normalization(dat26, exp26); + test_normalization(dat27, exp27); + test_normalization(dat28, exp28); + test_normalization(dat29, exp29); + test_normalization(dat30, exp30); + test_normalization(dat31, exp31); + test_normalization(dat32, exp32); + test_normalization(dat33, exp33); + test_normalization(dat34, exp34); + test_normalization(dat35, exp35); + test_normalization(dat36, exp36); + test_normalization(dat37, exp37); + test_normalization(dat38, exp38); + test_normalization(dat39, exp39); + test_normalization(dat40, exp40); + test_normalization(dat41, exp41); } SECTION("over inner scopes") @@ -4096,14 +4400,14 @@ TEST_CASE("ignored identifier chain tracking", "[JSNormalizer]") const char exp7[] = "eval(var_0000+var_0001).baz"; const char exp8[] = "eval(var_0000 var_0001).baz"; - NORMALIZE_S(dat1, exp1); - NORMALIZE_S(dat2, exp2); - NORMALIZE_S(dat3, exp3); - NORMALIZE_S(dat4, exp4); - NORMALIZE_S(dat5, exp5); - NORMALIZE_S(dat6, exp6); - NORMALIZE_S(dat7, exp7); - NORMALIZE_S(dat8, exp8); + test_normalization(dat1, exp1); + test_normalization(dat2, exp2); + test_normalization(dat3, exp3); + test_normalization(dat4, exp4); + test_normalization(dat5, exp5); + test_normalization(dat6, exp6); + test_normalization(dat7, exp7); + test_normalization(dat8, exp8); } SECTION("spaces and operators") @@ -4127,15 +4431,15 @@ TEST_CASE("ignored identifier chain tracking", "[JSNormalizer]") const char exp8[] = "console.foo.bar:var_0000"; const char exp9[] = "console.foo"; - NORMALIZE_S(dat1, exp1); - NORMALIZE_S(dat2, exp2); - NORMALIZE_S(dat3, exp3); - NORMALIZE_S(dat4, exp4); - NORMALIZE_S(dat5, exp5); - NORMALIZE_S(dat6, exp6); - NORMALIZE_S(dat7, exp7); - NORMALIZE_S(dat8, exp8); - NORMALIZE_S(dat9, exp9); + test_normalization(dat1, exp1); + test_normalization(dat2, exp2); + test_normalization(dat3, exp3); + test_normalization(dat4, exp4); + test_normalization(dat5, exp5); + test_normalization(dat6, exp6); + test_normalization(dat7, exp7); + test_normalization(dat8, exp8); + test_normalization(dat9, exp9); } SECTION("comments") @@ -4145,9 +4449,9 @@ TEST_CASE("ignored identifier chain tracking", "[JSNormalizer]") const char dat3[] = "console./*multi-line comment*/log(abc).foo"; const char exp[] = "console.log(var_0000).foo"; - NORMALIZE_S(dat1, exp); - NORMALIZE_S(dat2, exp); - NORMALIZE_S(dat3, exp); + test_normalization(dat1, exp); + test_normalization(dat2, exp); + test_normalization(dat3, exp); } } @@ -4172,13 +4476,13 @@ TEST_CASE("ignored identifier scope tracking", "[JSNormalizer]") const char exp6[] = "(var_0000 var_0001 console var_0002 eval var_0003)"; const char exp7[] = "var_0000 var_0001(console)var_0002(eval)var_0003"; - NORMALIZE_S(dat1, exp1); - NORMALIZE_S(dat2, exp2); - NORMALIZE_S(dat3, exp3); - NORMALIZE_S(dat4, exp4); - NORMALIZE_S(dat5, exp5); - NORMALIZE_S(dat6, exp6); - NORMALIZE_S(dat7, exp7); + test_normalization(dat1, exp1); + test_normalization(dat2, exp2); + test_normalization(dat3, exp3); + test_normalization(dat4, exp4); + test_normalization(dat5, exp5); + test_normalization(dat6, exp6); + test_normalization(dat7, exp7); } SECTION("function calls") @@ -4226,27 +4530,27 @@ TEST_CASE("ignored identifier scope tracking", "[JSNormalizer]") const char exp20[] = "eval['foo']()['bar'].baz"; const char exp21[] = "eval['foo']['bar'].baz()"; - NORMALIZE_S(dat1, exp1); - NORMALIZE_S(dat2, exp2); - NORMALIZE_S(dat3, exp3); - NORMALIZE_S(dat4, exp4); - NORMALIZE_S(dat5, exp5); - NORMALIZE_S(dat6, exp6); - NORMALIZE_S(dat7, exp7); - NORMALIZE_S(dat8, exp8); - NORMALIZE_S(dat9, exp9); - NORMALIZE_S(dat10, exp10); - NORMALIZE_S(dat11, exp11); - NORMALIZE_S(dat12, exp12); - NORMALIZE_S(dat13, exp13); - NORMALIZE_S(dat14, exp14); - NORMALIZE_S(dat15, exp15); - NORMALIZE_S(dat16, exp16); - NORMALIZE_S(dat17, exp17); - NORMALIZE_S(dat18, exp18); - NORMALIZE_S(dat19, exp19); - NORMALIZE_S(dat20, exp20); - NORMALIZE_S(dat21, exp21); + test_normalization(dat1, exp1); + test_normalization(dat2, exp2); + test_normalization(dat3, exp3); + test_normalization(dat4, exp4); + test_normalization(dat5, exp5); + test_normalization(dat6, exp6); + test_normalization(dat7, exp7); + test_normalization(dat8, exp8); + test_normalization(dat9, exp9); + test_normalization(dat10, exp10); + test_normalization(dat11, exp11); + test_normalization(dat12, exp12); + test_normalization(dat13, exp13); + test_normalization(dat14, exp14); + test_normalization(dat15, exp15); + test_normalization(dat16, exp16); + test_normalization(dat17, exp17); + test_normalization(dat18, exp18); + test_normalization(dat19, exp19); + test_normalization(dat20, exp20); + test_normalization(dat21, exp21); } SECTION("eval cases") @@ -4272,16 +4576,16 @@ TEST_CASE("ignored identifier scope tracking", "[JSNormalizer]") const char exp9[] = "eval(var_0000).baz;"; const char exp10[] = "var var_0000.var_0001=123;(eval).baz;"; - NORMALIZE_S(dat1, exp1); - NORMALIZE_S(dat2, exp2); - NORMALIZE_S(dat3, exp3); - NORMALIZE_S(dat4, exp4); - NORMALIZE_S(dat5, exp5); - NORMALIZE_S(dat6, exp6); - NORMALIZE_S(dat7, exp7); - NORMALIZE_S(dat8, exp8); - NORMALIZE_S(dat9, exp9); - NORMALIZE_S(dat10, exp10); + test_normalization(dat1, exp1); + test_normalization(dat2, exp2); + test_normalization(dat3, exp3); + test_normalization(dat4, exp4); + test_normalization(dat5, exp5); + test_normalization(dat6, exp6); + test_normalization(dat7, exp7); + test_normalization(dat8, exp8); + test_normalization(dat9, exp9); + test_normalization(dat10, exp10); } SECTION("console cases") @@ -4301,13 +4605,13 @@ TEST_CASE("ignored identifier scope tracking", "[JSNormalizer]") const char exp6[] = "console.log(var_0000,var_0001,var_0002,document.foo(var_0000))"; const char exp7[] = "console.log(document.getElementById(\"mem\").text).clear(var_0000)"; - NORMALIZE_S(dat1, exp1); - NORMALIZE_S(dat2, exp2); - NORMALIZE_S(dat3, exp3); - NORMALIZE_S(dat4, exp4); - NORMALIZE_S(dat5, exp5); - NORMALIZE_S(dat6, exp6); - NORMALIZE_S(dat7, exp7); + test_normalization(dat1, exp1); + test_normalization(dat2, exp2); + test_normalization(dat3, exp3); + test_normalization(dat4, exp4); + test_normalization(dat5, exp5); + test_normalization(dat6, exp6); + test_normalization(dat7, exp7); } SECTION("corner cases") @@ -4323,11 +4627,11 @@ TEST_CASE("ignored identifier scope tracking", "[JSNormalizer]") const char exp4[] = "var_0000=[var_0001,eval,document.getElementById]"; const char exp5[] = "var_0000[1]=eval"; - NORMALIZE_S(dat1, exp1); - NORMALIZE_S(dat2, exp2); - NORMALIZE_S(dat3, exp3); - NORMALIZE_S(dat4, exp4); - NORMALIZE_S(dat5, exp5); + test_normalization(dat1, exp1); + test_normalization(dat2, exp2); + test_normalization(dat3, exp3); + test_normalization(dat4, exp4); + test_normalization(dat5, exp5); } } @@ -4379,27 +4683,27 @@ TEST_CASE("ignored properties", "[JSNormalizer]") const char exp19[] = "var_0000['console']['watch'];"; const char exp20[] = "var_0000[\"console\"][\"unwatch\"]();"; - NORMALIZE_S(dat1, exp1); - NORMALIZE_S(dat2, exp2); - NORMALIZE_S(dat3, exp3); - NORMALIZE_S(dat4, exp4); - NORMALIZE_S(dat5, exp5); - NORMALIZE_S(dat6, exp6); - NORMALIZE_S(dat7, exp7); - NORMALIZE_S(dat8, exp8); - NORMALIZE_S(dat9, exp9); - NORMALIZE_S(dat10, exp10); - - NORMALIZE_S(dat11, exp11); - NORMALIZE_S(dat12, exp12); - NORMALIZE_S(dat13, exp13); - NORMALIZE_S(dat14, exp14); - NORMALIZE_S(dat15, exp15); - NORMALIZE_S(dat16, exp16); - NORMALIZE_S(dat17, exp17); - NORMALIZE_S(dat18, exp18); - NORMALIZE_S(dat19, exp19); - NORMALIZE_S(dat20, exp20); + test_normalization(dat1, exp1); + test_normalization(dat2, exp2); + test_normalization(dat3, exp3); + test_normalization(dat4, exp4); + test_normalization(dat5, exp5); + test_normalization(dat6, exp6); + test_normalization(dat7, exp7); + test_normalization(dat8, exp8); + test_normalization(dat9, exp9); + test_normalization(dat10, exp10); + + test_normalization(dat11, exp11); + test_normalization(dat12, exp12); + test_normalization(dat13, exp13); + test_normalization(dat14, exp14); + test_normalization(dat15, exp15); + test_normalization(dat16, exp16); + test_normalization(dat17, exp17); + test_normalization(dat18, exp18); + test_normalization(dat19, exp19); + test_normalization(dat20, exp20); } SECTION("chain tracking") @@ -4456,31 +4760,31 @@ TEST_CASE("ignored properties", "[JSNormalizer]") const char exp19[] = "var_0000.+var_0001().var_0002;"; const char exp20[] = "var_0000.+['unwatch']().var_0001;"; - NORMALIZE_S(dat1, exp1); - NORMALIZE_S(dat2, exp2); - NORMALIZE_S(dat3, exp3); - NORMALIZE_S(dat4, exp4); - NORMALIZE_S(dat5, exp5); + test_normalization(dat1, exp1); + test_normalization(dat2, exp2); + test_normalization(dat3, exp3); + test_normalization(dat4, exp4); + test_normalization(dat5, exp5); - NORMALIZE_S(dat6, exp6); - NORMALIZE_S(dat7, exp7); - NORMALIZE_S(dat8, exp8); - NORMALIZE_S(dat9, exp9); - NORMALIZE_S(dat10, exp10); + test_normalization(dat6, exp6); + test_normalization(dat7, exp7); + test_normalization(dat8, exp8); + test_normalization(dat9, exp9); + test_normalization(dat10, exp10); - NORMALIZE_S(dat11, exp11); - NORMALIZE_S(dat12, exp12); - NORMALIZE_S(dat13, exp13); + test_normalization(dat11, exp11); + test_normalization(dat12, exp12); + test_normalization(dat13, exp13); - NORMALIZE_S(dat14, exp14); - NORMALIZE_S(dat15, exp15); - NORMALIZE_S(dat16, exp16); + test_normalization(dat14, exp14); + test_normalization(dat15, exp15); + test_normalization(dat16, exp16); - NORMALIZE_S(dat17, exp17); - NORMALIZE_S(dat18, exp18); + test_normalization(dat17, exp17); + test_normalization(dat18, exp18); - NORMALIZE_S(dat19, exp19); - NORMALIZE_S(dat20, exp20); + test_normalization(dat19, exp19); + test_normalization(dat20, exp20); } SECTION("scope tracking") @@ -4497,11 +4801,11 @@ TEST_CASE("ignored properties", "[JSNormalizer]") const char exp3[] = "var_0000.var_0001(var_0002.unwatch.eval).var_0003;"; const char exp4[] = "var_0000.var_0001(var_0002['unwatch'].var_0003).var_0003;"; - NORMALIZE_S(dat1, exp1); - NORMALIZE_S(dat2, exp2); + test_normalization(dat1, exp1); + test_normalization(dat2, exp2); - NORMALIZE_S(dat3, exp3); - NORMALIZE_S(dat4, exp4); + test_normalization(dat3, exp3); + test_normalization(dat4, exp4); } SECTION("corner cases") @@ -4565,40 +4869,39 @@ TEST_CASE("ignored properties", "[JSNormalizer]") const char exp21[] = "var_0000.var_0001;u.a;var_0000.w;var_0002.var_0000;"; const char exp22[] = "u.a;var_0000.var_0001;var_0002.var_0000;var_0000.w;"; - NORMALIZE_S(dat1, exp1); - NORMALIZE_S(dat2, exp2); + test_normalization(dat1, exp1); + test_normalization(dat2, exp2); - NORMALIZE_S(dat3, exp3); - NORMALIZE_S(dat4, exp4); + test_normalization(dat3, exp3); + test_normalization(dat4, exp4); - NORMALIZE_S(dat5, exp5); - NORMALIZE_S(dat6, exp6); + test_normalization(dat5, exp5); + test_normalization(dat6, exp6); - NORMALIZE_S(dat7, exp7); - NORMALIZE_S(dat8, exp8); + test_normalization(dat7, exp7); + test_normalization(dat8, exp8); - NORMALIZE_S(dat9, exp9); - NORMALIZE_S(dat10, exp10); - NORMALIZE_S(dat11, exp11); - NORMALIZE_S(dat12, exp12); + test_normalization(dat9, exp9); + test_normalization(dat10, exp10); + test_normalization(dat11, exp11); + test_normalization(dat12, exp12); - NORMALIZE_S(dat13, exp13); - NORMALIZE_S(dat14, exp14); - NORMALIZE_S(dat15, exp15); - NORMALIZE_S(dat16, exp16); - NORMALIZE_S(dat17, exp17); + test_normalization(dat13, exp13); + test_normalization(dat14, exp14); + test_normalization(dat15, exp15); + test_normalization(dat16, exp16); + test_normalization(dat17, exp17); - NORMALIZE_S(dat18, exp18); - NORMALIZE_S(dat19, exp19); - NORMALIZE_S(dat20, exp20); - NORMALIZE_S(dat21, exp21); - NORMALIZE_S(dat22, exp22); + test_normalization(dat18, exp18); + test_normalization(dat19, exp19); + test_normalization(dat20, exp20); + test_normalization(dat21, exp21); + test_normalization(dat22, exp22); } } TEST_CASE("ignored identifier split", "[JSNormalizer]") { - #if JSTOKENIZER_MAX_STATES != 8 #error "ignored identifier split" tests are designed for 8 states depth #endif @@ -4627,15 +4930,37 @@ TEST_CASE("ignored identifier split", "[JSNormalizer]") const char exp11[] = "console.foo"; const char exp12[] = "eval.bar"; - NORMALIZE_T(dat1, dat2, exp1, exp2); - NORMALIZE_T(dat3, dat4, exp3, exp4); - NORMALIZE_T(dat5, dat6, exp5, exp6); - NORMALIZE_T(dat7, dat8, exp7, exp8); - - NORM_COMBINED_S_2(dat1, dat2, exp9); - NORM_COMBINED_S_2(dat3, dat4, exp10); - NORM_COMBINED_S_2(dat5, dat6, exp11); - NORM_COMBINED_S_2(dat7, dat8, exp12); + test_normalization( + { + {dat1, exp1}, + {dat2, exp2}, + }, + {use_expected_for_last_pdu(true)}); + test_normalization_combined({dat1, dat2}, exp9); + + test_normalization( + { + {dat3, exp3}, + {dat4, exp4}, + }, + {use_expected_for_last_pdu(true)}); + test_normalization_combined({dat3, dat4}, exp10); + + test_normalization( + { + {dat5, exp5}, + {dat6, exp6}, + }, + {use_expected_for_last_pdu(true)}); + test_normalization_combined({dat5, dat6}, exp11); + + test_normalization( + { + {dat7, exp7}, + {dat8, exp8}, + }, + {use_expected_for_last_pdu(true)}); + test_normalization_combined({dat7, dat8}, exp12); } SECTION("function calls") @@ -4667,17 +4992,45 @@ TEST_CASE("ignored identifier split", "[JSNormalizer]") const char exp14[] = "console().foo"; const char exp15[] = "console().rewrite"; - NORMALIZE_T(dat1, dat2, exp1, exp2); - NORMALIZE_T(dat3, dat4, exp3, exp4); - NORMALIZE_T(dat5, dat6, exp5, exp6); - NORMALIZE_T(dat7, dat8, exp7, exp8); - NORMALIZE_T(dat9, dat10, exp9, exp10); - - NORM_COMBINED_S_2(dat1, dat2, exp11); - NORM_COMBINED_S_2(dat3, dat4, exp12); - NORM_COMBINED_S_2(dat5, dat6, exp13); - NORM_COMBINED_S_2(dat7, dat8, exp14); - NORM_COMBINED_S_2(dat9, dat10, exp15); + test_normalization( + { + {dat1, exp1}, + {dat2, exp2}, + }, + {use_expected_for_last_pdu(true)}); + test_normalization_combined({dat1, dat2}, exp11); + + test_normalization( + { + {dat3, exp3}, + {dat4, exp4}, + }, + {use_expected_for_last_pdu(true)}); + test_normalization_combined({dat3, dat4}, exp12); + + test_normalization( + { + {dat5, exp5}, + {dat6, exp6}, + }, + {use_expected_for_last_pdu(true)}); + test_normalization_combined({dat5, dat6}, exp13); + + test_normalization( + { + {dat7, exp7}, + {dat8, exp8}, + }, + {use_expected_for_last_pdu(true)}); + test_normalization_combined({dat7, dat8}, exp14); + + test_normalization( + { + {dat9, exp9}, + {dat10, exp10}, + }, + {use_expected_for_last_pdu(true)}); + test_normalization_combined({dat9, dat10}, exp15); } SECTION("terminator split") @@ -4709,17 +5062,45 @@ TEST_CASE("ignored identifier split", "[JSNormalizer]") const char exp14[] = "eval.foo;var_0000.var_0001"; const char exp15[] = "eval.foo%=var_0000.var_0001"; - NORMALIZE_T(dat1, dat2, exp1, exp2); - NORMALIZE_T(dat3, dat4, exp3, exp4); - NORMALIZE_T(dat5, dat6, exp5, exp6); - NORMALIZE_T(dat7, dat8, exp7, exp8); - NORMALIZE_T(dat9, dat10, exp9, exp10); - - NORM_COMBINED_S_2(dat1, dat2, exp11); - NORM_COMBINED_S_2(dat3, dat4, exp12); - NORM_COMBINED_S_2(dat5, dat6, exp13); - NORM_COMBINED_S_2(dat7, dat8, exp14); - NORM_COMBINED_S_2(dat9, dat10, exp15); + test_normalization( + { + {dat1, exp1}, + {dat2, exp2}, + }, + {use_expected_for_last_pdu(true)}); + test_normalization_combined({dat1, dat2}, exp11); + + test_normalization( + { + {dat3, exp3}, + {dat4, exp4}, + }, + {use_expected_for_last_pdu(true)}); + test_normalization_combined({dat3, dat4}, exp12); + + test_normalization( + { + {dat5, exp5}, + {dat6, exp6}, + }, + {use_expected_for_last_pdu(true)}); + test_normalization_combined({dat5, dat6}, exp13); + + test_normalization( + { + {dat7, exp7}, + {dat8, exp8}, + }, + {use_expected_for_last_pdu(true)}); + test_normalization_combined({dat7, dat8}, exp14); + + test_normalization( + { + {dat9, exp9}, + {dat10, exp10}, + }, + {use_expected_for_last_pdu(true)}); + test_normalization_combined({dat9, dat10}, exp15); } SECTION("scope split") @@ -4735,8 +5116,13 @@ TEST_CASE("ignored identifier split", "[JSNormalizer]") const char exp3[] = "eval(var_0000,eval(var_0001,eval(var_0002['" "'].var_0003.var_0004+var_0004).foo).bar).baz+var_0004"; - NORMALIZE_T(dat1, dat2, exp1, exp2); - NORM_COMBINED_S_2(dat1, dat2, exp3); + test_normalization( + { + {dat1, exp1}, + {dat2, exp2}, + }, + {use_expected_for_last_pdu(true)}); + test_normalization_combined({dat1, dat2}, exp3); // "eval(foo,eval(bar,eval(baz[''].console.check+check).foo).bar).baz+check" // split here ^ @@ -4749,8 +5135,13 @@ TEST_CASE("ignored identifier split", "[JSNormalizer]") const char exp6[] = "eval(var_0000,eval(var_0001,eval(var_0002['']." "var_0004.var_0005+var_0005).foo).bar).baz+var_0005"; - NORMALIZE_T(dat3, dat4, exp4, exp5); - NORM_COMBINED_S_2(dat3, dat4, exp6); + test_normalization( + { + {dat3, exp4}, + {dat4, exp5}, + }, + {use_expected_for_last_pdu(true)}); + test_normalization_combined({dat3, dat4}, exp6); // "eval(foo,eval(bar,eval(baz[''].console.check+check).foo).bar).baz+check" // split here ^ @@ -4763,8 +5154,13 @@ TEST_CASE("ignored identifier split", "[JSNormalizer]") const char exp9[] = "eval(var_0000,eval(var_0001,eval(var_0002[''].var_0003.var_0004+var_0004).foo" ").bar).baz+var_0004"; - NORMALIZE_T(dat5, dat6, exp7, exp8); - NORM_COMBINED_S_2(dat5, dat6, exp9); + test_normalization( + { + {dat5, exp7}, + {dat6, exp8}, + }, + {use_expected_for_last_pdu(true)}); + test_normalization_combined({dat5, dat6}, exp9); } SECTION("normalized word in the previous PDU") @@ -4774,7 +5170,13 @@ TEST_CASE("ignored identifier split", "[JSNormalizer]") const char exp1[] = "!var_0000"; const char exp2[] = "()"; - NORMALIZE_T(dat1, dat2, exp1, exp2); + test_normalization( + { + {dat1, exp1}, + {dat2, exp2}, + }, + {use_expected_for_last_pdu(true)} + ); } SECTION("ignored word in the previous PDU") @@ -4784,13 +5186,18 @@ TEST_CASE("ignored identifier split", "[JSNormalizer]") const char exp1[] = "!eval"; const char exp2[] = "()"; - NORMALIZE_T(dat1, dat2, exp1, exp2); + test_normalization( + { + {dat1, exp1}, + {dat2, exp2}, + }, + {use_expected_for_last_pdu(true)} + ); } } TEST_CASE("ignored properties split", "[JSNormalizer]") { - #if JSTOKENIZER_MAX_STATES != 8 #error "ignored properties split" tests are designed for 8 states depth #endif @@ -4827,20 +5234,45 @@ TEST_CASE("ignored properties split", "[JSNormalizer]") const char exp10[] = ".watch;"; const char exp_comb_5[] = "var_0000.watch;"; - NORMALIZE_T(dat1, dat2, exp1, exp2); - NORM_COMBINED_S_2(dat1, dat2, exp_comb_1); - - NORMALIZE_T(dat3, dat4, exp3, exp4); - NORM_COMBINED_S_2(dat3, dat4, exp_comb_2); - - NORMALIZE_T(dat5, dat6, exp5, exp6); - NORM_COMBINED_S_2(dat5, dat6, exp_comb_3); - - NORMALIZE_T(dat7, dat8, exp7, exp8); - NORM_COMBINED_S_2(dat7, dat8, exp_comb_4); - - NORMALIZE_T(dat9, dat10, exp9, exp10); - NORM_COMBINED_S_2(dat9, dat10, exp_comb_5); + test_normalization( + { + {dat1, exp1}, + {dat2, exp2}, + }, + {use_expected_for_last_pdu(true)}); + test_normalization_combined({dat1, dat2}, exp_comb_1); + + test_normalization( + { + {dat3, exp3}, + {dat4, exp4}, + }, + {use_expected_for_last_pdu(true)}); + test_normalization_combined({dat3, dat4}, exp_comb_2); + + test_normalization( + { + {dat5, exp5}, + {dat6, exp6}, + }, + {use_expected_for_last_pdu(true)}); + test_normalization_combined({dat5, dat6}, exp_comb_3); + + test_normalization( + { + {dat7, exp7}, + {dat8, exp8}, + }, + {use_expected_for_last_pdu(true)}); + test_normalization_combined({dat7, dat8}, exp_comb_4); + + test_normalization( + { + {dat9, exp9}, + {dat10, exp10}, + }, + {use_expected_for_last_pdu(true)}); + test_normalization_combined({dat9, dat10}, exp_comb_5); } SECTION("chain tracking") @@ -4905,35 +5337,85 @@ TEST_CASE("ignored properties split", "[JSNormalizer]") const char exp20[] = "var_0001;"; const char exp_comb_10[] = "var_0000.unwatch()+var_0001;"; - NORMALIZE_T(dat1, dat2, exp1, exp2); - NORM_COMBINED_S_2(dat1, dat2, exp_comb_1); - - NORMALIZE_T(dat3, dat4, exp3, exp4); - NORM_COMBINED_S_2(dat3, dat4, exp_comb_2); - - NORMALIZE_T(dat5, dat6, exp5, exp6); - NORM_COMBINED_S_2(dat5, dat6, exp_comb_3); - - NORMALIZE_T(dat7, dat8, exp7, exp8); - NORM_COMBINED_S_2(dat7, dat8, exp_comb_4); - - NORMALIZE_T(dat9, dat10, exp9, exp10); - NORM_COMBINED_S_2(dat9, dat10, exp_comb_5); - - NORMALIZE_T(dat11, dat12, exp11, exp12); - NORM_COMBINED_S_2(dat11, dat12, exp_comb_6); - - NORMALIZE_T(dat13, dat14, exp13, exp14); - NORM_COMBINED_S_2(dat13, dat14, exp_comb_7); - - NORMALIZE_T(dat15, dat16, exp15, exp16); - NORM_COMBINED_S_2(dat15, dat16, exp_comb_8); - - NORMALIZE_T(dat17, dat18, exp17, exp18); - NORM_COMBINED_S_2(dat17, dat18, exp_comb_9); - - NORMALIZE_T(dat19, dat20, exp19, exp20); - NORM_COMBINED_S_2(dat19, dat20, exp_comb_10); + test_normalization( + { + {dat1, exp1}, + {dat2, exp2}, + }, + {use_expected_for_last_pdu(true)}); + test_normalization_combined({dat1, dat2}, exp_comb_1); + + test_normalization( + { + {dat3, exp3}, + {dat4, exp4}, + }, + {use_expected_for_last_pdu(true)}); + test_normalization_combined({dat3, dat4}, exp_comb_2); + + test_normalization( + { + {dat5, exp5}, + {dat6, exp6}, + }, + {use_expected_for_last_pdu(true)}); + test_normalization_combined({dat5, dat6}, exp_comb_3); + + test_normalization( + { + {dat7, exp7}, + {dat8, exp8}, + }, + {use_expected_for_last_pdu(true)}); + test_normalization_combined({dat7, dat8}, exp_comb_4); + + test_normalization( + { + {dat9, exp9}, + {dat10, exp10}, + }, + {use_expected_for_last_pdu(true)}); + test_normalization_combined({dat9, dat10}, exp_comb_5); + + test_normalization( + { + {dat11, exp11}, + {dat12, exp12}, + }, + {use_expected_for_last_pdu(true)}); + test_normalization_combined({dat11, dat12}, exp_comb_6); + + test_normalization( + { + {dat13, exp13}, + {dat14, exp14}, + }, + {use_expected_for_last_pdu(true)}); + test_normalization_combined({dat13, dat14}, exp_comb_7); + + test_normalization( + { + {dat15, exp15}, + {dat16, exp16}, + }, + {use_expected_for_last_pdu(true)}); + test_normalization_combined({dat15, dat16}, exp_comb_8); + + test_normalization( + { + {dat17, exp17}, + {dat18, exp18}, + }, + {use_expected_for_last_pdu(true)}); + test_normalization_combined({dat17, dat18}, exp_comb_9); + + test_normalization( + { + {dat19, exp19}, + {dat20, exp20}, + }, + {use_expected_for_last_pdu(true)}); + test_normalization_combined({dat19, dat20}, exp_comb_10); } SECTION("scope tracking") @@ -4962,301 +5444,321 @@ TEST_CASE("ignored properties split", "[JSNormalizer]") const char exp8[] = "unwatch']().var_0001).var_0000;"; const char exp_comb_4[] = "var_0000.var_0001(var_0002['unwatch']().var_0001).var_0000;"; - NORMALIZE_T(dat1, dat2, exp1, exp2); - NORM_COMBINED_S_2(dat1, dat2, exp_comb_1); - - NORMALIZE_T(dat3, dat4, exp3, exp4); - NORM_COMBINED_S_2(dat3, dat4, exp_comb_2); - - NORMALIZE_T(dat5, dat6, exp5, exp6); - NORM_COMBINED_S_2(dat5, dat6, exp_comb_3); - - NORMALIZE_T(dat7, dat8, exp7, exp8); - NORM_COMBINED_S_2(dat7, dat8, exp_comb_4); + test_normalization( + { + {dat1, exp1}, + {dat2, exp2}, + }, + {use_expected_for_last_pdu(true)}); + test_normalization_combined({dat1, dat2}, exp_comb_1); + + test_normalization( + { + {dat3, exp3}, + {dat4, exp4}, + }, + {use_expected_for_last_pdu(true)}); + test_normalization_combined({dat3, dat4}, exp_comb_2); + + test_normalization( + { + {dat5, exp5}, + {dat6, exp6}, + }, + {use_expected_for_last_pdu(true)}); + test_normalization_combined({dat5, dat6}, exp_comb_3); + + test_normalization( + { + {dat7, exp7}, + {dat8, exp8}, + }, + {use_expected_for_last_pdu(true)}); + test_normalization_combined({dat7, dat8}, exp_comb_4); } } -TEST_CASE("Scope tracking - basic","[JSNormalizer]") +TEST_CASE("Scope tracking - basic", "[JSNormalizer]") { SECTION("Global only") - test_scope("",{GLOBAL}); + test_scope("", {GLOBAL}); SECTION("Function scope - named function") - test_scope("function f(){",{GLOBAL,FUNCTION}); + test_scope("function f(){", {GLOBAL, FUNCTION}); SECTION("Function scope - anonymous function") - test_scope("var f = function(){",{GLOBAL,FUNCTION}); + test_scope("var f = function(){", {GLOBAL, FUNCTION}); SECTION("Function scope - arrow function") - test_scope("var f = (a,b)=>{",{GLOBAL,FUNCTION}); + test_scope("var f = (a,b)=>{", {GLOBAL, FUNCTION}); SECTION("Function scope - arrow function without scope") - test_scope("var f = (a,b)=> a",{GLOBAL,FUNCTION}); + test_scope("var f = (a,b)=> a", {GLOBAL, FUNCTION}); SECTION("Function scope - function call in an arrow function without scope") - test_scope("var f = (a,b)=> call(",{GLOBAL,FUNCTION}); + test_scope("var f = (a,b)=> call(", {GLOBAL, FUNCTION}); SECTION("Function scope - method in object initialization") - test_scope("var o = { f(){",{GLOBAL,BLOCK,BLOCK}); + test_scope("var o = { f(){", {GLOBAL, BLOCK, BLOCK}); SECTION("Function scope - method in object operation") - test_scope("+{ f(){",{GLOBAL,BLOCK,BLOCK}); + test_scope("+{ f(){", {GLOBAL, BLOCK, BLOCK}); SECTION("Function scope - method in object as a function parameter") - test_scope("call({ f(){",{GLOBAL,BLOCK,BLOCK}); + test_scope("call({ f(){", {GLOBAL, BLOCK, BLOCK}); SECTION("Function scope - keyword name method") - test_scope("var o = { let(){",{GLOBAL,BLOCK,BLOCK}); + test_scope("var o = { let(){", {GLOBAL, BLOCK, BLOCK}); SECTION("Function scope - 'get' name method") - test_scope("var o = { get(){",{GLOBAL,BLOCK,BLOCK}); + test_scope("var o = { get(){", {GLOBAL, BLOCK, BLOCK}); SECTION("Function scope - expression method") - test_scope("var o = { [a + 12](){",{GLOBAL,BLOCK,BLOCK}); + test_scope("var o = { [a + 12](){", {GLOBAL, BLOCK, BLOCK}); SECTION("Function scope - method as anonymous function") - test_scope("var o = { f: function(){",{GLOBAL,BLOCK,FUNCTION}); + test_scope("var o = { f: function(){", {GLOBAL, BLOCK, FUNCTION}); SECTION("Function scope - keyword name method as anonymous function") - test_scope("var o = { let: function(){",{GLOBAL,BLOCK,FUNCTION}); + test_scope("var o = { let: function(){", {GLOBAL, BLOCK, FUNCTION}); SECTION("Function scope - 'get' name method as anonymous function") - test_scope("var o = { get: function(){",{GLOBAL,BLOCK,FUNCTION}); + test_scope("var o = { get: function(){", {GLOBAL, BLOCK, FUNCTION}); SECTION("Function scope - expression method as anonymous function") - test_scope("var o = { [a + 12]: function(){",{GLOBAL,BLOCK,FUNCTION}); + test_scope("var o = { [a + 12]: function(){", {GLOBAL, BLOCK, FUNCTION}); SECTION("Function scope - getter") - test_scope("var o = { get f(){",{GLOBAL,BLOCK,BLOCK}); + test_scope("var o = { get f(){", {GLOBAL, BLOCK, BLOCK}); SECTION("Function scope - parametric getter") - test_scope("var o = { get [a + 12](){",{GLOBAL,BLOCK,BLOCK}); + test_scope("var o = { get [a + 12](){", {GLOBAL, BLOCK, BLOCK}); SECTION("Function scope - setter") - test_scope("var o = { set f(){",{GLOBAL,BLOCK,BLOCK}); + test_scope("var o = { set f(){", {GLOBAL, BLOCK, BLOCK}); SECTION("Function scope - parametric setter") - test_scope("var o = { set [a + 12](){",{GLOBAL,BLOCK,BLOCK}); + test_scope("var o = { set [a + 12](){", {GLOBAL, BLOCK, BLOCK}); SECTION("Block scope - regular block") - test_scope("{",{GLOBAL,BLOCK}); + test_scope("{", {GLOBAL, BLOCK}); SECTION("Block scope - object initializer") - test_scope("o = {",{GLOBAL,BLOCK}); + test_scope("o = {", {GLOBAL, BLOCK}); SECTION("Block scope - class") - test_scope("class C{",{GLOBAL,BLOCK}); + test_scope("class C{", {GLOBAL, BLOCK}); SECTION("Block scope - class with extends") - test_scope("class C extends A{",{GLOBAL,BLOCK}); + test_scope("class C extends A{", {GLOBAL, BLOCK}); SECTION("Block scope - if") - test_scope("if(true){",{GLOBAL,BLOCK}); + test_scope("if(true){", {GLOBAL, BLOCK}); SECTION("Block scope - single statement if") - test_scope("if(true) func()",{GLOBAL,BLOCK}); + test_scope("if(true) func()", {GLOBAL, BLOCK}); SECTION("Block scope - nested multiple single statement ifs") - test_scope("if(a) if(b) if(c) if(d) func()",{GLOBAL,BLOCK}); + test_scope("if(a) if(b) if(c) if(d) func()", {GLOBAL, BLOCK}); SECTION("Block scope - nested multiple single statement ifs with newline") - test_scope("if(a)\nif(b)\nif(c)\nif(d)\nfunc()",{GLOBAL,BLOCK}); + test_scope("if(a)\nif(b)\nif(c)\nif(d)\nfunc()", {GLOBAL, BLOCK}); SECTION("Block scope - else") - test_scope("if(true);else{",{GLOBAL,BLOCK}); + test_scope("if(true);else{", {GLOBAL, BLOCK}); SECTION("Block scope - single statement else") - test_scope("if(true);else func()",{GLOBAL,BLOCK}); + test_scope("if(true);else func()", {GLOBAL, BLOCK}); SECTION("Block scope - for loop") - test_scope("for(;;){",{GLOBAL,BLOCK}); + test_scope("for(;;){", {GLOBAL, BLOCK}); SECTION("Block scope - for loop in range") - test_scope("for(i in range()){",{GLOBAL,BLOCK}); + test_scope("for(i in range()){", {GLOBAL, BLOCK}); SECTION("Block scope - for loop of iterable") - test_scope("for(i of o){",{GLOBAL,BLOCK}); + test_scope("for(i of o){", {GLOBAL, BLOCK}); SECTION("Block scope - for await loop") - test_scope("for await(i of o){",{GLOBAL,BLOCK}); + test_scope("for await(i of o){", {GLOBAL, BLOCK}); SECTION("Block scope - inside for statement") - test_scope("for(",{GLOBAL,BLOCK}); + test_scope("for(", {GLOBAL, BLOCK}); SECTION("Block scope - inside for statement, after semicolon") - test_scope("for(;",{GLOBAL,BLOCK}); + test_scope("for(;", {GLOBAL, BLOCK}); SECTION("Block scope - single statement for") - test_scope("for(;;) func()",{GLOBAL,BLOCK}); + test_scope("for(;;) func()", {GLOBAL, BLOCK}); SECTION("Block scope - for nested in single line conditional") - test_scope("if(true) for(;;) a++",{GLOBAL,BLOCK}); + test_scope("if(true) for(;;) a++", {GLOBAL, BLOCK}); SECTION("Block scope - while") - test_scope("while(true){",{GLOBAL,BLOCK}); + test_scope("while(true){", {GLOBAL, BLOCK}); SECTION("Block scope - single statement while") - test_scope("while(true) func()",{GLOBAL,BLOCK}); + test_scope("while(true) func()", {GLOBAL, BLOCK}); SECTION("Block scope - do-while") - test_scope("do{",{GLOBAL,BLOCK}); + test_scope("do{", {GLOBAL, BLOCK}); SECTION("Block scope - single statement do-while") - test_scope("do func()",{GLOBAL,BLOCK}); + test_scope("do func()", {GLOBAL, BLOCK}); SECTION("Block scope - try") - test_scope("try{",{GLOBAL,BLOCK}); + test_scope("try{", {GLOBAL, BLOCK}); SECTION("Block scope - catch") - test_scope("try{}catch(e){",{GLOBAL,BLOCK}); + test_scope("try{}catch(e){", {GLOBAL, BLOCK}); SECTION("Block scope - catch exception declaration") - test_scope("try{}catch(",{GLOBAL,BLOCK}); + test_scope("try{}catch(", {GLOBAL, BLOCK}); SECTION("Block scope - finally") - test_scope("try{}finally{",{GLOBAL,BLOCK}); + test_scope("try{}finally{", {GLOBAL, BLOCK}); SECTION("Block scope - nested object - named") - test_scope("var o = {s:{",{GLOBAL,BLOCK,BLOCK}); + test_scope("var o = {s:{", {GLOBAL, BLOCK, BLOCK}); SECTION("Block scope - nested object - keyword named") - test_scope("var o = {let:{",{GLOBAL,BLOCK,BLOCK}); + test_scope("var o = {let:{", {GLOBAL, BLOCK, BLOCK}); SECTION("Block scope - nested object - 'get' named") - test_scope("var o = {get:{",{GLOBAL,BLOCK,BLOCK}); + test_scope("var o = {get:{", {GLOBAL, BLOCK, BLOCK}); SECTION("Block scope - nested object - parametric") - test_scope("var o = {[a+12]:{",{GLOBAL,BLOCK,BLOCK}); + test_scope("var o = {[a+12]:{", {GLOBAL, BLOCK, BLOCK}); } -TEST_CASE("Scope tracking - closing","[JSNormalizer]") +TEST_CASE("Scope tracking - closing", "[JSNormalizer]") { SECTION("Function scope - named function") - test_scope("function f(){}",{GLOBAL}); + test_scope("function f(){}", {GLOBAL}); SECTION("Function scope - anonymous function") - test_scope("var f = function(){}",{GLOBAL}); + test_scope("var f = function(){}", {GLOBAL}); SECTION("Function scope - arrow function") - test_scope("var f = (a,b)=>{}",{GLOBAL}); + test_scope("var f = (a, b)=>{}", {GLOBAL}); SECTION("Function scope - arrow function without scope") - test_scope("var f = (a,b)=>a;",{GLOBAL}); + test_scope("var f = (a, b)=>a;", {GLOBAL}); SECTION("Function scope - function call in an arrow function without scope") - test_scope("var f = a=>call();",{GLOBAL}); + test_scope("var f = a=>call();", {GLOBAL}); SECTION("Function scope - arrow function as a function parameter") - test_scope("console.log(a=>c)",{GLOBAL}); + test_scope("console.log(a=>c)", {GLOBAL}); SECTION("Function scope - method") - test_scope("var o = { f(){}",{GLOBAL,BLOCK}); + test_scope("var o = { f(){}", {GLOBAL, BLOCK}); SECTION("Function scope - keyword name method") - test_scope("var o = { let(){}",{GLOBAL,BLOCK}); + test_scope("var o = { let(){}", {GLOBAL, BLOCK}); SECTION("Function scope - expression method") - test_scope("var o = { [a + 12](){}",{GLOBAL,BLOCK}); + test_scope("var o = { [a + 12](){}", {GLOBAL, BLOCK}); SECTION("Function scope - method as anonymous function") - test_scope("var o = { f: function(){}",{GLOBAL,BLOCK}); + test_scope("var o = { f: function(){}", {GLOBAL, BLOCK}); SECTION("Function scope - keyword name method as anonymous function") - test_scope("var o = { let: function(){}",{GLOBAL,BLOCK}); + test_scope("var o = { let: function(){}", {GLOBAL, BLOCK}); SECTION("Function scope - expression method as anonymous function") - test_scope("var o = { [a + 12]: function(){}",{GLOBAL,BLOCK}); + test_scope("var o = { [a + 12]: function(){}", {GLOBAL, BLOCK}); SECTION("Function scope - getter") - test_scope("var o = { get f(){}",{GLOBAL,BLOCK}); + test_scope("var o = { get f(){}", {GLOBAL, BLOCK}); SECTION("Function scope - parametric getter") - test_scope("var o = { get [a + 12](){}",{GLOBAL,BLOCK}); + test_scope("var o = { get [a + 12](){}", {GLOBAL, BLOCK}); SECTION("Function scope - setter") - test_scope("var o = { set f(){}",{GLOBAL,BLOCK}); + test_scope("var o = { set f(){}", {GLOBAL, BLOCK}); SECTION("Function scope - parametric setter") - test_scope("var o = { set [a + 12](){}",{GLOBAL,BLOCK}); + test_scope("var o = { set [a + 12](){}", {GLOBAL, BLOCK}); SECTION("Block scope - regular block") - test_scope("{}",{GLOBAL}); + test_scope("{}", {GLOBAL}); SECTION("Block scope - object initializer") - test_scope("o = {}",{GLOBAL}); + test_scope("o = {}", {GLOBAL}); SECTION("Block scope - class") - test_scope("class C{}",{GLOBAL}); + test_scope("class C{}", {GLOBAL}); SECTION("Block scope - class with extends") - test_scope("class C extends A{}",{GLOBAL}); + test_scope("class C extends A{}", {GLOBAL}); SECTION("Block scope - if") - test_scope("if(true){}",{GLOBAL}); + test_scope("if(true){}", {GLOBAL}); SECTION("Block scope - single statement if") - test_scope("if(true);",{GLOBAL}); + test_scope("if(true);", {GLOBAL}); SECTION("Block scope - single statement if, semicolon group terminated") - test_scope("if(true)\na++\nreturn",{GLOBAL}); + test_scope("if(true)\na++\nreturn", {GLOBAL}); SECTION("Block scope - nested multiple single statement ifs") - test_scope("if(a) if(b) if(c) if(d) func();",{GLOBAL}); + test_scope("if(a) if(b) if(c) if(d) func();", {GLOBAL}); SECTION("Block scope - nested multiple single statement ifs with newline") - test_scope("if(a)\nif(b)\nif(c)\nif(d)\nfunc()\nfunc()",{GLOBAL}); + test_scope("if(a)\nif(b)\nif(c)\nif(d)\nfunc()\nfunc()", {GLOBAL}); SECTION("Block scope - else") - test_scope("if(true);else{}",{GLOBAL}); + test_scope("if(true);else{}", {GLOBAL}); SECTION("Block scope - single statement else") - test_scope("if(true);else;",{GLOBAL}); + test_scope("if(true);else;", {GLOBAL}); SECTION("Block scope - for loop") - test_scope("for(;;){}",{GLOBAL}); + test_scope("for(;;){}", {GLOBAL}); SECTION("Block scope - for loop in range") - test_scope("for(i in range()){}",{GLOBAL}); + test_scope("for(i in range()){}", {GLOBAL}); SECTION("Block scope - for loop of iterable") - test_scope("for(i of o){}",{GLOBAL}); + test_scope("for(i of o){}", {GLOBAL}); SECTION("Block scope - for await loop") - test_scope("for await(i of o){}",{GLOBAL}); + test_scope("for await(i of o){}", {GLOBAL}); SECTION("Block scope - single statement for") - test_scope("for(;;);",{GLOBAL}); + test_scope("for(;;);", {GLOBAL}); SECTION("Block scope - while") - test_scope("while(true){}",{GLOBAL}); + test_scope("while(true){}", {GLOBAL}); SECTION("Block scope - single statement while") - test_scope("while(true);",{GLOBAL}); + test_scope("while(true);", {GLOBAL}); SECTION("Block scope - do-while") - test_scope("do{}while(",{GLOBAL, BLOCK}); + test_scope("do{}while(", {GLOBAL, BLOCK}); SECTION("Block scope - single statement do-while") - test_scope("do;while(",{GLOBAL, BLOCK}); + test_scope("do;while(", {GLOBAL, BLOCK}); SECTION("Block scope - try") - test_scope("try{}",{GLOBAL}); + test_scope("try{}", {GLOBAL}); SECTION("Block scope - catch") - test_scope("try{}catch(e){}",{GLOBAL}); + test_scope("try{}catch(e){}", {GLOBAL}); SECTION("Block scope - finally") - test_scope("try{}finally{}",{GLOBAL}); + test_scope("try{}finally{}", {GLOBAL}); SECTION("Block scope - nested object - named") - test_scope("var o = {s:{}",{GLOBAL,BLOCK}); + test_scope("var o = {s:{}", {GLOBAL, BLOCK}); SECTION("Block scope - nested object - keyword named") - test_scope("var o = {let:{}",{GLOBAL,BLOCK}); + test_scope("var o = {let:{}", {GLOBAL, BLOCK}); SECTION("Block scope - nested object - parametric") - test_scope("var o = {[a+12]:{}",{GLOBAL,BLOCK}); + test_scope("var o = {[a+12]:{}", {GLOBAL, BLOCK}); SECTION("Block scope - advanced automatic semicolon insertion") test_scope( @@ -5274,7 +5776,7 @@ TEST_CASE("Scope tracking - closing","[JSNormalizer]") test_scope("function() { if (true)\nfor ( ; ; ) a = 2 }", {GLOBAL}); } -TEST_CASE("Scope tracking - over multiple PDU","[JSNormalizer]") +TEST_CASE("Scope tracking - over multiple PDU", "[JSNormalizer]") { // Every line represents a PDU. Each pdu has input buffer, expected script // and expected scope stack, written in that order @@ -5306,7 +5808,7 @@ TEST_CASE("Scope tracking - over multiple PDU","[JSNormalizer]") SECTION("general - variable extension that overwrites existing variable inside inner scope") test_normalization({ - {"f(a, x=>{var an", "var_0000(var_0001,var_0002=>{var var_0003", {GLOBAL,FUNCTION}}, + {"f(a, x=>{var an", "var_0000(var_0001,var_0002=>{var var_0003", {GLOBAL, FUNCTION}}, {"other = a})", "var_0000(var_0001,var_0002=>{var var_0004=var_0001})", {GLOBAL}} }); @@ -5367,7 +5869,7 @@ TEST_CASE("Scope tracking - over multiple PDU","[JSNormalizer]") SECTION("function scope: split in 'function'") test_normalization({ {"func", "var_0000", {GLOBAL}}, - {"tion(", "function(", {GLOBAL,FUNCTION}} + {"tion(", "function(", {GLOBAL, FUNCTION}} }); SECTION("function scope: fake function") @@ -5385,13 +5887,13 @@ TEST_CASE("Scope tracking - over multiple PDU","[JSNormalizer]") SECTION("function scope: inside parameters") test_normalization({ {"function(", "function(", {GLOBAL, FUNCTION}}, - {")", "function()", {GLOBAL,FUNCTION}} + {")", "function()", {GLOBAL, FUNCTION}} }); SECTION("function scope: between parameters and body") test_normalization({ {"function()", "function()", {GLOBAL, FUNCTION}}, - {"{", "function(){", {GLOBAL,FUNCTION}} + {"{", "function(){", {GLOBAL, FUNCTION}} }); SECTION("function scope: inside code") @@ -5456,7 +5958,7 @@ TEST_CASE("Scope tracking - error handling", "[JSNormalizer]") SECTION("scope mismatch") test_normalization_bad( - "function f() { if (true) } }", + "function f() { if (true)} }", "function var_0000(){if(true)}", JSTokenizer::WRONG_CLOSING_SYMBOL ); @@ -5472,24 +5974,15 @@ TEST_CASE("Scope tracking - error handling", "[JSNormalizer]") { const char src[] = "function() { if (true) { } }"; const char exp[] = "function(){if"; - uint32_t scope_depth = 2; - - JSIdentifierCtx ident_ctx(norm_depth, scope_depth, s_ignored_ids, s_ignored_props); - JSNormalizer normalizer(ident_ctx, norm_depth, max_template_nesting, max_bracket_depth); - auto ret = normalizer.normalize(src, strlen(src)); - std::string dst(normalizer.get_script(), normalizer.script_size()); - CHECK(ret == JSTokenizer::SCOPE_NESTING_OVERFLOW); - CHECK(dst == exp); + test_normalization(src, exp, + {max_scope_depth(2), return_code(JSTokenizer::SCOPE_NESTING_OVERFLOW)}); } } TEST_CASE("Function call tracking - basic", "[JSNormalizer]") { - JSTokenizerTester tester(norm_depth, max_scope_depth, s_ignored_ids, s_ignored_props, - max_template_nesting, max_bracket_depth); - - using FuncType = JSTokenizerTester::FuncType; + auto tester = default_config; SECTION("Global only") { @@ -5536,8 +6029,7 @@ TEST_CASE("Function call tracking - basic", "[JSNormalizer]") SECTION("ignored fake defined function identifier") { const std::unordered_set s_ignored_ids_fake {"fake_unescape"}; - JSTokenizerTester tester_fake(norm_depth, max_scope_depth, s_ignored_ids_fake, - s_ignored_props, max_template_nesting, max_bracket_depth); + auto tester_fake = tester.derive({ignored_ids_list(s_ignored_ids_fake)}); tester_fake.test_function_scopes({ {"fake_unescape(", "fake_unescape(", {FuncType::NOT_FUNC, FuncType::GENERAL}} }); @@ -5769,10 +6261,7 @@ TEST_CASE("Function call tracking - basic", "[JSNormalizer]") TEST_CASE("Function call tracking - nesting", "[JSNormalizer]") { - JSTokenizerTester tester(norm_depth, max_scope_depth, s_ignored_ids, s_ignored_props, - max_template_nesting, max_bracket_depth); - - using FuncType = JSTokenizerTester::FuncType; + auto tester = default_config; SECTION("Opening") { @@ -5780,41 +6269,41 @@ TEST_CASE("Function call tracking - nesting", "[JSNormalizer]") { tester.test_function_scopes({ { "general( general( general(", "var_0000(var_0000(var_0000(", - { FuncType::NOT_FUNC, FuncType::GENERAL, FuncType::GENERAL, FuncType::GENERAL}} - }); - CHECK(!tester.is_unescape_nesting_seen()); + {FuncType::NOT_FUNC, FuncType::GENERAL, FuncType::GENERAL, FuncType::GENERAL}} + }, + {check_unescape_nesting(false)}); } SECTION("Multiple unescape functions") { tester.test_function_scopes({ {"unescape( unescape( unescape(", "", {FuncType::NOT_FUNC, FuncType::UNESCAPE, FuncType::UNESCAPE, FuncType::UNESCAPE}} - }); - CHECK(tester.is_unescape_nesting_seen()); + }, + {check_unescape_nesting(true)}); } SECTION("Multiple different unescape functions") { tester.test_function_scopes({ {"unescape( decodeURI( decodeURIComponent(", "", {FuncType::NOT_FUNC, FuncType::UNESCAPE, FuncType::UNESCAPE, FuncType::UNESCAPE}} - }); - CHECK(tester.is_unescape_nesting_seen()); + }, + {check_unescape_nesting(true)}); } SECTION("Multiple String.fromCharCode functions") { tester.test_function_scopes({ {"String.fromCharCode( String.fromCharCode( String.fromCharCode(", "'' '' '", {FuncType::NOT_FUNC, FuncType::CHAR_CODE, FuncType::CHAR_CODE, FuncType::CHAR_CODE}} - }); - CHECK(!tester.is_unescape_nesting_seen()); + }, + {check_unescape_nesting(false)}); } SECTION("Mixed function calls") { tester.test_function_scopes({ {"general( unescape( String.fromCharCode(", "var_0000('", {FuncType::NOT_FUNC, FuncType::GENERAL, FuncType::UNESCAPE, FuncType::CHAR_CODE}} - }); - CHECK(!tester.is_unescape_nesting_seen()); + }, + {check_unescape_nesting(false)}); } } SECTION("Closing") @@ -5861,11 +6350,7 @@ TEST_CASE("Function call tracking - nesting", "[JSNormalizer]") TEST_CASE("Function call tracking - over multiple PDU", "[JSNormalizer]") { - JSTokenizerTester tester(norm_depth, max_scope_depth, s_ignored_ids, s_ignored_props, - max_template_nesting, max_bracket_depth); - - using FuncType = JSTokenizerTester::FuncType; - + auto tester = default_config; SECTION("split in the middle of the identifier") { tester.test_function_scopes({ @@ -6165,260 +6650,5 @@ TEST_CASE("String Concatenation - Multiple PDU", "[JSNormalizer]") } } -#endif // CATCH_TEST_BUILD - -// Benchmark tests - -#ifdef BENCHMARK_TEST - -static constexpr const char* s_closing_tag = ""; - -static const std::string make_input(const char* begin, const char* mid, - const char* end, size_t len) -{ - std::string s(begin); - int fill = (len - strlen(begin) - strlen(end) - strlen(s_closing_tag)) / strlen(mid); - for (int i = 0; i < fill; ++i) - s.append(mid); - s.append(end); - s.append(s_closing_tag); - return s; -} - -static const std::string make_input_repeat(const char* pattern, size_t depth) -{ - std::string s; - size_t fill = (depth - strlen(s_closing_tag))/strlen(pattern); - for (size_t it = 0; it < fill; ++it) - s.append(pattern); - - s.append(s_closing_tag); - return s; -} - -static JSTokenizer::JSRet norm_ret(JSNormalizer& normalizer, const std::string& input) -{ - normalizer.rewind_output(); - return normalizer.normalize(input.c_str(), input.size()); -} - -TEST_CASE("JS Normalizer, literals by 8 K", "[JSNormalizer]") -{ - JSIdentifierCtxStub ident_ctx; - JSNormalizer normalizer(ident_ctx, unlim_depth, max_template_nesting, max_bracket_depth); - char dst[norm_depth]; - - constexpr size_t size = 1 << 13; - - auto data_pl = make_input("", ".", "", size); - auto data_ws = make_input("", " ", "", size); - auto data_bc = make_input("/*", " ", "*/", size); - auto data_dq = make_input("\"", " ", "\"", size); - - BENCHMARK("memcpy()") - { - return memcpy(dst, data_pl.c_str(), data_pl.size()); - }; - - REQUIRE(norm_ret(normalizer, data_ws) == JSTokenizer::SCRIPT_ENDED); - BENCHMARK("whitespaces") - { - normalizer.rewind_output(); - return normalizer.normalize(data_ws.c_str(), data_ws.size()); - }; - - REQUIRE(norm_ret(normalizer, data_bc) == JSTokenizer::SCRIPT_ENDED); - BENCHMARK("block comment") - { - normalizer.rewind_output(); - return normalizer.normalize(data_bc.c_str(), data_bc.size()); - }; - - REQUIRE(norm_ret(normalizer, data_dq) == JSTokenizer::SCRIPT_ENDED); - BENCHMARK("double quotes string") - { - normalizer.rewind_output(); - return normalizer.normalize(data_dq.c_str(), data_dq.size()); - }; -} - -TEST_CASE("JS Normalizer, literals by 64 K", "[JSNormalizer]") -{ - JSIdentifierCtxStub ident_ctx; - JSNormalizer normalizer(ident_ctx, unlim_depth, max_template_nesting, max_scope_depth); - char dst[norm_depth]; - - constexpr size_t size = 1 << 16; - - auto data_pl = make_input("", ".", "", size); - auto data_ws = make_input("", " ", "", size); - auto data_bc = make_input("/*", " ", "*/", size); - auto data_dq = make_input("\"", " ", "\"", size); - - BENCHMARK("memcpy()") - { - return memcpy(dst, data_pl.c_str(), data_pl.size()); - }; - - REQUIRE(norm_ret(normalizer, data_ws) == JSTokenizer::SCRIPT_ENDED); - BENCHMARK("whitespaces") - { - normalizer.rewind_output(); - return normalizer.normalize(data_ws.c_str(), data_ws.size()); - }; - - REQUIRE(norm_ret(normalizer, data_bc) == JSTokenizer::SCRIPT_ENDED); - BENCHMARK("block comment") - { - normalizer.rewind_output(); - return normalizer.normalize(data_bc.c_str(), data_bc.size()); - }; - - REQUIRE(norm_ret(normalizer, data_dq) == JSTokenizer::SCRIPT_ENDED); - BENCHMARK("double quotes string") - { - normalizer.rewind_output(); - return normalizer.normalize(data_dq.c_str(), data_dq.size()); - }; -} - -TEST_CASE("JS Normalizer, id normalization", "[JSNormalizer]") -{ - // around 11 000 identifiers - std::string input; - for (int it = 0; it < norm_depth; ++it) - input.append("n" + std::to_string(it) + " "); - - input.resize(norm_depth - strlen(s_closing_tag)); - input.append(s_closing_tag, strlen(s_closing_tag)); - - JSIdentifierCtxStub ident_ctx_mock; - JSNormalizer normalizer_wo_ident(ident_ctx_mock, unlim_depth, - max_template_nesting, max_bracket_depth); - - REQUIRE(norm_ret(normalizer_wo_ident, input) == JSTokenizer::SCRIPT_ENDED); - BENCHMARK("without substitution") - { - normalizer_wo_ident.rewind_output(); - return normalizer_wo_ident.normalize(input.c_str(), input.size()); - }; - - const std::unordered_set ids{}; - const std::unordered_set props{}; - JSIdentifierCtx ident_ctx(norm_depth, max_scope_depth, ids, props); - JSNormalizer normalizer_w_ident(ident_ctx, unlim_depth, max_template_nesting, max_bracket_depth); - - REQUIRE(norm_ret(normalizer_w_ident, input) == JSTokenizer::SCRIPT_ENDED); - BENCHMARK("with substitution") - { - normalizer_w_ident.rewind_output(); - return normalizer_w_ident.normalize(input.c_str(), input.size()); - }; - - const std::unordered_set ids_n { "n" }; - const std::unordered_set props_n { "n" }; - JSIdentifierCtx ident_ctx_ids_n(norm_depth, max_scope_depth, ids_n, props_n); - JSNormalizer normalizer_iids(ident_ctx_ids_n, unlim_depth, - max_template_nesting, max_bracket_depth); - - REQUIRE(norm_ret(normalizer_iids, input) == JSTokenizer::SCRIPT_ENDED); - BENCHMARK("with ignored identifiers") - { - normalizer_iids.rewind_output(); - return normalizer_iids.normalize(input.c_str(), input.size()); - }; -} - -TEST_CASE("JS Normalizer, scope tracking", "[JSNormalizer]") -{ - constexpr uint32_t depth = 65535; - JSIdentifierCtxStub ident_ctx; - JSNormalizer normalizer(ident_ctx, unlim_depth, max_template_nesting, depth); - - auto src_ws = make_input("", " ", "", depth); - auto src_brace_rep = make_input_repeat("{}", depth); - auto src_paren_rep = make_input_repeat("()", depth); - auto src_bracket_rep = make_input_repeat("[]", depth); - - REQUIRE(norm_ret(normalizer, src_ws) == JSTokenizer::SCRIPT_ENDED); - BENCHMARK("whitespaces") - { - normalizer.rewind_output(); - return normalizer.normalize(src_ws.c_str(), src_ws.size()); - }; - - REQUIRE(norm_ret(normalizer, src_brace_rep) == JSTokenizer::SCRIPT_ENDED); - BENCHMARK("...{}{}{}...") - { - normalizer.rewind_output(); - return normalizer.normalize(src_brace_rep.c_str(), src_brace_rep.size()); - }; - - REQUIRE(norm_ret(normalizer, src_paren_rep) == JSTokenizer::SCRIPT_ENDED); - BENCHMARK("...()()()...") - { - normalizer.rewind_output(); - return normalizer.normalize(src_paren_rep.c_str(), src_paren_rep.size()); - }; - - REQUIRE(norm_ret(normalizer, src_bracket_rep) == JSTokenizer::SCRIPT_ENDED); - BENCHMARK("...[][][]...") - { - normalizer.rewind_output(); - return normalizer.normalize(src_bracket_rep.c_str(), src_bracket_rep.size()); - }; -} - -TEST_CASE("JS Normalizer, automatic semicolon", "[JSNormalizer]") -{ - auto w_semicolons = make_input("", "a;\n", "", norm_depth); - auto wo_semicolons = make_input("", "a \n", "", norm_depth); - const char* src_w_semicolons = w_semicolons.c_str(); - const char* src_wo_semicolons = wo_semicolons.c_str(); - size_t src_len = w_semicolons.size(); - - JSIdentifierCtxStub ident_ctx_mock; - JSNormalizer normalizer_wo_ident(ident_ctx_mock, unlim_depth, max_template_nesting, norm_depth); - - REQUIRE(norm_ret(normalizer_wo_ident, w_semicolons) == JSTokenizer::SCRIPT_ENDED); - BENCHMARK("without semicolon insertion") - { - normalizer_wo_ident.rewind_output(); - return normalizer_wo_ident.normalize(src_w_semicolons, src_len); - }; - - REQUIRE(norm_ret(normalizer_wo_ident, wo_semicolons) == JSTokenizer::SCRIPT_ENDED); - BENCHMARK("with semicolon insertion") - { - normalizer_wo_ident.rewind_output(); - return normalizer_wo_ident.normalize(src_wo_semicolons, src_len); - }; -} - -TEST_CASE("JS Normalizer, unescape", "[JSNormalizer]") -{ - auto str_unescape = make_input("'", "\\u0061", "'", norm_depth); - auto f_unescape = make_input_repeat("unescape('')", norm_depth); - const char* src_str_unescape = str_unescape.c_str(); - const char* src_f_unescape = f_unescape.c_str(); - size_t src_len = norm_depth; - - JSIdentifierCtx ident_ctx(norm_depth, max_scope_depth, s_ignored_ids, s_ignored_props); - JSNormalizer norm(ident_ctx, unlim_depth, max_template_nesting, norm_depth); - - REQUIRE(norm_ret(norm, str_unescape) == JSTokenizer::SCRIPT_ENDED); - BENCHMARK("unescape sequence") - { - norm.rewind_output(); - return norm.normalize(src_str_unescape, src_len); - }; - - REQUIRE(norm_ret(norm, f_unescape) == JSTokenizer::SCRIPT_ENDED); - BENCHMARK("unescape function tracking") - { - norm.rewind_output(); - return norm.normalize(src_f_unescape, src_len); - }; -} +#endif -#endif // BENCHMARK_TEST diff --git a/src/utils/test/js_test_options.cc b/src/utils/test/js_test_options.cc new file mode 100644 index 000000000..cd5c9dffa --- /dev/null +++ b/src/utils/test/js_test_options.cc @@ -0,0 +1,198 @@ +//-------------------------------------------------------------------------- +// Copyright (C) 2022-2022 Cisco and/or its affiliates. All rights reserved. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License Version 2 as published +// by the Free Software Foundation. You may not use, modify or distribute +// this program under any other version of the GNU General Public License. +// +// This program is distributed in the hope that it will be useful, but +// WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with this program; if not, write to the Free Software Foundation, Inc., +// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +//-------------------------------------------------------------------------- +// js_test_options.cc author Danylo Kyrylov + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "js_test_options.h" + +Config::Config(const Config& other) : type(other.type) +{ + switch (other.type) + { + case ReturnCode: + val_jsret = other.val_jsret; + break; + case NormalizeIdentifiers: + case NormalizeAsExternal: + case CheckMixedEncoding: + case CheckUnescapeNesting: + case UseExpectedForLastPDU: + case CheckOpenTag: + case CheckClosingTag: + val_bool = other.val_bool; + break; + case IgnoredIdsList: + case IgnoredPropertiesList: + val_string_set = other.val_string_set; + break; + case NormDepth: + case IdentifierDepth: + case MaxTemplateNesting: + case MaxBracketDepth: + case MaxScopeDepth: + case MaxTokenBufSize: + case ExpectedCursorPos: + val_int = other.val_int; + break; + case TemporaryBuffer: + val_string = other.val_string; + break; + default: + assert(false); + } +} + +Config::~Config() +{ + // Explicitly destroy composite datatypes + switch (type) + { + case IgnoredIdsList: + case IgnoredPropertiesList: + val_string_set.~unordered_set(); + break; + case TemporaryBuffer: + val_string.~basic_string(); + default: + break; + } +} + +void ConfigSet::set_overrides(const Overrides& values) +{ + for (const auto& conf : values) + { + switch (conf.type) + { + case ReturnCode: + return_code = conf.val_jsret; + break; + case NormalizeIdentifiers: + normalize_identifiers = conf.val_bool; + break; + case NormalizeAsExternal: + normalize_as_external = conf.val_bool; + break; + case CheckMixedEncoding: + check_mixed_encoding = conf.val_bool; + break; + case CheckUnescapeNesting: + check_mixed_encoding = conf.val_bool; + break; + case UseExpectedForLastPDU: + use_expected_for_last_pdu = conf.val_bool; + break; + case IgnoredIdsList: + ignored_ids_list = conf.val_string_set; + break; + case IgnoredPropertiesList: + ignored_properties_list = conf.val_string_set; + break; + case NormDepth: + norm_depth = conf.val_int; + break; + case IdentifierDepth: + identifier_depth = conf.val_int; + break; + case MaxTemplateNesting: + max_template_nesting = conf.val_int; + break; + case MaxBracketDepth: + max_bracket_depth = conf.val_int; + break; + case MaxScopeDepth: + max_scope_depth = conf.val_int; + break; + case MaxTokenBufSize: + max_token_buf_size = conf.val_int; + break; + case ExpectedCursorPos: + expected_cursor_pos = conf.val_int; + break; + case CheckOpenTag: + check_open_tag = conf.val_bool; + break; + case CheckClosingTag: + check_closing_tag = conf.val_bool; + break; + case TemporaryBuffer: + temporary_buffer = conf.val_string; + break; + default: + assert(false); + } + } +} + +Config return_code(JSTokenizer::JSRet val) +{ return {ConfigType::ReturnCode, val}; } + +Config normalize_identifiers(bool val) +{ return {ConfigType::NormalizeIdentifiers, val}; } + +Config normalize_as_external(bool val) +{ return {ConfigType::NormalizeAsExternal, val}; } + +Config ignored_ids_list(StringSet val) +{ return {ConfigType::IgnoredIdsList, val}; } + +Config ignored_properties_list(StringSet val) +{ return {ConfigType::IgnoredPropertiesList, val}; } + +Config norm_depth(int val) +{ return {ConfigType::NormDepth, val}; } + +Config identifier_depth(int val) +{ return {ConfigType::IdentifierDepth, val}; } + +Config max_template_nesting(int val) +{ return {ConfigType::MaxTemplateNesting, val}; } + +Config max_bracket_depth(int val) +{ return {ConfigType::MaxBracketDepth, val}; } + +Config max_scope_depth(int val) +{ return {ConfigType::MaxScopeDepth, val}; } + +Config max_token_buf_size(int val) +{ return {ConfigType::MaxTokenBufSize, val}; } + +Config check_mixed_encoding(bool val) +{ return {ConfigType::CheckMixedEncoding, val}; } + +Config check_unescape_nesting(bool val) +{ return {ConfigType::CheckUnescapeNesting, val}; } + +Config use_expected_for_last_pdu(bool val) +{ return {ConfigType::UseExpectedForLastPDU, val}; } + +Config expected_cursor_pos(int val) +{ return {ConfigType::ExpectedCursorPos, val}; } + +Config check_open_tag(bool val) +{ return {ConfigType::CheckOpenTag, val}; } + +Config check_closing_tag(bool val) +{ return {ConfigType::CheckClosingTag, val}; } + +Config temporary_buffer(std::string val) +{ return {ConfigType::TemporaryBuffer, val}; } + diff --git a/src/utils/test/js_test_options.h b/src/utils/test/js_test_options.h new file mode 100644 index 000000000..e27102d81 --- /dev/null +++ b/src/utils/test/js_test_options.h @@ -0,0 +1,155 @@ +//-------------------------------------------------------------------------- +// Copyright (C) 2022-2022 Cisco and/or its affiliates. All rights reserved. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License Version 2 as published +// by the Free Software Foundation. You may not use, modify or distribute +// this program under any other version of the GNU General Public License. +// +// This program is distributed in the hope that it will be useful, but +// WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with this program; if not, write to the Free Software Foundation, Inc., +// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +//-------------------------------------------------------------------------- +// js_test_options.h author Danylo Kyrylov + +#ifndef JS_TEST_OPTIONS_H +#define JS_TEST_OPTIONS_H + +#include +#include +#include +#include +#include + +#include "utils/js_identifier_ctx.h" +#include "utils/js_normalizer.h" + +typedef std::unordered_set StringSet; + +enum ConfigType +{ + ReturnCode, + NormalizeIdentifiers, + NormalizeAsExternal, + IgnoredIdsList, + IgnoredPropertiesList, + NormDepth, + IdentifierDepth, + MaxTemplateNesting, + MaxBracketDepth, + MaxScopeDepth, + MaxTokenBufSize, + CheckMixedEncoding, + CheckUnescapeNesting, + UseExpectedForLastPDU, + ExpectedCursorPos, + CheckOpenTag, + CheckClosingTag, + TemporaryBuffer +}; + +struct Config +{ + ConfigType type; + union + { + JSTokenizer::JSRet val_jsret; + bool val_bool; + int val_int; + StringSet val_string_set; + std::string val_string; + }; + + Config(const Config& other); + Config(ConfigType type, JSTokenizer::JSRet val) : type(type), val_jsret(val){} + Config(ConfigType type, bool val) : type(type), val_bool(val){} + Config(ConfigType type, int val) : type(type), val_int(val){} + Config(ConfigType type, const StringSet& val) : type(type), val_string_set(val){} + Config(ConfigType type, const std::string& val) : type(type), val_string(val){} + ~Config(); +}; + +typedef std::initializer_list Overrides; + +class ConfigSet +{ +protected: + template + class Field + { + T val; + bool set = false; + + public: + Field() + { } + Field(const T& val): val(val) + { } + operator const T&() const + { return val; } + T& operator =(const T& new_value) + { val = new_value; set = true; return val; } + bool is_set() const + { return set; } + void unset() + { set = false; } + }; + +public: + Field return_code; + Field normalize_identifiers; + Field normalize_as_external; + Field ignored_ids_list; + Field ignored_properties_list; + Field norm_depth; + Field identifier_depth; + Field max_template_nesting; + Field max_bracket_depth; + Field max_scope_depth; + Field max_token_buf_size; + Field check_mixed_encoding; + Field check_unescape_nesting; + + // If true, check only new normalized part, otherwise check the whole normalized script + Field use_expected_for_last_pdu; + Field expected_cursor_pos; + Field check_open_tag; + Field check_closing_tag; + + // Add check for contents of the temporary buffer + Field temporary_buffer; + +protected: + void set_overrides(const Overrides& overrides); +}; + +Config return_code(JSTokenizer::JSRet val); +Config normalize_identifiers(bool val); +Config normalize_as_external(bool val); +Config ignored_ids_list(StringSet val); +Config ignored_properties_list(StringSet val); +Config norm_depth(int val); +Config identifier_depth(int val); +Config max_template_nesting(int val); +Config max_bracket_depth(int val); +Config max_scope_depth(int val); +Config max_token_buf_size(int val); +Config check_mixed_encoding(bool val); +Config check_unescape_nesting(bool val); + +// If true, check only new normalized part, otherwise check the whole normalized script +Config use_expected_for_last_pdu(bool val); +Config expected_cursor_pos(int val); +Config check_open_tag(bool val); +Config check_closing_tag(bool val); + +// Add check for contents of the temporary buffer +Config temporary_buffer(std::string val); + +#endif + diff --git a/src/utils/test/js_test_utils.cc b/src/utils/test/js_test_utils.cc index e20dee247..7b415dce0 100644 --- a/src/utils/test/js_test_utils.cc +++ b/src/utils/test/js_test_utils.cc @@ -21,7 +21,7 @@ #include "config.h" #endif -#include "utils/test/js_test_utils.h" +#include "js_test_utils.h" #include "catch/catch.hpp" @@ -38,20 +38,87 @@ THREAD_LOCAL const snort::Trace* http_trace = nullptr; using namespace snort; -void JSTokenizerTester::test_function_scopes(const std::list& pdus) +JSTokenizerTester::JSTokenizerTester(const JSTestConfig& conf) : + ident_ctx(conf.identifier_depth, + conf.max_scope_depth, + conf.ignored_ids_list, + conf.ignored_properties_list), + normalizer( + conf.normalize_identifiers ? + static_cast(ident_ctx) : + static_cast(ident_ctx_stub), + conf.norm_depth, + conf.max_template_nesting, + conf.max_bracket_depth, + conf.max_token_buf_size + ), + config(conf) +{ } + +void JSTokenizerTester::do_pdu(const std::string& source) +{ + last_source = source; + last_return = normalizer.normalize(last_source.c_str(), last_source.size(), + config.normalize_as_external.is_set() and config.normalize_as_external); +} + +void JSTokenizerTester::check_output(const std::string& expected) +{ + std::string result_str; + + if (config.use_expected_for_last_pdu.is_set() and config.use_expected_for_last_pdu) + { + auto size = normalizer.script_size(); + auto temp_buf = normalizer.take_script(); + result_str = {temp_buf, size}; + delete[] temp_buf; + } + else + result_str = {normalizer.get_script(), normalizer.script_size()}; + + CHECK(result_str == expected); +} + +void JSTokenizerTester::run_checks(const JSTestConfig& checks) +{ + if (checks.return_code.is_set()) + CHECK(last_return == checks.return_code); + + if (checks.check_open_tag.is_set()) + CHECK(normalizer.is_opening_tag_seen() == checks.check_open_tag); + + if (checks.check_closing_tag.is_set()) + CHECK(normalizer.is_closing_tag_seen() == checks.check_closing_tag); + + if (checks.check_mixed_encoding.is_set()) + CHECK(normalizer.is_mixed_encoding_seen() == checks.check_mixed_encoding); + + if (checks.check_unescape_nesting.is_set()) + CHECK(normalizer.is_unescape_nesting_seen() == checks.check_unescape_nesting); + + if (checks.expected_cursor_pos.is_set()) + CHECK((normalizer.get_src_next() - last_source.c_str()) == checks.expected_cursor_pos); + + if (checks.temporary_buffer.is_set()) + CHECK(std::string(normalizer.get_tmp_buf(), + normalizer.get_tmp_buf_size()) == static_cast(checks.temporary_buffer)); +} + +void JSTestConfig::test_function_scopes(const std::list& pdus) { + JSTokenizerTester tester(*this); + for (auto pdu : pdus) { - const char* source; - const char* expected; + std::string source; + std::string expected; std::list exp_stack; std::tie(source, expected, exp_stack) = pdu; - normalizer.normalize(source, strlen(source)); - std::string result_buf(normalizer.get_script(), normalizer.script_size()); - CHECK(result_buf == expected); + tester.do_pdu(source); + tester.check_output(expected); - auto tmp_stack(normalizer.get_tokenizer().scope_stack); + auto tmp_stack(tester.normalizer.get_tokenizer().scope_stack); CHECK(tmp_stack.size() == exp_stack.size()); for (auto func_it = exp_stack.rbegin(); func_it != exp_stack.rend() and !tmp_stack.empty(); func_it++) @@ -62,79 +129,152 @@ void JSTokenizerTester::test_function_scopes(const std::list& pdus) } } -bool JSTokenizerTester::is_unescape_nesting_seen() const +void JSTestConfig::test_function_scopes(const std::list& pdus, const Overrides& overrides) +{ + derive(overrides).test_function_scopes(pdus); +} + +JSTestConfig::JSTestConfig(const Overrides& values) +{ + set_overrides(values); +} + +JSTestConfig JSTestConfig::derive(const Overrides& values) const { - return normalizer.is_unescape_nesting_seen(); + JSTestConfig new_config(*this); + new_config.set_overrides(values); + return new_config; } -void test_scope(const char* context, const std::list& stack) +void JSTestConfig::test_scope(const std::string& context, const std::list& stack) const { - std::string buf(context); - buf += ""; - JSIdentifierCtx ident_ctx(norm_depth, max_scope_depth, s_ignored_ids, s_ignored_props); - JSNormalizer normalizer(ident_ctx, norm_depth, max_template_nesting, max_bracket_depth); - normalizer.normalize(buf.c_str(), buf.size()); - CHECK(ident_ctx.get_types() == stack); + JSTokenizerTester tester(*this); + std::string buf = context + ""; + + tester.do_pdu(buf); + CHECK(tester.ident_ctx.get_types() == stack); } -void test_normalization(const char* source, const char* expected) +void JSTestConfig::test_scope(const std::string& context, const std::list& stack, + const Overrides& overrides) const { - JSIdentifierCtx ident_ctx(norm_depth, max_scope_depth, s_ignored_ids, s_ignored_props); - JSNormalizer normalizer(ident_ctx, norm_depth, max_template_nesting, max_bracket_depth); - normalizer.normalize(source, strlen(source)); - std::string result_buf(normalizer.get_script(), normalizer.script_size()); - CHECK(result_buf == expected); + derive(overrides).test_scope(context, stack); } -void test_normalization_bad(const char* source, const char* expected, JSTokenizer::JSRet eret) +void JSTestConfig::test_normalization(const std::string& source, const std::string& expected) const { - JSIdentifierCtx ident_ctx(norm_depth, max_scope_depth, s_ignored_ids, s_ignored_props); - JSNormalizer normalizer(ident_ctx, norm_depth, max_template_nesting, max_bracket_depth); - auto ret = normalizer.normalize(source, strlen(source)); - std::string result_buf(normalizer.get_script(), normalizer.script_size()); - CHECK(eret == ret); - CHECK(result_buf == expected); + JSTokenizerTester tester(*this); + + tester.do_pdu(source); + + tester.check_output(expected); + tester.run_checks(*this); } -void test_normalization_mixed_encoding(const char* source, const char* expected) +void JSTestConfig::test_normalization(const std::string& source, const std::string& expected, + const Overrides& overrides) const { - JSIdentifierCtx ident_ctx(norm_depth, max_scope_depth, s_ignored_ids, s_ignored_props); - JSNormalizer normalizer(ident_ctx, norm_depth, max_template_nesting, max_bracket_depth); - auto ret = normalizer.normalize(source, strlen(source)); - std::string result_buf(normalizer.get_script(), normalizer.script_size()); - CHECK(ret == JSTokenizer::JSRet::SCRIPT_CONTINUE); - CHECK(normalizer.is_mixed_encoding_seen()); - CHECK(result_buf == expected); + derive(overrides).test_normalization(source, expected); } -void test_normalization(const std::vector& pdus) +void JSTestConfig::test_normalization(const std::vector& pdus) const { - JSIdentifierCtx ident_ctx(norm_depth, max_scope_depth, s_ignored_ids, s_ignored_props); - JSNormalizer normalizer(ident_ctx, norm_depth, max_template_nesting, max_bracket_depth); + JSTokenizerTester tester(*this); for (const auto& pdu : pdus) { - const char* source = pdu.first; - const char* expected = pdu.second; - normalizer.normalize(source, strlen(source)); - std::string result_buf(normalizer.get_script(), normalizer.script_size()); - CHECK(result_buf == expected); + auto source = pdu.first; + auto expected = pdu.second; + tester.do_pdu(source); + tester.check_output(expected); } + + tester.run_checks(*this); } -void test_normalization(const std::list& pdus) +void JSTestConfig::test_normalization(const std::vector& pdus, const Overrides& overrides) const { - JSIdentifierCtx ident_ctx(norm_depth, max_scope_depth, s_ignored_ids, s_ignored_props); - JSNormalizer normalizer(ident_ctx, norm_depth, max_template_nesting, max_bracket_depth); + derive(overrides).test_normalization(pdus); +} + +void JSTestConfig::test_normalization(const std::list& pdus) const +{ + JSTokenizerTester tester(*this); + for (auto pdu:pdus) { - const char* source; - const char* expected; + std::string source; + std::string expected; std::list stack; - std::tie(source,expected,stack) = pdu; - normalizer.normalize(source, strlen(source)); - std::string result_buf(normalizer.get_script(), normalizer.script_size()); - CHECK(ident_ctx.get_types() == stack); - CHECK(result_buf == expected); + std::tie(source, expected, stack) = pdu; + tester.do_pdu(source); + tester.check_output(expected); + CHECK(tester.ident_ctx.get_types() == stack); } + + tester.run_checks(*this); } + +void JSTestConfig::test_normalization(const std::list& pdus, const Overrides& overrides) const +{ + derive(overrides).test_normalization(pdus); +} + +void JSTestConfig::test_normalization_combined(const std::list& pdu_sources, + const std::string& combined_expected) const +{ + JSTokenizerTester tester(*this); + + for (const auto& source : pdu_sources) + tester.do_pdu(source); + + tester.check_output(combined_expected); + tester.run_checks(*this); +} + +void JSTestConfig::test_normalization_combined(const std::list& pdu_sources, + const std::string& combined_expected, const Overrides& overrides) const +{ + derive(overrides).test_normalization_combined(pdu_sources, combined_expected); +} + +void test_scope(const std::string& context, const std::list& stack) +{ + default_config.test_scope(context, stack); +} +void test_normalization(const std::string& source, const std::string& expected, const Overrides& overrides) +{ + default_config.test_normalization(source, expected, overrides); +} + +void test_normalization_noident(const std::string& source, const std::string& expected, const Overrides& overrides) +{ + default_config.derive(overrides).test_normalization(source, expected, {normalize_identifiers(false)}); +} + +void test_normalization_bad(const std::string& source, const std::string& expected, JSTokenizer::JSRet eret) +{ + default_config.test_normalization(source, expected, {return_code(eret)}); +} + +void test_normalization_mixed_encoding(const std::string& source, const std::string& expected) +{ + default_config.test_normalization(source, expected, {check_mixed_encoding(true)}); +} + +void test_normalization(const std::vector& pdus, const Overrides& overrides) +{ + default_config.test_normalization(pdus, overrides); +} + +void test_normalization(const std::list& pdus, const Overrides& overrides) +{ + default_config.test_normalization(pdus, overrides); +} + +void test_normalization_combined(const std::list& pdu_sources, const std::string& total_expected, + const Overrides& overrides) +{ + default_config.test_normalization_combined(pdu_sources, total_expected, overrides); +} + diff --git a/src/utils/test/js_test_utils.h b/src/utils/test/js_test_utils.h index 23a136edc..f407fa55d 100644 --- a/src/utils/test/js_test_utils.h +++ b/src/utils/test/js_test_utils.h @@ -29,19 +29,9 @@ #include "utils/js_identifier_ctx.h" #include "utils/js_normalizer.h" -constexpr int unlim_depth = -1; -constexpr int norm_depth = 65535; -constexpr int max_template_nesting = 4; -constexpr int max_bracket_depth = 256; -constexpr int max_scope_depth = 256; -static const std::unordered_set s_ignored_ids { - "console", "eval", "document", "unescape", "decodeURI", "decodeURIComponent", "String", - "name", "u" -}; +#include "js_test_options.h" -static const std::unordered_set s_ignored_props { - "watch", "unwatch", "split", "reverse", "join", "name", "w" -}; +constexpr int unlim_depth = -1; namespace snort { @@ -67,37 +57,93 @@ public: size_t size() const override { return 0; } }; +class JSTestConfig; + class JSTokenizerTester { public: - JSTokenizerTester(int32_t depth, uint32_t max_scope_depth, - const std::unordered_set& ignored_ids, - const std::unordered_set& ignored_props, - uint8_t max_template_nesting, uint32_t max_bracket_depth) - : - ident_ctx(depth, max_scope_depth, ignored_ids, ignored_props), - normalizer(ident_ctx, depth, max_template_nesting, max_bracket_depth) - { } - typedef JSTokenizer::FuncType FuncType; - typedef std::tuple> ScopeCase; - void test_function_scopes(const std::list& pdus); - bool is_unescape_nesting_seen() const; -private: + JSTokenizerTester(const JSTestConfig& conf); + + void do_pdu(const std::string& source); + void check_output(const std::string& expected); + void run_checks(const JSTestConfig& checks); + JSIdentifierCtx ident_ctx; + JSIdentifierCtxStub ident_ctx_stub; snort::JSNormalizer normalizer; + +private: + const JSTestConfig& config; + JSTokenizer::JSRet last_return; + std::string last_source; }; -void test_scope(const char* context, const std::list& stack); -void test_normalization(const char* source, const char* expected); -void test_normalization_bad(const char* source, const char* expected, JSTokenizer::JSRet eret); -void test_normalization_mixed_encoding(const char* source, const char* expected); -typedef std::pair PduCase; +typedef JSTokenizerTester::FuncType FuncType; + // source, expected for a single PDU -void test_normalization(const std::vector& pdus); -typedef std::tuple> ScopedPduCase; +typedef std::pair PduCase; + // source, expected, and current scope type stack for a single PDU -void test_normalization(const std::list& pdus); +typedef std::tuple> ScopedPduCase; +typedef std::tuple> FunctionScopeCase; + +class JSTestConfig : public ConfigSet +{ +public: + JSTestConfig(const Overrides& values); + JSTestConfig derive(const Overrides& values) const; + + snort::JSNormalizer&& make_normalizer() const; + + void test_scope(const std::string& context, const std::list& stack) const; + void test_scope(const std::string& context, const std::list& stack, + const Overrides& overrides) const; + + void test_function_scopes(const std::list& pdus); + void test_function_scopes(const std::list& pdus, const Overrides& overrides); + + void test_normalization(const std::string& source, const std::string& expected) const; + void test_normalization(const std::string& source, const std::string& expected, const Overrides& overrides) const; + + void test_normalization(const std::vector& pdus) const; + void test_normalization(const std::vector& pdus, const Overrides& overrides) const; + + void test_normalization(const std::list& pdus) const; + void test_normalization(const std::list& pdus, const Overrides& overrides) const; + + void test_normalization_combined(const std::list& pdu_sources, + const std::string& total_expected) const; + void test_normalization_combined(const std::list& pdu_sources, + const std::string& total_expected, const Overrides& overrides) const; +}; + +static const JSTestConfig default_config({ + norm_depth(65535), + identifier_depth(65535), + max_template_nesting(4), + max_bracket_depth(256), + max_scope_depth(256), + max_token_buf_size(256), + ignored_ids_list({ + "console", "eval", "document", "unescape", "decodeURI", "decodeURIComponent", "String", + "name", "u"}), + ignored_properties_list({ + "watch", "unwatch", "split", "reverse", "join", "name", "w"}), + normalize_identifiers(true) +}); + +void test_scope(const std::string& context, const std::list& stack); +void test_normalization(const std::string& source, const std::string& expected, const Overrides& overrides = {}); +void test_normalization_noident(const std::string& source, const std::string& expected, + const Overrides& overrides = {}); +void test_normalization_bad(const std::string& source, const std::string& expected, JSTokenizer::JSRet eret); +void test_normalization_mixed_encoding(const std::string& source, const std::string& expected); +void test_normalization(const std::vector& pdus, const Overrides& overrides = {}); +void test_normalization(const std::list& pdus, const Overrides& overrides = {}); +void test_normalization_combined(const std::list& pdu_sources, const std::string& total_expected, + const Overrides& overrides = {}); + +#endif -#endif // JS_TEST_UTILS_H