From: Mike Stepanek (mstepane) Date: Tue, 17 Aug 2021 13:25:57 +0000 (+0000) Subject: Merge pull request #2996 in SNORT/snort3 from ~DKYRYLOV/snort3:js_norm_template_liter... X-Git-Tag: 3.1.11.0~19 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=ad42fd764860cb206bc2b039e52848fcbfeda55f;p=thirdparty%2Fsnort3.git Merge pull request #2996 in SNORT/snort3 from ~DKYRYLOV/snort3:js_norm_template_literals to master Squashed commit of the following: commit 0272c1a9b1d0b449b197120df5283fef1a9d2ee0 Author: dkyrylov Date: Wed Jul 21 15:44:37 2021 +0300 http_inspect: Add JavaScript template literals normalization --- diff --git a/src/service_inspectors/http_inspect/http_enum.h b/src/service_inspectors/http_inspect/http_enum.h index 47177cb73..4228d2de0 100755 --- a/src/service_inspectors/http_inspect/http_enum.h +++ b/src/service_inspectors/http_inspect/http_enum.h @@ -272,6 +272,7 @@ enum Infraction INF_JS_CODE_IN_EXTERNAL, INF_JS_SHORTENED_TAG, INF_JS_IDENTIFIER_OVERFLOW, + INF_JS_TMPL_NEST_OVFLOW, INF__MAX_VALUE }; @@ -401,6 +402,7 @@ enum EventSid EVENT_JS_CODE_IN_EXTERNAL = 268, EVENT_JS_SHORTENED_TAG = 269, EVENT_JS_IDENTIFIER_OVERFLOW = 270, + EVENT_JS_TMPL_NEST_OVFLOW = 271, EVENT__MAX_VALUE }; diff --git a/src/service_inspectors/http_inspect/http_flow_data.cc b/src/service_inspectors/http_inspect/http_flow_data.cc index 777b7ab00..7778a0155 100644 --- a/src/service_inspectors/http_inspect/http_flow_data.cc +++ b/src/service_inspectors/http_inspect/http_flow_data.cc @@ -243,7 +243,8 @@ void HttpFlowData::reset_js_ident_ctx() js_ident_ctx->reset(); } -snort::JSNormalizer& HttpFlowData::acquire_js_ctx(int32_t ident_depth, size_t norm_depth) +snort::JSNormalizer& HttpFlowData::acquire_js_ctx(int32_t ident_depth, size_t norm_depth, + uint8_t max_template_nesting) { if (js_normalizer) return *js_normalizer; @@ -254,7 +255,7 @@ snort::JSNormalizer& HttpFlowData::acquire_js_ctx(int32_t ident_depth, size_t no update_allocations(js_ident_ctx->size()); } - js_normalizer = new JSNormalizer(*js_ident_ctx, norm_depth); + js_normalizer = new JSNormalizer(*js_ident_ctx, norm_depth, max_template_nesting); update_allocations(JSNormalizer::size()); return *js_normalizer; @@ -271,7 +272,7 @@ void HttpFlowData::release_js_ctx() } #else void HttpFlowData::reset_js_ident_ctx() {} -snort::JSNormalizer& HttpFlowData::acquire_js_ctx(int32_t, size_t) +snort::JSNormalizer& HttpFlowData::acquire_js_ctx(int32_t, size_t, uint8_t) { return *js_normalizer; } void HttpFlowData::release_js_ctx() {} #endif diff --git a/src/service_inspectors/http_inspect/http_flow_data.h b/src/service_inspectors/http_inspect/http_flow_data.h index 2a1dfc148..36b4dab70 100644 --- a/src/service_inspectors/http_inspect/http_flow_data.h +++ b/src/service_inspectors/http_inspect/http_flow_data.h @@ -199,7 +199,8 @@ private: bool js_built_in_event = false; void reset_js_ident_ctx(); - snort::JSNormalizer& acquire_js_ctx(int32_t ident_depth, size_t norm_depth); + snort::JSNormalizer& acquire_js_ctx(int32_t ident_depth, size_t norm_depth, + uint8_t max_template_nesting); void release_js_ctx(); // *** Transaction management including pipelining diff --git a/src/service_inspectors/http_inspect/http_inspect.cc b/src/service_inspectors/http_inspect/http_inspect.cc index cdcaefac8..3060bab20 100755 --- a/src/service_inspectors/http_inspect/http_inspect.cc +++ b/src/service_inspectors/http_inspect/http_inspect.cc @@ -161,6 +161,8 @@ void HttpInspect::show(const SnortConfig*) const ConfigLogger::log_value("js_normalization_depth", params->js_norm_param.js_normalization_depth); ConfigLogger::log_value("js_norm_identifier_depth", params->js_norm_param.js_identifier_depth); + ConfigLogger::log_value("js_norm_max_tmpl_nest", + params->js_norm_param.max_template_nesting); ConfigLogger::log_value("bad_characters", bad_chars.c_str()); ConfigLogger::log_value("ignore_unreserved", unreserved_chars.c_str()); ConfigLogger::log_flag("percent_u", params->uri_param.percent_u); diff --git a/src/service_inspectors/http_inspect/http_js_norm.cc b/src/service_inspectors/http_inspect/http_js_norm.cc index f1536d381..44f806d4b 100644 --- a/src/service_inspectors/http_inspect/http_js_norm.cc +++ b/src/service_inspectors/http_inspect/http_js_norm.cc @@ -48,10 +48,11 @@ static inline JSTokenizer::JSRet js_normalize(JSNormalizer& ctx, const char* con } HttpJsNorm::HttpJsNorm(const HttpParaList::UriParam& uri_param_, int64_t normalization_depth_, - int32_t identifier_depth_) : + int32_t identifier_depth_, uint8_t max_template_nesting_) : uri_param(uri_param_), normalization_depth(normalization_depth_), identifier_depth(identifier_depth_), + max_template_nesting(max_template_nesting_), mpse_otag(nullptr), mpse_attr(nullptr), mpse_type(nullptr) @@ -127,7 +128,7 @@ void HttpJsNorm::enhanced_external_normalize(const Field& input, Field& output, dst_end = buffer + len; } - auto& ctx = ssn->acquire_js_ctx(identifier_depth, normalization_depth); + auto& ctx = ssn->acquire_js_ctx(identifier_depth, normalization_depth, max_template_nesting); auto ret = js_normalize(ctx, end, dst_end, ptr, dst); switch (ret) @@ -157,6 +158,11 @@ void HttpJsNorm::enhanced_external_normalize(const Field& input, Field& output, events->create_event(EVENT_JS_IDENTIFIER_OVERFLOW); ssn->js_built_in_event = true; break; + case JSTokenizer::TEMPLATE_NESTING_OVERFLOW: + *infractions += INF_JS_TMPL_NEST_OVFLOW; + events->create_event(EVENT_JS_TMPL_NEST_OVFLOW); + ssn->js_built_in_event = true; + break; default: assert(false); break; @@ -235,7 +241,7 @@ void HttpJsNorm::enhanced_inline_normalize(const Field& input, Field& output, dst_end = buffer + len; } - auto& ctx = ssn->acquire_js_ctx(identifier_depth, normalization_depth); + auto& ctx = ssn->acquire_js_ctx(identifier_depth, normalization_depth, max_template_nesting); auto dst_before = dst; auto ret = js_normalize(ctx, end, dst_end, ptr, dst); @@ -272,6 +278,11 @@ void HttpJsNorm::enhanced_inline_normalize(const Field& input, Field& output, events->create_event(EVENT_JS_IDENTIFIER_OVERFLOW); script_continue = false; break; + case JSTokenizer::TEMPLATE_NESTING_OVERFLOW: + *infractions += INF_JS_TMPL_NEST_OVFLOW; + events->create_event(EVENT_JS_TMPL_NEST_OVFLOW); + script_continue = false; + break; default: assert(false); script_continue = false; diff --git a/src/service_inspectors/http_inspect/http_js_norm.h b/src/service_inspectors/http_inspect/http_js_norm.h index c21c2462a..64b27c4e7 100644 --- a/src/service_inspectors/http_inspect/http_js_norm.h +++ b/src/service_inspectors/http_inspect/http_js_norm.h @@ -37,7 +37,7 @@ class HttpJsNorm { public: HttpJsNorm(const HttpParaList::UriParam&, int64_t normalization_depth, - int32_t identifier_depth); + int32_t identifier_depth, uint8_t max_template_nesting); ~HttpJsNorm(); void legacy_normalize(const Field& input, Field& output, HttpInfractions*, HttpEventGen*, @@ -61,6 +61,7 @@ private: const HttpParaList::UriParam& uri_param; int64_t normalization_depth; int32_t identifier_depth; + uint8_t max_template_nesting; bool configure_once = false; snort::SearchTool* mpse_otag; diff --git a/src/service_inspectors/http_inspect/http_module.cc b/src/service_inspectors/http_inspect/http_module.cc index c0d8e1184..e56b66f90 100755 --- a/src/service_inspectors/http_inspect/http_module.cc +++ b/src/service_inspectors/http_inspect/http_module.cc @@ -82,6 +82,10 @@ const Parameter HttpModule::http_params[] = { "js_norm_identifier_depth", Parameter::PT_INT, "0:260000", "260000", "max number of unique JavaScript identifiers to normalize" }, + { "js_norm_max_tmpl_nest", Parameter::PT_INT, "0:255", "32", + "maximum depth of template literal nesting that enhanced javascript normalizer " + "will process (experimental)" }, + { "max_javascript_whitespaces", Parameter::PT_INT, "1:65535", "200", "maximum consecutive whitespaces allowed within the JavaScript obfuscated data" }, @@ -222,6 +226,10 @@ bool HttpModule::set(const char*, Value& val, SnortConfig*) params->js_norm_param.is_javascript_normalization = params->js_norm_param.is_javascript_normalization or (v != 0); } + else if (val.is("js_norm_max_tmpl_nest")) + { + params->js_norm_param.max_template_nesting = val.get_uint8(); + } else if (val.is("max_javascript_whitespaces")) { params->js_norm_param.max_javascript_whitespaces = val.get_uint16(); @@ -410,7 +418,8 @@ bool HttpModule::end(const char*, int, SnortConfig*) if ( params->js_norm_param.is_javascript_normalization ) params->js_norm_param.js_norm = new HttpJsNorm(params->uri_param, - params->js_norm_param.js_normalization_depth, params->js_norm_param.js_identifier_depth); + params->js_norm_param.js_normalization_depth, params->js_norm_param.js_identifier_depth, + params->js_norm_param.max_template_nesting); params->script_detection_handle = script_detection_handle; diff --git a/src/service_inspectors/http_inspect/http_module.h b/src/service_inspectors/http_inspect/http_module.h index e1297abb9..cab31b8a3 100755 --- a/src/service_inspectors/http_inspect/http_module.h +++ b/src/service_inspectors/http_inspect/http_module.h @@ -57,6 +57,7 @@ public: bool is_javascript_normalization = false; int64_t js_normalization_depth = 0; int32_t js_identifier_depth = 0; + uint8_t max_template_nesting = 32; int max_javascript_whitespaces = 200; class HttpJsNorm* js_norm = nullptr; }; diff --git a/src/service_inspectors/http_inspect/http_tables.cc b/src/service_inspectors/http_inspect/http_tables.cc index 1177839b7..ffa484da7 100755 --- a/src/service_inspectors/http_inspect/http_tables.cc +++ b/src/service_inspectors/http_inspect/http_tables.cc @@ -433,6 +433,7 @@ const RuleMap HttpModule::http_events[] = { EVENT_JS_CODE_IN_EXTERNAL, "JavaScript code under the external script tags" }, { EVENT_JS_SHORTENED_TAG, "script opening tag in a short form" }, { EVENT_JS_IDENTIFIER_OVERFLOW, "max number of unique JavaScript identifiers reached" }, + { EVENT_JS_TMPL_NEST_OVFLOW, "JavaScript template literal nesting is over capacity" }, { 0, nullptr } }; diff --git a/src/service_inspectors/http_inspect/test/http_module_test.cc b/src/service_inspectors/http_inspect/test/http_module_test.cc index 134377823..584f7d8f9 100755 --- a/src/service_inspectors/http_inspect/test/http_module_test.cc +++ b/src/service_inspectors/http_inspect/test/http_module_test.cc @@ -65,10 +65,10 @@ long HttpTestManager::print_amount {}; bool HttpTestManager::print_hex {}; HttpJsNorm::HttpJsNorm(const HttpParaList::UriParam& uri_param_, int64_t normalization_depth_, - int32_t identifier_depth_) : + int32_t identifier_depth_, uint8_t max_template_nesting_) : uri_param(uri_param_), normalization_depth(normalization_depth_), - identifier_depth(identifier_depth_), mpse_otag(nullptr), mpse_attr(nullptr), - mpse_type(nullptr) {} + identifier_depth(identifier_depth_), max_template_nesting(max_template_nesting_), + mpse_otag(nullptr), mpse_attr(nullptr), mpse_type(nullptr) {} HttpJsNorm::~HttpJsNorm() = default; void HttpJsNorm::configure(){} int64_t Parameter::get_int(char const*) { return 0; } diff --git a/src/service_inspectors/http_inspect/test/http_uri_norm_test.cc b/src/service_inspectors/http_inspect/test/http_uri_norm_test.cc index 376e3d1e7..f285760b4 100755 --- a/src/service_inspectors/http_inspect/test/http_uri_norm_test.cc +++ b/src/service_inspectors/http_inspect/test/http_uri_norm_test.cc @@ -54,10 +54,10 @@ void show_stats(PegCount*, const PegInfo*, unsigned, const char*) { } void show_stats(PegCount*, const PegInfo*, const IndexVec&, const char*, FILE*) { } HttpJsNorm::HttpJsNorm(const HttpParaList::UriParam& uri_param_, int64_t normalization_depth_, - int32_t identifier_depth_) : + int32_t identifier_depth_, uint8_t max_template_nesting_) : uri_param(uri_param_), normalization_depth(normalization_depth_), - identifier_depth(identifier_depth_), mpse_otag(nullptr), mpse_attr(nullptr), - mpse_type(nullptr) {} + identifier_depth(identifier_depth_), max_template_nesting(max_template_nesting_), + mpse_otag(nullptr), mpse_attr(nullptr), mpse_type(nullptr) {} HttpJsNorm::~HttpJsNorm() = default; void HttpJsNorm::configure() {} int64_t Parameter::get_int(char const*) { return 0; } diff --git a/src/utils/js_normalizer.cc b/src/utils/js_normalizer.cc index 86d2d9ae5..3687be6ce 100644 --- a/src/utils/js_normalizer.cc +++ b/src/utils/js_normalizer.cc @@ -25,15 +25,15 @@ using namespace snort; -JSNormalizer::JSNormalizer(JSIdentifierCtxBase& js_ident_ctx, size_t norm_depth) +JSNormalizer::JSNormalizer(JSIdentifierCtxBase& js_ident_ctx, size_t norm_depth, + uint8_t max_template_nesting) : depth(norm_depth), rem_bytes(norm_depth), - unlim(true), + unlim(norm_depth == (size_t) - 1), src_next(nullptr), dst_next(nullptr), - tokenizer(in, out, js_ident_ctx) + tokenizer(in, out, js_ident_ctx, max_template_nesting) { - unlim = depth == (size_t)-1; } JSTokenizer::JSRet JSNormalizer::normalize(const char* src, size_t src_len, char* dst, size_t dst_len) diff --git a/src/utils/js_normalizer.h b/src/utils/js_normalizer.h index 13673e4a9..84e58bc3f 100644 --- a/src/utils/js_normalizer.h +++ b/src/utils/js_normalizer.h @@ -32,7 +32,7 @@ namespace snort class JSNormalizer { public: - JSNormalizer(JSIdentifierCtxBase& js_ident_ctx, size_t depth); + JSNormalizer(JSIdentifierCtxBase& js_ident_ctx, size_t depth, uint8_t max_template_nesting); const char* get_src_next() const { return src_next; } diff --git a/src/utils/js_tokenizer.h b/src/utils/js_tokenizer.h index e2612ac10..3bb13a99f 100644 --- a/src/utils/js_tokenizer.h +++ b/src/utils/js_tokenizer.h @@ -21,6 +21,8 @@ #define JS_TOKENIZER_H #include +#include +#include #include "log/messages.h" @@ -49,10 +51,11 @@ public: OPENING_TAG, CLOSING_TAG, BAD_TOKEN, - IDENTIFIER_OVERFLOW + IDENTIFIER_OVERFLOW, + TEMPLATE_NESTING_OVERFLOW }; - JSTokenizer(std::istream& in, std::ostream& out, JSIdentifierCtxBase& ident_ctx); + JSTokenizer(std::istream& in, std::ostream& out, JSIdentifierCtxBase& ident_ctx, uint8_t max_template_nesting); ~JSTokenizer() override; // returns JSRet @@ -70,12 +73,16 @@ private: JSRet do_operator_spacing(JSToken cur_token); JSRet do_identifier_substitution(const char* lexeme); bool unescape(const char* lexeme); + void process_punctuator(); + void process_closing_bracket(); + JSRet process_subst_open(); private: void* cur_buffer; void* tmp_buffer = nullptr; std::stringstream tmp; - + uint8_t max_template_nesting; + std::stack> bracket_depth; JSToken token = UNDEFINED; JSIdentifierCtxBase& ident_ctx; }; diff --git a/src/utils/js_tokenizer.l b/src/utils/js_tokenizer.l index 8182d4379..11972f120 100644 --- a/src/utils/js_tokenizer.l +++ b/src/utils/js_tokenizer.l @@ -94,7 +94,9 @@ KEYWORD break|case|debugger|in|import|protected|do|else|function|try|implemen /* punctuators */ /* according to https://ecma-international.org/ecma-262/5.1/#sec-7.7 */ CLOSING_BRACES ")"|"]" -PUNCTUATOR "{"|"}"|"("|"["|">="|"=="|"!="|"==="|"!=="|"."|";"|","|"<"|">"|"<="|"<<"|">>"|">>>"|"&"|"|"|"^"|"!"|"&&"|"||"|"?"|":"|"="|"+="|"-="|"*="|"%="|"<<="|">>="|">>>="|"&="|"|="|"^="|"~" +OPEN_BRACKET "{" +CLOSE_BRACKET "}" +PUNCTUATOR "("|"["|">="|"=="|"!="|"==="|"!=="|"."|";"|","|"<"|">"|"<="|"<<"|">>"|">>>"|"&"|"|"|"^"|"!"|"&&"|"||"|"?"|":"|"="|"+="|"-="|"*="|"%="|"<<="|">>="|">>>="|"&="|"|="|"^="|"~" OPERATOR "+"|"-"|"*"|"++"|"--"|"%" DIV_OPERATOR "/" DIV_ASSIGNMENT_OPERATOR "/=" @@ -882,6 +884,9 @@ LITERAL_DQ_STRING_SKIP \\\" LITERAL_SQ_STRING_START \' LITERAL_SQ_STRING_END \' LITERAL_SQ_STRING_SKIP \\\' +LITERAL_TEMPLATE_START \` +LITERAL_TEMPLATE_END \` +LITERAL_TEMPLATE_SUBST_START \$\{ LITERAL_REGEX_START \/[^*\/] LITERAL_REGEX_END \/[gimsuy]* LITERAL_REGEX_SKIP \\\/ @@ -920,6 +925,9 @@ ALL_UNICODE [\0-\x7F]|[\xC2-\xDF][\x80-\xBF]|(\xE0[\xA0-\xBF]|[\xE1-\xEF][\x8 /* in a double-quoted string */ %x dqstr +/* in a literal part of a template string */ +%x tmpll + /* in a regular expression */ %x regex @@ -969,6 +977,18 @@ ALL_UNICODE [\0-\x7F]|[\xC2-\xDF][\x80-\xBF]|(\xE0[\xA0-\xBF]|[\xE1-\xEF][\x8 . { ECHO; } <> { return SCRIPT_CONTINUE; } +{OPEN_BRACKET} { if (not bracket_depth.empty()) bracket_depth.top()++; process_punctuator(); } +{CLOSE_BRACKET} { process_closing_bracket(); } + + {LITERAL_TEMPLATE_START} { EXEC(do_spacing(LITERAL)) ECHO; BEGIN(tmpll); } +(\\\\)*{LITERAL_TEMPLATE_END} { ECHO; BEGIN(divop); } +(\\\\)*{LITERAL_TEMPLATE_SUBST_START} { EXEC(process_subst_open()) } +{HTML_TAG_SCRIPT_CLOSE} { BEGIN(regst); return CLOSING_TAG; } +(\\\\)*\\{LITERAL_TEMPLATE_SUBST_START} | /* escaped template substitution */ +(\\\\)*\\{LITERAL_TEMPLATE_END} | /* escaped backtick */ +. { ECHO; } +<> { return SCRIPT_CONTINUE; } + {LITERAL_REGEX_START} { EXEC(do_spacing(LITERAL)) yyout << '/'; yyless(1); BEGIN(regex); } {LITERAL_REGEX_END} { ECHO; BEGIN(divop); } {HTML_TAG_SCRIPT_CLOSE} { BEGIN(regst); return CLOSING_TAG; } @@ -983,7 +1003,7 @@ ALL_UNICODE [\0-\x7F]|[\xC2-\xDF][\x80-\xBF]|(\xE0[\xA0-\xBF]|[\xE1-\xEF][\x8 {DIV_ASSIGNMENT_OPERATOR} { ECHO; token = PUNCTUATOR; BEGIN(INITIAL); } {CLOSING_BRACES} { ECHO; token = PUNCTUATOR; BEGIN(divop); } -{PUNCTUATOR} { ECHO; token = PUNCTUATOR; BEGIN(regst); } +{PUNCTUATOR} { process_punctuator(); } {USE_STRICT_DIRECTIVE} { EXEC(do_spacing(DIRECTIVE)) ECHO; BEGIN(INITIAL); yyout << ';'; } {USE_STRICT_DIRECTIVE_SC} { EXEC(do_spacing(DIRECTIVE)) ECHO; BEGIN(INITIAL); } @@ -1073,8 +1093,10 @@ static std::string unescape_unicode(const char* lexeme) // JSTokenizer members -JSTokenizer::JSTokenizer(std::istream& in, std::ostream& out, JSIdentifierCtxBase& ident_ctx) +JSTokenizer::JSTokenizer(std::istream& in, std::ostream& out, JSIdentifierCtxBase& ident_ctx, + uint8_t max_template_nesting) : yyFlexLexer(in, out), + max_template_nesting(max_template_nesting), ident_ctx(ident_ctx) { BEGIN(regst); @@ -1190,3 +1212,38 @@ bool JSTokenizer::unescape(const char* lexeme) return true; } + +void JSTokenizer::process_punctuator() +{ + ECHO; + token = PUNCTUATOR; + BEGIN(regst); +} + +void JSTokenizer::process_closing_bracket() +{ + if ( not bracket_depth.empty() ) + { + if ( bracket_depth.top() ) + bracket_depth.top()--; + else + { + bracket_depth.pop(); + ECHO; + BEGIN(tmpll); + return; + } + } + process_punctuator(); +} + +JSTokenizer::JSRet JSTokenizer::process_subst_open() +{ + if ( bracket_depth.size() >= max_template_nesting ) + return TEMPLATE_NESTING_OVERFLOW; + bracket_depth.push(0); + token = PUNCTUATOR; + ECHO; + BEGIN(divop); + return EOS; +} \ No newline at end of file diff --git a/src/utils/test/js_normalizer_test.cc b/src/utils/test/js_normalizer_test.cc index 7c27c51a0..79fbb9278 100644 --- a/src/utils/test/js_normalizer_test.cc +++ b/src/utils/test/js_normalizer_test.cc @@ -49,11 +49,12 @@ public: using namespace snort; #define DEPTH 65535 +#define MAX_TEMPLATE_NESTNIG 4 #define NORMALIZE(src, expected) \ char dst[sizeof(expected)]; \ JSIdentifierCtxTest ident_ctx; \ - JSNormalizer norm(ident_ctx, DEPTH); \ + JSNormalizer norm(ident_ctx, DEPTH, MAX_TEMPLATE_NESTNIG); \ auto ret = norm.normalize(src, sizeof(src), dst, sizeof(dst)); \ const char* ptr = norm.get_src_next(); \ int act_len = norm.get_dst_next() - dst; @@ -73,7 +74,7 @@ using namespace snort; #define NORMALIZE_L(src, src_len, dst, dst_len, depth, ret, ptr, len) \ { \ JSIdentifierCtxTest ident_ctx; \ - JSNormalizer norm(ident_ctx, depth); \ + JSNormalizer norm(ident_ctx, depth, MAX_TEMPLATE_NESTNIG); \ ret = norm.normalize(src, src_len, dst, dst_len); \ ptr = norm.get_src_next(); \ len = norm.get_dst_next() - dst; \ @@ -322,13 +323,13 @@ static const char all_patterns_buf4[] = "/regex/g undefined null true false 2 23 2.3 2.23 .2 .02 4. +2 -2 " "+3.3 -3.3 +23 -32 2.3E45 3.E34 -2.3E45 -3.E34 +2.3E45 +3.E34 0x1234 0XFFFF Infinity " "\xE2\x88\x9E NaN \"\" \"double string\" \"d\" '' 'single string' 's' x=/regex/gs " - "x=2/2/1"; + "x=2/2/1 `\ntemplate\n`"; static const char all_patterns_expected4[] = "/regex/g undefined null true false 2 23 2.3 2.23 .2 .02 4.+2-2" "+3.3-3.3+23-32 2.3E45 3.E34-2.3E45-3.E34+2.3E45+3.E34 0x1234 0XFFFF Infinity " "\xE2\x88\x9E NaN \"\" \"double string\" \"d\" '' 'single string' 's' x=/regex/gs " - "x=2/2/1"; + "x=2/2/1 `\ntemplate\n`"; static const char all_patterns_buf5[] = "$2abc _2abc abc $__$ 肖晗 XÆA12 \\u0041abc \\u00FBdef \\u1234ghi ab\xE2\x80\xA8ww " @@ -338,6 +339,13 @@ static const char all_patterns_expected5[] = "$2abc _2abc abc $__$ 肖晗 XÆA12 \u0041abc \u00FBdef \u1234ghi ab ww " "ab ww ab ww ab ∞ ww 2 abc"; +static const char all_patterns_buf6[] = + "tag` template\n ${ a + b } template`"; + +static const char all_patterns_expected6[] = + "tag ` template\n ${a+b} template`"; + + TEST_CASE("all patterns", "[JSNormalizer]") { SECTION("whitespaces and special characters") @@ -423,6 +431,11 @@ TEST_CASE("all patterns", "[JSNormalizer]") NORMALIZE(all_patterns_buf5, all_patterns_expected5); VALIDATE(all_patterns_buf5, all_patterns_expected5); } + SECTION("template literals") + { + NORMALIZE(all_patterns_buf6, all_patterns_expected6); + VALIDATE(all_patterns_buf6, all_patterns_expected6); + } } // Tests for different syntax cases @@ -730,6 +743,19 @@ static const char syntax_cases_buf21[] = static const char syntax_cases_expected21[] = "var invalid_str='abc"; +static const char syntax_cases_buf22[] = + "tag`template\n \\\\\\${ } \\\\${ a + ` template ${ 1 + c }` }`"; + +static const char syntax_cases_expected22[] = + "tag `template\n \\\\\\${ } \\\\${a+` template ${1+c}`}`"; + +static const char syntax_cases_buf23[] = + "`${`${`${`${`${}`}`}`}`}`}"; + +static const char syntax_cases_expected23[] = + "`${`${`${`${`"; + + TEST_CASE("syntax cases", "[JSNormalizer]") { SECTION("variables") @@ -807,6 +833,11 @@ TEST_CASE("syntax cases", "[JSNormalizer]") NORMALIZE(syntax_cases_buf14, syntax_cases_expected14); VALIDATE(syntax_cases_buf14, syntax_cases_expected14); } + SECTION("template literals") + { + NORMALIZE(syntax_cases_buf22, syntax_cases_expected22); + VALIDATE(syntax_cases_buf22, syntax_cases_expected22); + } } TEST_CASE("bad tokens", "[JSNormalizer]") @@ -848,6 +879,16 @@ TEST_CASE("bad tokens", "[JSNormalizer]") } } +TEST_CASE("template literal overflow", "[JSNormalizer]") +{ + SECTION("exceeding template literal limit") + { + NORMALIZE(syntax_cases_buf23, syntax_cases_expected23); + VALIDATE_FAIL(syntax_cases_buf23, syntax_cases_expected23, + JSTokenizer::TEMPLATE_NESTING_OVERFLOW, 15); + } +} + TEST_CASE("endings", "[JSNormalizer]") { SECTION("script closing tag is present", "[JSNormalizer]") @@ -882,7 +923,7 @@ TEST_CASE("endings", "[JSNormalizer]") int ret; JSIdentifierCtxTest ident_ctx; - JSNormalizer norm(ident_ctx, 7); + JSNormalizer norm(ident_ctx, 7, MAX_TEMPLATE_NESTNIG); ret = norm.normalize(src, sizeof(src), dst, sizeof(dst)); ptr = norm.get_src_next(); act_len = norm.get_dst_next() - dst;