Pull request #3334: Opening/closing tags in external scripts.

Merge in SNORT/snort3 from ~OSHUMEIK/snort3:js_oc_tags to master

Squashed commit of the following:

commit 0ee5e10bae28eaed6ef387cb487cf51d102e1b84
Author: Oleksii Shumeiko <oshumeik@cisco.com>
Date:   Wed Mar 30 18:38:41 2022 +0300

    utils: allow opening/closing tags in external scripts

diff --git a/src/service_inspectors/http_inspect/http_js_norm.cc b/src/service_inspectors/http_inspect/http_js_norm.cc
index deb0722f140fd33538b470232d9ea63325ebffef..0964132521f0d7011da117bf6acb1ee3e9e891a8 100644 (file)
--- a/src/service_inspectors/http_inspect/http_js_norm.cc
+++ b/src/service_inspectors/http_inspect/http_js_norm.cc
@@ -59,12 +59,12 @@ static const char* ret2str(JSTokenizer::JSRet ret)
 }
 
 static inline JSTokenizer::JSRet js_normalize(JSNormalizer& ctx, const Packet* current_packet,
-    const char* const end, const char*& ptr)
+    const char* const end, const char*& ptr, bool external_script)
 {
     trace_logf(3, http_trace, TRACE_JS_DUMP, current_packet,
         "original[%zu]: %.*s\n", end - ptr, static_cast<int>(end - ptr), ptr);
 
-    auto ret = ctx.normalize(ptr, end - ptr);
+    auto ret = ctx.normalize(ptr, end - ptr, external_script);
     auto src_next = ctx.get_src_next();
 
     trace_logf(3, http_trace, TRACE_JS_PROC, current_packet,
@@ -167,7 +167,7 @@ void HttpJsNorm::do_external(const Field& input, Field& output,
         trace_logf(1, http_trace, TRACE_JS_PROC, current_packet,
             "external script at %zd offset\n", ptr - (const char*)input.start());
 
-        auto ret = js_normalize(js_ctx, current_packet, end, ptr);
+        auto ret = js_normalize(js_ctx, current_packet, end, ptr, true);
 
         switch (ret)
         {
@@ -310,7 +310,7 @@ void HttpJsNorm::do_inline(const Field& input, Field& output,
             max_template_nesting, max_bracket_depth, max_scope_depth, ignored_ids);
         auto output_size_before = js_ctx.script_size();
 
-        auto ret = js_normalize(js_ctx, current_packet, end, ptr);
+        auto ret = js_normalize(js_ctx, current_packet, end, ptr, false);
 
         switch (ret)
         {

diff --git a/src/utils/js_normalizer.cc b/src/utils/js_normalizer.cc
index 1cce7fa7f3cd1c661caf35a833b183a2a9650be1..d2ae20955c5bbc54be4a243b7f0772132c80a7d6 100644 (file)
--- a/src/utils/js_normalizer.cc
+++ b/src/utils/js_normalizer.cc
@@ -49,7 +49,7 @@ JSNormalizer::~JSNormalizer()
     tmp_buf_size = 0;
 }
 
-JSTokenizer::JSRet JSNormalizer::normalize(const char* src, size_t src_len)
+JSTokenizer::JSRet JSNormalizer::normalize(const char* src, size_t src_len, bool external_script)
 {
     assert(src);
 
@@ -81,7 +81,7 @@ JSTokenizer::JSRet JSNormalizer::normalize(const char* src, size_t src_len)
     out_buf.reserve(src_len * BUFF_EXP_FACTOR);
 
     size_t r_bytes = in_buf.last_chunk_offset();
-    auto ret = tokenizer.process(r_bytes);
+    auto ret = tokenizer.process(r_bytes, external_script);
 
     rem_bytes -= r_bytes;
     src_next = src + r_bytes;

diff --git a/src/utils/js_normalizer.h b/src/utils/js_normalizer.h
index fa53bb6e3784cef818fc1fbf9d8016117955e74e..dcfb1e94f8eadb13a8a61013e2c89b953166f54b 100644 (file)
--- a/src/utils/js_normalizer.h
+++ b/src/utils/js_normalizer.h
@@ -38,7 +38,8 @@ public:
         int tmp_cap_size = JSTOKENIZER_BUF_MAX_SIZE);
     ~JSNormalizer();
 
-    JSTokenizer::JSRet normalize(const char* src, size_t src_len);
+    JSTokenizer::JSRet normalize(const char* src, size_t src_len,
+        bool external_script = false);
 
     const char* get_src_next() const
     { return src_next; }

diff --git a/src/utils/js_tokenizer.h b/src/utils/js_tokenizer.h
index 21a1fd7258d6fdfb76505165b084c340ab5d75c7..c263a1a2cca4b87b81af49932ee1d26586f2d304 100644 (file)
--- a/src/utils/js_tokenizer.h
+++ b/src/utils/js_tokenizer.h
@@ -164,7 +164,7 @@ public:
         int cap_size = JSTOKENIZER_BUF_MAX_SIZE);
     ~JSTokenizer() override;
 
-    JSRet process(size_t& bytes_in);
+    JSRet process(size_t& bytes_in, bool external_script = false);
 
     void reset_output()
     { ignored_id_pos = -1; }
@@ -320,6 +320,7 @@ private:
     JSIdentifierCtxBase& ident_ctx;
     size_t bytes_read;
     size_t tmp_bytes_read;
+    bool ext_script;
 
     struct
     {

diff --git a/src/utils/js_tokenizer.l b/src/utils/js_tokenizer.l
index 263a917860911a37d5467e4062775c2046e5e4b7..d61b21555d2298ef0f4209f37f868ed8cf2637d1 100644 (file)
--- a/src/utils/js_tokenizer.l
+++ b/src/utils/js_tokenizer.l
@@ -1055,14 +1055,14 @@ ALL_UNICODE    [\0-\x7F]|[\xC2-\xDF][\x80-\xBF]|(\xE0[\xA0-\xBF]|[\xE1-\xEF][\x8
 
 %%
 
-<INITIAL,divop,regst,char_code>{WHITESPACES}              { /* skip */ }
-<INITIAL,divop,regst,char_code>{CHAR_ESCAPE_SEQUENCES}    { /* skip */ }
+<INITIAL,divop,regst,char_code>{WHITESPACES}           { /* skip */ }
+<INITIAL,divop,regst,char_code>{CHAR_ESCAPE_SEQUENCES} { /* skip */ }
 
-{LINE_TERMINATORS}                    { BEGIN(regst); newline_found = true; }
-<char_code>{LINE_TERMINATORS}         { newline_found = true; }
+{LINE_TERMINATORS}                            { BEGIN(regst); newline_found = true; }
+<char_code>{LINE_TERMINATORS}                 { newline_found = true; }
 
-<INITIAL,regex,dqstr,regst,sqstr,divop,char_code>{HTML_TAG_SCRIPT_OPEN} { BEGIN(regst); RETURN(OPENING_TAG) }
-{HTML_TAG_SCRIPT_CLOSE}             { EXEC(html_closing_script_tag()) }
+<INITIAL,regex,regst,divop,char_code>{HTML_TAG_SCRIPT_OPEN} { BEGIN(regst); RETURN(OPENING_TAG) }
+{HTML_TAG_SCRIPT_CLOSE}                       { EXEC(html_closing_script_tag()) }
 
     {HTML_COMMENT_OPEN}                       { BEGIN(lcomm); }
     {LINE_COMMENT_START}                      { BEGIN(lcomm); }
@@ -1072,25 +1072,26 @@ ALL_UNICODE    [\0-\x7F]|[\xC2-\xDF][\x80-\xBF]|(\xE0[\xA0-\xBF]|[\xE1-\xEF][\x8
 <lcomm>{LINE_COMMENT_END2}                    { BEGIN(regst); newline_found = true; }
 <char_code_lcomm>{LINE_COMMENT_END1}          { BEGIN(char_code); newline_found = true; }
 <char_code_lcomm>{LINE_COMMENT_END2}          { BEGIN(char_code); newline_found = true; }
-<lcomm,char_code_lcomm>{LINE_COMMENT_END3}    { BEGIN(regst); RETURN(OPENING_TAG) }
-<lcomm,char_code_lcomm>{LINE_COMMENT_END4}    { BEGIN(regst); RETURN(CLOSING_TAG) }
+<lcomm,char_code_lcomm>{LINE_COMMENT_END3}    { if (!ext_script) { BEGIN(regst); RETURN(OPENING_TAG) } }
+<lcomm,char_code_lcomm>{LINE_COMMENT_END4}    { if (!ext_script) { BEGIN(regst); RETURN(CLOSING_TAG) } }
 <lcomm,char_code_lcomm>{LINE_COMMENT_SKIP}    { /* skip */ }
 <lcomm,char_code_lcomm><<EOF>>                { RETURN(SCRIPT_CONTINUE) }
 
-    {BLOCK_COMMENT_START}                       { BEGIN(bcomm); }
-<char_code>{BLOCK_COMMENT_START}                { BEGIN(char_code_bcomm); }
-<bcomm>{BLOCK_COMMENT_END1}                     { BEGIN(regst); }
-<char_code_bcomm>{BLOCK_COMMENT_END1}           { BEGIN(char_code); }
-<bcomm,char_code_bcomm>{BLOCK_COMMENT_END2}     { BEGIN(regst); RETURN(OPENING_TAG) }
-<bcomm,char_code_bcomm>{BLOCK_COMMENT_END3}     { BEGIN(regst); RETURN(CLOSING_TAG) }
-<bcomm,char_code_bcomm>{BLOCK_COMMENT_LINE1}    |
-<bcomm,char_code_bcomm>{BLOCK_COMMENT_LINE2}    { newline_found = true; }
-<bcomm,char_code_bcomm>{BLOCK_COMMENT_SKIP}     { /* skip */ }
-<bcomm,char_code_bcomm><<EOF>>                  { RETURN(SCRIPT_CONTINUE) }
+    {BLOCK_COMMENT_START}                     { BEGIN(bcomm); }
+<char_code>{BLOCK_COMMENT_START}              { BEGIN(char_code_bcomm); }
+<bcomm>{BLOCK_COMMENT_END1}                   { BEGIN(regst); }
+<char_code_bcomm>{BLOCK_COMMENT_END1}         { BEGIN(char_code); }
+<bcomm,char_code_bcomm>{BLOCK_COMMENT_END2}   { if (!ext_script) { BEGIN(regst); RETURN(OPENING_TAG) } }
+<bcomm,char_code_bcomm>{BLOCK_COMMENT_END3}   { if (!ext_script) { BEGIN(regst); RETURN(CLOSING_TAG) } }
+<bcomm,char_code_bcomm>{BLOCK_COMMENT_LINE1}  |
+<bcomm,char_code_bcomm>{BLOCK_COMMENT_LINE2}  { newline_found = true; }
+<bcomm,char_code_bcomm>{BLOCK_COMMENT_SKIP}   { /* skip */ }
+<bcomm,char_code_bcomm><<EOF>>                { RETURN(SCRIPT_CONTINUE) }
 
     {LITERAL_DQ_STRING_START}                 { EXEC(literal_dq_string_start()) }
 <dqstr,unesc_dqstr>{LITERAL_DQ_STRING_END}    { dealias_append(); ECHO; BEGIN(divop); }
-<dqstr,unesc_dqstr>{HTML_TAG_SCRIPT_CLOSE}    { BEGIN(regst); RETURN(CLOSING_TAG) }
+<dqstr,unesc_dqstr>{HTML_TAG_SCRIPT_OPEN}     { if (ext_script) { ECHO; } else { BEGIN(regst); RETURN(OPENING_TAG) } }
+<dqstr,unesc_dqstr>{HTML_TAG_SCRIPT_CLOSE}    { if (ext_script) { ECHO; } else { BEGIN(regst); RETURN(CLOSING_TAG) } }
 <dqstr,unesc_dqstr>\\{CR}{LF}                 { /* skip */ }
 <dqstr,unesc_dqstr>\\{LF}                     { /* skip */ }
 <dqstr,unesc_dqstr>\\{CR}                     { /* skip */ }
@@ -1110,7 +1111,8 @@ ALL_UNICODE    [\0-\x7F]|[\xC2-\xDF][\x80-\xBF]|(\xE0[\xA0-\xBF]|[\xE1-\xEF][\x8
 
     {LITERAL_SQ_STRING_START}                 { EXEC(literal_sq_string_start()) }
 <sqstr,unesc_sqstr>{LITERAL_SQ_STRING_END}    { dealias_append(); ECHO; BEGIN(divop); }
-<sqstr,unesc_sqstr>{HTML_TAG_SCRIPT_CLOSE}    { BEGIN(regst); RETURN(CLOSING_TAG) }
+<sqstr,unesc_sqstr>{HTML_TAG_SCRIPT_OPEN}     { if (ext_script) { ECHO; } else { BEGIN(regst); RETURN(OPENING_TAG) } }
+<sqstr,unesc_sqstr>{HTML_TAG_SCRIPT_CLOSE}    { if (ext_script) { ECHO; } else { BEGIN(regst); RETURN(CLOSING_TAG) } }
 <sqstr,unesc_sqstr>\\{CR}{LF}                 { /* skip */ }
 <sqstr,unesc_sqstr>\\{LF}                     { /* skip */ }
 <sqstr,unesc_sqstr>\\{CR}                     { /* skip */ }
@@ -2818,11 +2820,12 @@ void JSTokenizer::char_code_no_match()
     memset((void*)(states + sp), 0, sizeof(states[0]));
 }
 
-JSTokenizer::JSRet JSTokenizer::process(size_t& bytes_in)
+JSTokenizer::JSRet JSTokenizer::process(size_t& bytes_in, bool external_script)
 {
     yy_flush_buffer(YY_CURRENT_BUFFER);
     unescape_nest_seen = false;
     mixed_encoding_seen = false;
+    ext_script = external_script;
 
     auto r = yylex();
 

diff --git a/src/utils/test/js_normalizer_test.cc b/src/utils/test/js_normalizer_test.cc
index d17044a0a7c9d947db28c5d3959c7f8b5b186d7a..d6a04bc558d76388337f8772ea3f3d49ed624e3c 100644 (file)
--- a/src/utils/test/js_normalizer_test.cc
+++ b/src/utils/test/js_normalizer_test.cc
@@ -45,6 +45,14 @@ using namespace snort;
     int act_len = norm.script_size();                              \
     const char* dst = norm.take_script();
 
+#define NORMALIZE_EXT(src)                                         \
+    JSIdentifierCtxStub ident_ctx;                                 \
+    JSNormalizer norm(ident_ctx, norm_depth, max_template_nesting, max_bracket_depth); \
+    auto ret = norm.normalize(src, sizeof(src), true);             \
+    const char* ptr = norm.get_src_next();                         \
+    int act_len = norm.script_size();                              \
+    const char* dst = norm.take_script();
+
 #define VALIDATE(src, expected)                 \
     CHECK(ret == JSTokenizer::SCRIPT_CONTINUE); \
     CHECK((ptr - src) == sizeof(src));          \
@@ -1574,6 +1582,9 @@ static const char unexpected_tag_buf0[] =
 static const char unexpected_tag_expected0[] =
     "var a=1;";
 
+static const char unexpected_tag_expected0_ext[] =
+    "var a=1;";
+
 static const char unexpected_tag_buf1[] =
     "var a = 1;\n"
     "<script type=application/javascript>\n"
@@ -1582,6 +1593,9 @@ static const char unexpected_tag_buf1[] =
 static const char unexpected_tag_expected1[] =
     "var a=1;";
 
+static const char unexpected_tag_expected1_ext[] =
+    "var a=1;";
+
 static const char unexpected_tag_buf2[] =
     "var a = 1;\n"
     "var str = '<script> something';\n"
@@ -1590,6 +1604,9 @@ static const char unexpected_tag_buf2[] =
 static const char unexpected_tag_expected2[] =
     "var a=1;var str='";
 
+static const char unexpected_tag_expected2_ext[] =
+    "var a=1;var str='<script> something';var b=2;";
+
 static const char unexpected_tag_buf3[] =
     "var a = 1;\n"
     "var str = 'something <script> something';\n"
@@ -1598,6 +1615,9 @@ static const char unexpected_tag_buf3[] =
 static const char unexpected_tag_expected3[] =
     "var a=1;var str='something ";
 
+static const char unexpected_tag_expected3_ext[] =
+    "var a=1;var str='something <script> something';var b=2;";
+
 static const char unexpected_tag_buf4[] =
     "var a = 1;\n"
     "var str = 'something <script>';\n"
@@ -1606,6 +1626,9 @@ static const char unexpected_tag_buf4[] =
 static const char unexpected_tag_expected4[] =
     "var a=1;var str='something ";
 
+static const char unexpected_tag_expected4_ext[] =
+    "var a=1;var str='something <script>';var b=2;";
+
 static const char unexpected_tag_buf5[] =
     "var a = 1;\n"
     "var str = '</script> something';\n"
@@ -1614,6 +1637,9 @@ static const char unexpected_tag_buf5[] =
 static const char unexpected_tag_expected5[] =
     "var a=1;var str='";
 
+static const char unexpected_tag_expected5_ext[] =
+    "var a=1;var str='</script> something';var b=2;";
+
 static const char unexpected_tag_buf6[] =
     "var a = 1;\n"
     "var str = 'something </script> something';\n"
@@ -1622,6 +1648,9 @@ static const char unexpected_tag_buf6[] =
 static const char unexpected_tag_expected6[] =
     "var a=1;var str='something ";
 
+static const char unexpected_tag_expected6_ext[] =
+    "var a=1;var str='something </script> something';var b=2;";
+
 static const char unexpected_tag_buf7[] =
     "var a = 1;\n"
     "var str = 'something </script>';\n"
@@ -1630,6 +1659,9 @@ static const char unexpected_tag_buf7[] =
 static const char unexpected_tag_expected7[] =
     "var a=1;var str='something ";
 
+static const char unexpected_tag_expected7_ext[] =
+    "var a=1;var str='something </script>';var b=2;";
+
 static const char unexpected_tag_buf8[] =
     "var a = 1;\n"
     "var str = 'something \\<script\\> something';\n"
@@ -1638,6 +1670,9 @@ static const char unexpected_tag_buf8[] =
 static const char unexpected_tag_expected8[] =
     "var a=1;var str='something \\";
 
+static const char unexpected_tag_expected8_ext[] =
+    "var a=1;var str='something \\<script\\> something';var b=2;";
+
 static const char unexpected_tag_buf9[] =
     "var a = 1;\n"
     "var str = 'something \\<\\/script\\> something';\n"
@@ -1646,6 +1681,9 @@ static const char unexpected_tag_buf9[] =
 static const char unexpected_tag_expected9[] =
     "var a=1;var str='something \\<\\/script\\> something';var b=2;";
 
+static const char unexpected_tag_expected9_ext[] =
+    "var a=1;var str='something \\<\\/script\\> something';var b=2;";
+
 static const char unexpected_tag_buf10[] =
     "var a = 1;\n"
     "//<script> something\n"
@@ -1654,6 +1692,9 @@ static const char unexpected_tag_buf10[] =
 static const char unexpected_tag_expected10[] =
     "var a=1;";
 
+static const char unexpected_tag_expected10_ext[] =
+    "var a=1;var b=2;";
+
 static const char unexpected_tag_buf11[] =
     "var a = 1;\n"
     "//something <script> something\n"
@@ -1662,6 +1703,9 @@ static const char unexpected_tag_buf11[] =
 static const char unexpected_tag_expected11[] =
     "var a=1;";
 
+static const char unexpected_tag_expected11_ext[] =
+    "var a=1;var b=2;";
+
 static const char unexpected_tag_buf12[] =
     "var a = 1;\n"
     "//something <script>\n"
@@ -1670,6 +1714,9 @@ static const char unexpected_tag_buf12[] =
 static const char unexpected_tag_expected12[] =
     "var a=1;";
 
+static const char unexpected_tag_expected12_ext[] =
+    "var a=1;var b=2;";
+
 static const char unexpected_tag_buf13[] =
     "var a = 1;\n"
     "/*<script> something*/\n"
@@ -1678,6 +1725,9 @@ static const char unexpected_tag_buf13[] =
 static const char unexpected_tag_expected13[] =
     "var a=1;";
 
+static const char unexpected_tag_expected13_ext[] =
+    "var a=1;var b=2;";
+
 static const char unexpected_tag_buf14[] =
     "var a = 1;\n"
     "/*something <script> something*/\n"
@@ -1686,6 +1736,9 @@ static const char unexpected_tag_buf14[] =
 static const char unexpected_tag_expected14[] =
     "var a=1;";
 
+static const char unexpected_tag_expected14_ext[] =
+    "var a=1;var b=2;";
+
 static const char unexpected_tag_buf15[] =
     "var a = 1;\n"
     "/*something <script>*/\n"
@@ -1694,6 +1747,9 @@ static const char unexpected_tag_buf15[] =
 static const char unexpected_tag_expected15[] =
     "var a=1;";
 
+static const char unexpected_tag_expected15_ext[] =
+    "var a=1;var b=2;";
+
 static const char unexpected_tag_buf16[] =
     "var a = 1;\n"
     "//</script> something\n"
@@ -1702,6 +1758,9 @@ static const char unexpected_tag_buf16[] =
 static const char unexpected_tag_expected16[] =
     "var a=1;";
 
+static const char unexpected_tag_expected16_ext[] =
+    "var a=1;var b=2;";
+
 static const char unexpected_tag_buf17[] =
     "var a = 1;\n"
     "<!--something </script> something//-->\n"
@@ -1710,6 +1769,9 @@ static const char unexpected_tag_buf17[] =
 static const char unexpected_tag_expected17[] =
     "var a=1;";
 
+static const char unexpected_tag_expected17_ext[] =
+    "var a=1;var b=2;";
+
 static const char unexpected_tag_buf18[] =
     "var a = 1;\n"
     "//something </script>\n"
@@ -1718,6 +1780,9 @@ static const char unexpected_tag_buf18[] =
 static const char unexpected_tag_expected18[] =
     "var a=1;";
 
+static const char unexpected_tag_expected18_ext[] =
+    "var a=1;var b=2;";
+
 static const char unexpected_tag_buf19[] =
     "var a = 1;\n"
     "/*</script>\n"
@@ -1727,6 +1792,9 @@ static const char unexpected_tag_buf19[] =
 static const char unexpected_tag_expected19[] =
     "var a=1;";
 
+static const char unexpected_tag_expected19_ext[] =
+    "var a=1;var b=2;";
+
 static const char unexpected_tag_buf20[] =
     "var a = 1;\n"
     "/*something\n"
@@ -1737,6 +1805,9 @@ static const char unexpected_tag_buf20[] =
 static const char unexpected_tag_expected20[] =
     "var a=1;";
 
+static const char unexpected_tag_expected20_ext[] =
+    "var a=1;var b=2;";
+
 static const char unexpected_tag_buf21[] =
     "var a = 1;\n"
     "/*something\n"
@@ -1746,6 +1817,9 @@ static const char unexpected_tag_buf21[] =
 static const char unexpected_tag_expected21[] =
     "var a=1;";
 
+static const char unexpected_tag_expected21_ext[] =
+    "var a=1;var b=2;";
+
 static const char unexpected_tag_buf22[] =
     "var a = 1;\n"
     "var str = 'script somescript /script something';\n"
@@ -1754,6 +1828,9 @@ static const char unexpected_tag_buf22[] =
 static const char unexpected_tag_expected22[] =
     "var a=1;var str='script somescript /script something';var b=2;";
 
+static const char unexpected_tag_expected22_ext[] =
+    "var a=1;var str='script somescript /script something';var b=2;";
+
 static const char unexpected_tag_buf23[] =
     "var a = 1;\n"
     "var str = 'script somescript /script something <script>';\n"
@@ -1762,6 +1839,9 @@ static const char unexpected_tag_buf23[] =
 static const char unexpected_tag_expected23[] =
     "var a=1;var str='script somescript /script something ";
 
+static const char unexpected_tag_expected23_ext[] =
+    "var a=1;var str='script somescript /script something <script>';var b=2;";
+
 static const char unexpected_tag_buf24[] =
     "var a = 1;\n"
     "var str = 'something <sCrIpT>';\n"
@@ -1770,6 +1850,9 @@ static const char unexpected_tag_buf24[] =
 static const char unexpected_tag_expected24[] =
     "var a=1;var str='something ";
 
+static const char unexpected_tag_expected24_ext[] =
+    "var a=1;var str='something <sCrIpT>';var b=2;";
+
 TEST_CASE("nested script tags", "[JSNormalizer]")
 {
     SECTION("explicit open tag - simple")
@@ -1899,6 +1982,135 @@ TEST_CASE("nested script tags", "[JSNormalizer]")
     }
 }
 
+TEST_CASE("nested script tags in an external script", "[JSNormalizer]")
+{
+    SECTION("explicit open tag - simple")
+    {
+        NORMALIZE_EXT(unexpected_tag_buf0);
+        VALIDATE_FAIL(unexpected_tag_buf0, unexpected_tag_expected0_ext, JSTokenizer::OPENING_TAG, 18);
+    }
+    SECTION("explicit open tag - complex")
+    {
+        NORMALIZE_EXT(unexpected_tag_buf1);
+        VALIDATE_FAIL(unexpected_tag_buf1, unexpected_tag_expected1_ext, JSTokenizer::OPENING_TAG, 18);
+    }
+    SECTION("open tag within literal - start")
+    {
+        NORMALIZE_EXT(unexpected_tag_buf2);
+        VALIDATE(unexpected_tag_buf2, unexpected_tag_expected2_ext);
+    }
+    SECTION("open tag within literal - mid")
+    {
+        NORMALIZE_EXT(unexpected_tag_buf3);
+        VALIDATE(unexpected_tag_buf3, unexpected_tag_expected3_ext);
+    }
+    SECTION("open tag within literal - end")
+    {
+        NORMALIZE_EXT(unexpected_tag_buf4);
+        VALIDATE(unexpected_tag_buf4, unexpected_tag_expected4_ext);
+    }
+    SECTION("close tag within literal - start")
+    {
+        NORMALIZE_EXT(unexpected_tag_buf5);
+        VALIDATE(unexpected_tag_buf5, unexpected_tag_expected5_ext);
+    }
+    SECTION("close tag within literal - mid")
+    {
+        NORMALIZE_EXT(unexpected_tag_buf6);
+        VALIDATE(unexpected_tag_buf6, unexpected_tag_expected6_ext);
+    }
+    SECTION("close tag within literal - end")
+    {
+        NORMALIZE_EXT(unexpected_tag_buf7);
+        VALIDATE(unexpected_tag_buf7, unexpected_tag_expected7_ext);
+    }
+    SECTION("open tag within literal - escaped")
+    {
+        NORMALIZE_EXT(unexpected_tag_buf8);
+        VALIDATE(unexpected_tag_buf8, unexpected_tag_expected8_ext);
+    }
+    SECTION("close tag within literal - escaped")
+    {
+        NORMALIZE_EXT(unexpected_tag_buf9);
+        VALIDATE(unexpected_tag_buf9, unexpected_tag_expected9_ext);
+    }
+    SECTION("open tag within single-line comment - start")
+    {
+        NORMALIZE_EXT(unexpected_tag_buf10);
+        VALIDATE(unexpected_tag_buf10, unexpected_tag_expected10_ext);
+    }
+    SECTION("open tag within single-line comment - mid")
+    {
+        NORMALIZE_EXT(unexpected_tag_buf11);
+        VALIDATE(unexpected_tag_buf11, unexpected_tag_expected11_ext);
+    }
+    SECTION("open tag within single-line comment - end")
+    {
+        NORMALIZE_EXT(unexpected_tag_buf12);
+        VALIDATE(unexpected_tag_buf12, unexpected_tag_expected12_ext);
+    }
+    SECTION("open tag within multi-line comment - start")
+    {
+        NORMALIZE_EXT(unexpected_tag_buf13);
+        VALIDATE(unexpected_tag_buf13, unexpected_tag_expected13_ext);
+    }
+    SECTION("open tag within multi-line comment - mid")
+    {
+        NORMALIZE_EXT(unexpected_tag_buf14);
+        VALIDATE(unexpected_tag_buf14, unexpected_tag_expected14_ext);
+    }
+    SECTION("open tag within multi-line comment - end")
+    {
+        NORMALIZE_EXT(unexpected_tag_buf15);
+        VALIDATE(unexpected_tag_buf15, unexpected_tag_expected15_ext);
+    }
+    SECTION("close tag within single-line comment - start")
+    {
+        NORMALIZE_EXT(unexpected_tag_buf16);
+        VALIDATE(unexpected_tag_buf16, unexpected_tag_expected16_ext);
+    }
+    SECTION("close tag within single-line comment - mid")
+    {
+        NORMALIZE_EXT(unexpected_tag_buf17);
+        VALIDATE(unexpected_tag_buf17, unexpected_tag_expected17_ext);
+    }
+    SECTION("close tag within single-line comment - end")
+    {
+        NORMALIZE_EXT(unexpected_tag_buf18);
+        VALIDATE(unexpected_tag_buf18, unexpected_tag_expected18_ext);
+    }
+    SECTION("close tag within multi-line comment - start")
+    {
+        NORMALIZE_EXT(unexpected_tag_buf19);
+        VALIDATE(unexpected_tag_buf19, unexpected_tag_expected19_ext);
+    }
+    SECTION("close tag within multi-line comment - mid")
+    {
+        NORMALIZE_EXT(unexpected_tag_buf20);
+        VALIDATE(unexpected_tag_buf20, unexpected_tag_expected20_ext);
+    }
+    SECTION("close tag within multi-line comment - end")
+    {
+        NORMALIZE_EXT(unexpected_tag_buf21);
+        VALIDATE(unexpected_tag_buf21, unexpected_tag_expected21_ext);
+    }
+    SECTION("multiple patterns - not matched")
+    {
+        NORMALIZE_EXT(unexpected_tag_buf22);
+        VALIDATE(unexpected_tag_buf22, unexpected_tag_expected22_ext);
+    }
+    SECTION("multiple patterns - matched")
+    {
+        NORMALIZE_EXT(unexpected_tag_buf23);
+        VALIDATE(unexpected_tag_buf23, unexpected_tag_expected23_ext);
+    }
+    SECTION("mixed lower and upper case")
+    {
+        NORMALIZE_EXT(unexpected_tag_buf24);
+        VALIDATE(unexpected_tag_buf24, unexpected_tag_expected24_ext);
+    }
+}
+
 TEST_CASE("split between tokens", "[JSNormalizer]")
 {
     SECTION("operator string")