Pull request #3518: utils: fix Unicode LS PS handling in JavaScript

Merge in SNORT/snort3 from ~OSERHIIE/snort3:js_fix_lsps to master

Squashed commit of the following:

commit 0a5bd2f42ba011e233b4e4cef21e7530f005b97f
Author: Oleksandr Serhiienko <oserhiie@cisco.com>
Date:   Thu Jul 14 13:58:19 2022 +0300

    utils: fix Unicode LS PS handling in JavaScript

diff --git a/src/utils/js_tokenizer.l b/src/utils/js_tokenizer.l
index 45a4bcb873c50d63f6210f629cda79d99d224768..d5e06b3738c134bc09ff30c245cd2f890be24428 100644 (file)
--- a/src/utils/js_tokenizer.l
+++ b/src/utils/js_tokenizer.l
@@ -1110,7 +1110,8 @@ ALL_UNICODE    [\0-\x7F]|[\xC2-\xDF][\x80-\xBF]|(\xE0[\xA0-\xBF]|[\xE1-\xEF][\x8
 <dqstr,unesc_dqstr>\\{CR}{LF}                 { /* skip */ }
 <dqstr,unesc_dqstr>\\{LF}                     { /* skip */ }
 <dqstr,unesc_dqstr>\\{CR}                     { /* skip */ }
-<dqstr,unesc_dqstr>{LINE_TERMINATORS}         { BEGIN(regst); RETURN(BAD_TOKEN) }
+<dqstr,unesc_dqstr>{CR}                       { BEGIN(regst); RETURN(BAD_TOKEN) }
+<dqstr,unesc_dqstr>{LF}                       { BEGIN(regst); RETURN(BAD_TOKEN) }
 <dqstr,unesc_dqstr><<EOF>>                    { RETURN(SCRIPT_CONTINUE) }
 <dqstr>{UNICODE_ESCAPE_SEQUENCE}              |
 <dqstr>{HEX_ESCAPE_SEQUENCE}                  { escaped_unicode_utf_8(); }
@@ -1131,7 +1132,8 @@ ALL_UNICODE    [\0-\x7F]|[\xC2-\xDF][\x80-\xBF]|(\xE0[\xA0-\xBF]|[\xE1-\xEF][\x8
 <sqstr,unesc_sqstr>\\{CR}{LF}                 { /* skip */ }
 <sqstr,unesc_sqstr>\\{LF}                     { /* skip */ }
 <sqstr,unesc_sqstr>\\{CR}                     { /* skip */ }
-<sqstr,unesc_sqstr>{LINE_TERMINATORS}         { BEGIN(regst); RETURN(BAD_TOKEN) }
+<sqstr,unesc_sqstr>{CR}                       { BEGIN(regst); RETURN(BAD_TOKEN) }
+<sqstr,unesc_sqstr>{LF}                       { BEGIN(regst); RETURN(BAD_TOKEN) }
 <sqstr,unesc_sqstr><<EOF>>                    { RETURN(SCRIPT_CONTINUE) }
 <sqstr>{UNICODE_ESCAPE_SEQUENCE}              |
 <sqstr>{HEX_ESCAPE_SEQUENCE}                  { escaped_unicode_utf_8(); }
@@ -1166,9 +1168,10 @@ ALL_UNICODE    [\0-\x7F]|[\xC2-\xDF][\x80-\xBF]|(\xE0[\xA0-\xBF]|[\xE1-\xEF][\x8
 <regst>{LITERAL_REGEX_START}        { EXEC(literal_regex_start()) }
 <regex>{LITERAL_REGEX_END}          { EXEC(literal_regex_end()) }
 <regex>{HTML_TAG_SCRIPT_CLOSE}      { BEGIN(regst); RETURN(CLOSING_TAG) }
-<regex>\\{LF}                       |
-<regex>\\{CR}                       |
-<regex>{LINE_TERMINATORS}           { BEGIN(regst); RETURN(BAD_TOKEN) }
+<regex>\\{CR}                       { BEGIN(regst); RETURN(BAD_TOKEN) }
+<regex>\\{LF}                       { BEGIN(regst); RETURN(BAD_TOKEN) }
+<regex>{CR}                         { BEGIN(regst); RETURN(BAD_TOKEN) }
+<regex>{LF}                         { BEGIN(regst); RETURN(BAD_TOKEN) }
 <regex>{LITERAL_REGEX_G_OPEN}       { EXEC(literal_regex_g_open()) }
 <regex>{LITERAL_REGEX_G_CLOSE}      { EXEC(literal_regex_g_close()) }
 <regex>{UNICODE_ESCAPE_SEQUENCE}    |

diff --git a/src/utils/test/js_normalizer_test.cc b/src/utils/test/js_normalizer_test.cc
index d6709f6bc0add29bb34fe5d71d873c3ccd49b0a8..926f6f1b4fdfb48125e40e29053c99bcf7a2b7fb 100644 (file)
--- a/src/utils/test/js_normalizer_test.cc
+++ b/src/utils/test/js_normalizer_test.cc
@@ -1074,7 +1074,7 @@ static const char syntax_cases_buf15[] =
     "var invalid_str = 'abc\u2028 def' ;\n";
 
 static const char syntax_cases_expected15[] =
-    "var invalid_str='abc";
+    "var invalid_str='abc\u2028 def';";
 
 static const char syntax_cases_buf16[] =
     "var invalid_str = \"abc\n def\"";
@@ -1110,7 +1110,7 @@ static const char syntax_cases_buf21[] =
     "var invalid_str = 'abc\u2029 def' ;\n\r";
 
 static const char syntax_cases_expected21[] =
-    "var invalid_str='abc";
+    "var invalid_str='abc\u2029 def';";
 
 static const char syntax_cases_buf22[] =
     "tag`template\n \\\\\\${   }   \\\\${   a  + ` template ${ 1 + c  }`  }`";
@@ -1140,6 +1140,12 @@ static const char syntax_cases_expected25[] =
     "return /regex0/.var_0000+/regex1/.var_0001;"
     "return /regex2/.var_0002*/regex3/.var_0003;";
 
+static const char syntax_cases_buf26[] =
+    "var invalid_re = /abc \n def/ ;";
+
+static const char syntax_cases_expected26[] =
+    "var invalid_re=/abc ";
+
 TEST_CASE("syntax cases", "[JSNormalizer]")
 {
     SECTION("variables")
@@ -1217,6 +1223,16 @@ TEST_CASE("syntax cases", "[JSNormalizer]")
         NORMALIZE(syntax_cases_buf14);
         VALIDATE(syntax_cases_buf14, syntax_cases_expected14);
     }
+    SECTION("LS within literal")
+    {
+        NORMALIZE(syntax_cases_buf15);
+        VALIDATE(syntax_cases_buf15, syntax_cases_expected15);
+    }
+    SECTION("PS within literal")
+    {
+        NORMALIZE(syntax_cases_buf21);
+        VALIDATE(syntax_cases_buf21, syntax_cases_expected21);
+    }
     SECTION("template literals")
     {
         NORMALIZE(syntax_cases_buf22);
@@ -1230,16 +1246,6 @@ TEST_CASE("syntax cases", "[JSNormalizer]")
 
 TEST_CASE("bad tokens", "[JSNormalizer]")
 {
-    SECTION("LS chars within literal")
-    {
-        NORMALIZE(syntax_cases_buf15);
-        VALIDATE_FAIL(syntax_cases_buf15, syntax_cases_expected15, JSTokenizer::BAD_TOKEN, 25);
-    }
-    SECTION("PS chars within literal")
-    {
-        NORMALIZE(syntax_cases_buf21);
-        VALIDATE_FAIL(syntax_cases_buf21, syntax_cases_expected21, JSTokenizer::BAD_TOKEN, 25);
-    }
     SECTION("explicit LF within literal")
     {
         NORMALIZE(syntax_cases_buf16);
@@ -1265,6 +1271,11 @@ TEST_CASE("bad tokens", "[JSNormalizer]")
         NORMALIZE(syntax_cases_buf20);
         VALIDATE_FAIL(syntax_cases_buf20, syntax_cases_expected20, JSTokenizer::BAD_TOKEN, 23);
     }
+    SECTION("explicit LF within regex literal")
+    {
+        NORMALIZE(syntax_cases_buf26);
+        VALIDATE_FAIL(syntax_cases_buf26, syntax_cases_expected26, JSTokenizer::BAD_TOKEN, 23);
+    }
 }
 
 TEST_CASE("braces overflow", "[JSNormalizer]")