Pull request #3200: utils: (js_tokenizer) fixup in states adjustment

Merge in SNORT/snort3 from ~OSERHIIE/snort3:js_states_adjustment_fix to master

Squashed commit of the following:

commit 05ac203e5388a0e4cae715cd0e25d6bb46dad66a
Author: Oleksandr Serhiienko <oserhiie@cisco.com>
Date:   Mon Dec 6 11:35:37 2021 +0200

    utils: fix state adjustment in JS Tokenizer

    The state before EOF ought to be cleaned up during states adjustment.

    Add test coverage.

diff --git a/src/utils/js_normalizer.h b/src/utils/js_normalizer.h
index 2aea26f0332877eb8cb21400eaac6c7be95f3bcc..508ac20aae4f55969cf75e90025d5e1abc74e30f 100644 (file)
--- a/src/utils/js_normalizer.h
+++ b/src/utils/js_normalizer.h
@@ -58,6 +58,13 @@ public:
     static size_t size()
     { return sizeof(JSNormalizer) + 16834; /* YY_BUF_SIZE */ }
 
+#ifdef CATCH_TEST_BUILD
+    const char* get_tmp_buf() const
+    { return tmp_buf; }
+    size_t get_tmp_buf_size() const
+    { return tmp_buf_size; }
+#endif
+
 #ifdef BENCHMARK_TEST
     void rewind_output()
     { out_buf.pubseekoff(0, std::ios_base::beg, std::ios_base::out); }

diff --git a/src/utils/js_tokenizer.l b/src/utils/js_tokenizer.l
index 762583ea8bdcedce5f17045f4dc78466a72d0aaf..369cd0c1cc2230c0bb2ceedd8092b852136f4e23 100644 (file)
--- a/src/utils/js_tokenizer.l
+++ b/src/utils/js_tokenizer.l
@@ -1655,11 +1655,11 @@ void JSTokenizer::states_adjust()
 
     // Reset all the states after the current state till the state before EOF
     if (sp <= eof_sp)
-        memset((void*)(states + sp), 0, sizeof(states[0]) * (eof_sp - sp));
+        memset((void*)(states + sp), 0, sizeof(states[0]) * (eof_sp - sp + 1));
     else
     {
         memset((void*)(states + sp), 0, sizeof(states[0]) * (JSTOKENIZER_MAX_STATES - sp));
-        memset(&states, 0, sizeof(states[0]) * eof_sp);
+        memset(&states, 0, sizeof(states[0]) * (eof_sp + 1));
     }
     --sp;
 }

diff --git a/src/utils/test/js_normalizer_test.cc b/src/utils/test/js_normalizer_test.cc
index 4ac4d1444616adfbad45dd330af5029a735ce215..5e4e0661ed6c5a48e35b3222cc7c1567225088f1 100644 (file)
--- a/src/utils/test/js_normalizer_test.cc
+++ b/src/utils/test/js_normalizer_test.cc
@@ -2455,7 +2455,7 @@ TEST_CASE("split and continuation combined", "[JSNormalizer]")
         char dst4[sizeof(exp4)];
 
         JSIdentifierCtx ident_ctx(DEPTH, MAX_SCOPE_DEPTH, s_ident_built_in);
-        JSNormalizer norm(ident_ctx, DEPTH, MAX_TEMPLATE_NESTING, MAX_SCOPE_DEPTH);
+        JSNormalizer norm(ident_ctx, DEPTH, MAX_TEMPLATE_NESTING, MAX_BRACKET_DEPTH);
 
         DO(src1, sizeof(src1) - 1, dst1, sizeof(dst1) - 1);
         CHECK(!memcmp(exp1, dst1, sizeof(exp1) - 1));
@@ -2469,6 +2469,44 @@ TEST_CASE("split and continuation combined", "[JSNormalizer]")
         DO(src4, sizeof(src4) - 1, dst4, sizeof(dst4) - 1);
         CHECK(!memcmp(exp4, dst4, sizeof(exp4) - 1));
 
+        CLOSE();
+    }
+    SECTION("PDU 1 [cont] PDU 2 [cont] PDU 3 [end]")
+    {
+        const char src1[] = "<";
+        const char src2[] = "!-";
+        const char src3[] = "-comment\n";
+
+        const char exp1[] = "<";
+        const char exp2[] = "<!-";
+        const char exp3[] = "";
+
+        const char tmp_buf1[] = "<";
+        const char tmp_buf2[] = "<!-";
+        const char tmp_buf3[] = "<!--comment\n";
+
+        char dst1[sizeof(exp1)];
+        char dst2[sizeof(exp2)];
+        char dst3[sizeof(exp3)];
+
+        JSIdentifierCtx ident_ctx(DEPTH, MAX_SCOPE_DEPTH, s_ident_built_in);
+        JSNormalizer norm(ident_ctx, DEPTH, MAX_TEMPLATE_NESTING, MAX_BRACKET_DEPTH);
+
+        TRY(src1, sizeof(src1) - 1, dst1, sizeof(dst1) - 1, JSTokenizer::SCRIPT_CONTINUE);
+        CHECK(!memcmp(exp1, dst1, sizeof(exp1) - 1));
+        REQUIRE(norm.get_tmp_buf_size() == sizeof(tmp_buf1) - 1);
+        CHECK(!memcmp(norm.get_tmp_buf(), tmp_buf1, sizeof(tmp_buf1) - 1));
+
+        TRY(src2, sizeof(src2) - 1, dst2, sizeof(dst2) - 1, JSTokenizer::SCRIPT_CONTINUE);
+        CHECK(!memcmp(exp2, dst2, sizeof(exp2) - 1));
+        REQUIRE(norm.get_tmp_buf_size() == sizeof(tmp_buf2) - 1);
+        CHECK(!memcmp(norm.get_tmp_buf(), tmp_buf2, sizeof(tmp_buf2) - 1));
+
+        TRY(src3, sizeof(src3) - 1, dst3, sizeof(dst3) - 1, JSTokenizer::SCRIPT_CONTINUE);
+        CHECK(!memcmp(exp3, dst3, sizeof(exp3) - 1));
+        REQUIRE(norm.get_tmp_buf_size() == sizeof(tmp_buf3) - 1);
+        CHECK(!memcmp(norm.get_tmp_buf(), tmp_buf3, sizeof(tmp_buf3) - 1));
+
         CLOSE();
     }
 }