app-layer/pd: review bailout conditions

To take TCP window into account
And to actually bail out if we received too much data
where the limit is configured by stream.reassembly.depth

(cherry picked from commit f77b027ada0436855109f422bfa4d491dba846c7)

diff --git a/src/app-layer.c b/src/app-layer.c
index 4335984480b3478930b072260a17f03c63d6137d..6430128370c470f4f0073af27f1405b7cf7ef053 100644 (file)
--- a/src/app-layer.c
+++ b/src/app-layer.c
@@ -208,64 +208,60 @@ static void TCPProtoDetectCheckBailConditions(ThreadVars *tv,
     const uint32_t size_tc = StreamDataAvailableForProtoDetect(&ssn->server);
     SCLogDebug("size_ts %" PRIu32 ", size_tc %" PRIu32, size_ts, size_tc);
 
-#ifdef DEBUG_VALIDATION
-    if (!(ssn->client.flags & STREAMTCP_STREAM_FLAG_GAP))
-        BUG_ON(size_ts > 1000000UL);
-    if (!(ssn->server.flags & STREAMTCP_STREAM_FLAG_GAP))
-        BUG_ON(size_tc > 1000000UL);
-#endif /* DEBUG_VALIDATION */
+    /* at least 100000 whatever the conditions
+     * and can be more if window is bigger and if configuration allows it */
+    const uint32_t size_tc_limit =
+        MAX(100000, MIN(ssn->client.window, stream_config.reassembly_depth));
+    const uint32_t size_ts_limit =
+        MAX(100000, MIN(ssn->server.window, stream_config.reassembly_depth));
 
     if (ProtoDetectDone(f, ssn, STREAM_TOSERVER) &&
         ProtoDetectDone(f, ssn, STREAM_TOCLIENT))
     {
         goto failure;
 
-    } else if (FLOW_IS_PM_DONE(f, STREAM_TOSERVER) && FLOW_IS_PP_DONE(f, STREAM_TOSERVER) &&
-            size_ts > 100000 && size_tc == 0)
-    {
-        AppLayerDecoderEventsSetEventRaw(&p->app_layer_events,
-                APPLAYER_PROTO_DETECTION_SKIPPED);
-        goto failure;
-
-    } else if (FLOW_IS_PM_DONE(f, STREAM_TOCLIENT) && FLOW_IS_PP_DONE(f, STREAM_TOCLIENT) &&
-            size_tc > 100000 && size_ts == 0)
-    {
-        AppLayerDecoderEventsSetEventRaw(&p->app_layer_events,
-                APPLAYER_PROTO_DETECTION_SKIPPED);
-        goto failure;
-
-    /* little data in ts direction, pp done, pm not done (max
-     * depth not reached), ts direction done, lots of data in
-     * tc direction. */
-    } else if (size_tc > 100000 &&
-            FLOW_IS_PP_DONE(f, STREAM_TOSERVER) && !(FLOW_IS_PM_DONE(f, STREAM_TOSERVER)) &&
-            FLOW_IS_PM_DONE(f, STREAM_TOCLIENT) && FLOW_IS_PP_DONE(f, STREAM_TOCLIENT))
-    {
-        AppLayerDecoderEventsSetEventRaw(&p->app_layer_events,
-                APPLAYER_PROTO_DETECTION_SKIPPED);
-        goto failure;
-
-    /* little data in tc direction, pp done, pm not done (max
-     * depth not reached), tc direction done, lots of data in
-     * ts direction. */
-    } else if (size_ts > 100000 &&
-            FLOW_IS_PP_DONE(f, STREAM_TOCLIENT) && !(FLOW_IS_PM_DONE(f, STREAM_TOCLIENT)) &&
-            FLOW_IS_PM_DONE(f, STREAM_TOSERVER) && FLOW_IS_PP_DONE(f, STREAM_TOSERVER))
-    {
-        AppLayerDecoderEventsSetEventRaw(&p->app_layer_events,
-                APPLAYER_PROTO_DETECTION_SKIPPED);
-        goto failure;
-
-    /* in case of really low TS data (e.g. 4 bytes) we can have
-     * the PP complete, PM not complete (depth not reached) and
-     * the TC side also not recognized (proto unknown) */
-    } else if (size_tc > 100000 &&
-            FLOW_IS_PP_DONE(f, STREAM_TOSERVER) && !(FLOW_IS_PM_DONE(f, STREAM_TOSERVER)) &&
-            (!FLOW_IS_PM_DONE(f, STREAM_TOCLIENT) && !FLOW_IS_PP_DONE(f, STREAM_TOCLIENT)))
-    {
-        AppLayerDecoderEventsSetEventRaw(&p->app_layer_events,
-                APPLAYER_PROTO_DETECTION_SKIPPED);
-        goto failure;
+        /* we bail out whatever the pp and pm states if
+         * we received too much data */
+    } else if (size_tc > 2 * size_tc_limit || size_ts > 2 * size_ts_limit) {
+      AppLayerDecoderEventsSetEventRaw(&p->app_layer_events,
+                                       APPLAYER_PROTO_DETECTION_SKIPPED);
+      goto failure;
+
+    } else if (FLOW_IS_PM_DONE(f, STREAM_TOSERVER) &&
+               FLOW_IS_PP_DONE(f, STREAM_TOSERVER) && size_ts > size_ts_limit &&
+               size_tc == 0) {
+      AppLayerDecoderEventsSetEventRaw(&p->app_layer_events,
+                                       APPLAYER_PROTO_DETECTION_SKIPPED);
+      goto failure;
+
+    } else if (FLOW_IS_PM_DONE(f, STREAM_TOCLIENT) &&
+               FLOW_IS_PP_DONE(f, STREAM_TOCLIENT) && size_tc > size_tc_limit &&
+               size_ts == 0) {
+      AppLayerDecoderEventsSetEventRaw(&p->app_layer_events,
+                                       APPLAYER_PROTO_DETECTION_SKIPPED);
+      goto failure;
+
+      /* little data in ts direction, pp done, pm not done (max
+       * depth not reached), ts direction done, lots of data in
+       * tc direction. */
+    } else if (size_tc > size_tc_limit && FLOW_IS_PP_DONE(f, STREAM_TOSERVER) &&
+               !(FLOW_IS_PM_DONE(f, STREAM_TOSERVER)) &&
+               FLOW_IS_PM_DONE(f, STREAM_TOCLIENT) &&
+               FLOW_IS_PP_DONE(f, STREAM_TOCLIENT)) {
+      AppLayerDecoderEventsSetEventRaw(&p->app_layer_events,
+                                       APPLAYER_PROTO_DETECTION_SKIPPED);
+      goto failure;
+
+      /* little data in tc direction, pp done, pm not done (max
+       * depth not reached), tc direction done, lots of data in
+       * ts direction. */
+    } else if (size_ts > size_ts_limit && FLOW_IS_PP_DONE(f, STREAM_TOCLIENT) &&
+               !(FLOW_IS_PM_DONE(f, STREAM_TOCLIENT)) &&
+               FLOW_IS_PM_DONE(f, STREAM_TOSERVER) &&
+               FLOW_IS_PP_DONE(f, STREAM_TOSERVER)) {
+      AppLayerDecoderEventsSetEventRaw(&p->app_layer_events,
+                                       APPLAYER_PROTO_DETECTION_SKIPPED);
+      goto failure;
     }
     return;