proto detection: add limit for one sided sessions

If a session only has data in one direction, like ftp data sessions,
protocol detection will only run in one direction. This led to a
situation where reassembly would hold all the segments as proto
detection was never flagged as complete.

This patch introduces a limit for protocol detection in this case.
If the limit is reached, detection will give up.

diff --git a/src/app-layer-detect-proto.c b/src/app-layer-detect-proto.c
index 19aefd043dc4d3e1e9db97a98adbe6b0c2ce68f7..386f544abdad60ea734abc9556981a477f825553 100644 (file)
--- a/src/app-layer-detect-proto.c
+++ b/src/app-layer-detect-proto.c
@@ -62,7 +62,10 @@
 #include "util-cuda.h"
 #include "util-debug.h"
 
+#include "conf.h"
+
 #define INSPECT_BYTES  32
+#define ASYNC_MAX 75000
 
 /** global app layer detection context */
 AlpProtoDetectCtx alp_proto_ctx;
@@ -82,6 +85,28 @@ void AlpProtoInit(AlpProtoDetectCtx *ctx) {
     ctx->toclient.min_len = INSPECT_BYTES;
     ctx->toserver.min_len = INSPECT_BYTES;
 
+    intmax_t value = 0;
+    if ((ConfGetInt("app-layer.proto-detect.toclient-async-max", &value)) == 1) {
+        if (value >= 0 && value <= 1048576) {
+            ctx->toclient.async_max = (uint32_t)value;
+        } else {
+            ctx->toclient.async_max = (uint32_t)ASYNC_MAX;
+        }
+    } else {
+        ctx->toclient.async_max = (uint32_t)ASYNC_MAX;
+    }
+    if ((ConfGetInt("app-layer.proto-detect.toserver-async-max", &value)) == 1) {
+        if (value >= 0 && value <= 1048576) {
+            ctx->toserver.async_max = (uint32_t)value;
+        } else {
+            ctx->toserver.async_max = (uint32_t)ASYNC_MAX;
+        }
+    } else {
+        ctx->toserver.async_max = (uint32_t)ASYNC_MAX;
+    }
+    SCLogDebug("toclient.async_max %u toserver.async_max %u",
+            ctx->toclient.async_max, ctx->toserver.async_max);
+
     ctx->mpm_pattern_id_store = MpmPatternIdTableInitHash();
 }
 

diff --git a/src/app-layer-detect-proto.h b/src/app-layer-detect-proto.h
index f069f9fa2919b002dab0c5dd97761948baea0a58..54e63bda26fe710ecbed416c671a2d95b46c326d 100644 (file)
--- a/src/app-layer-detect-proto.h
+++ b/src/app-layer-detect-proto.h
@@ -46,12 +46,14 @@ typedef struct AlpProtoDetectDirection_ {
     uint32_t id;
     uint16_t map[ALP_DETECT_MAX];   /**< a mapping between condition id's and
                                          protocol */
-    uint16_t max_len;              /**< max length of all patterns, so we can
+    uint16_t max_len;               /**< max length of all patterns, so we can
                                          limit the search */
-    uint16_t min_len;              /**< min length of all patterns, so we can
+    uint16_t min_len;               /**< min length of all patterns, so we can
                                          tell the stream engine to feed data
                                          to app layer as soon as it has min
                                          size data */
+    uint32_t async_max;             /**< max bytes in this direction while 0 in
+                                         the other, before we give up. */
 } AlpProtoDetectDirection;
 
 typedef struct AlpProtoDetectCtx_ {

diff --git a/src/app-layer.c b/src/app-layer.c
index 194c03520fd1b8df905f4d1ffdfb8ce0bb205e07..0ce187d9259f8d83349af165af350d5321795f40 100644 (file)
--- a/src/app-layer.c
+++ b/src/app-layer.c
@@ -156,6 +156,9 @@ int AppLayerHandleTCPData(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx,
         alproto_otherdir = &f->alproto_ts;
         dir = 1;
     }
+    SCLogDebug("dir %u alproto %u alproto_other_dir %u",
+            dir, *alproto, *alproto_otherdir);
+    //PrintRawDataFp(stdout, data, data_len);
 
     /* if we don't know the proto yet and we have received a stream
      * initializer message, we run proto detection.
@@ -187,6 +190,7 @@ int AppLayerHandleTCPData(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx,
         *alproto = AppLayerDetectGetProto(&alp_proto_ctx, dp_ctx, f,
                                           data, data_len, flags, IPPROTO_TCP);
         PACKET_PROFILING_APP_PD_END(dp_ctx);
+        SCLogDebug("alproto %u", *alproto);
 
         if (*alproto != ALPROTO_UNKNOWN) {
             if (*alproto_otherdir != ALPROTO_UNKNOWN && *alproto_otherdir != *alproto) {
@@ -204,6 +208,9 @@ int AppLayerHandleTCPData(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx,
             }
 
             f->alproto = *alproto;
+            SCLogDebug("calling StreamTcpSetStreamFlagAppProtoDetectionCompleted "
+                    "on stream %p (%s)", stream, (stream == &ssn->client) ?
+                    "ssn->client" : "ssn->server");
             StreamTcpSetStreamFlagAppProtoDetectionCompleted(stream);
 
             /* if we have seen data from the other direction first, send
@@ -215,6 +222,7 @@ int AppLayerHandleTCPData(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx,
              * will now call shortly for the opposing direction. */
             if ((ssn->data_first_seen_dir & (STREAM_TOSERVER | STREAM_TOCLIENT)) &&
                 !(flags & ssn->data_first_seen_dir)) {
+                SCLogDebug("entering opposing dir hack");
                 TcpStream *opposing_stream = NULL;
                 if (stream == &ssn->client) {
                     opposing_stream = &ssn->server;
@@ -260,8 +268,11 @@ int AppLayerHandleTCPData(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx,
                         p->flowflags |= FLOW_PKT_TOSERVER;
                     }
                 }
+                SCLogDebug("ret %d", ret);
                 if (ret < 0) {
                     FlowSetSessionNoApplayerInspectionFlag(f);
+                    SCLogDebug("calling StreamTcpSetStreamFlagAppProtoDetectionCompleted "
+                            "on both streams");
                     StreamTcpSetStreamFlagAppProtoDetectionCompleted(&ssn->client);
                     StreamTcpSetStreamFlagAppProtoDetectionCompleted(&ssn->server);
                     r = -1;
@@ -292,6 +303,8 @@ int AppLayerHandleTCPData(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx,
                     AppLayerDecoderEventsSetEventRaw(p->app_layer_events,
                                                      APPLAYER_WRONG_DIRECTION_FIRST_DATA);
                     FlowSetSessionNoApplayerInspectionFlag(f);
+                    SCLogDebug("calling StreamTcpSetStreamFlagAppProtoDetectionCompleted "
+                            "on both streams");
                     StreamTcpSetStreamFlagAppProtoDetectionCompleted(&ssn->server);
                     StreamTcpSetStreamFlagAppProtoDetectionCompleted(&ssn->client);
                     /* Set a value that is neither STREAM_TOSERVER, nor STREAM_TOCLIENT */
@@ -328,6 +341,7 @@ int AppLayerHandleTCPData(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx,
             PACKET_PROFILING_APP_END(dp_ctx, *alproto);
             f->data_al_so_far[dir] = 0;
         } else {
+            SCLogDebug("alproto == ALPROTO_UNKNOWN (%u)", *alproto);
             if (*alproto_otherdir != ALPROTO_UNKNOWN) {
                 /* this would handle this test case -
                  * http parser which says it wants to see toserver data first only.
@@ -350,6 +364,8 @@ int AppLayerHandleTCPData(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx,
                 {
                     r = -1;
                     FlowSetSessionNoApplayerInspectionFlag(f);
+                    SCLogDebug("calling StreamTcpSetStreamFlagAppProtoDetectionCompleted "
+                            "on both streams");
                     StreamTcpSetStreamFlagAppProtoDetectionCompleted(&ssn->server);
                     StreamTcpSetStreamFlagAppProtoDetectionCompleted(&ssn->client);
                     goto end;
@@ -365,14 +381,48 @@ int AppLayerHandleTCPData(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx,
                 if (FLOW_IS_PM_DONE(f, flags) && FLOW_IS_PP_DONE(f, flags)) {
                     AppLayerDecoderEventsSetEventRaw(p->app_layer_events,
                                                      APPLAYER_DETECT_PROTOCOL_ONLY_ONE_DIRECTION);
+                    SCLogDebug("calling StreamTcpSetStreamFlagAppProtoDetectionCompleted "
+                            "on stream %p (%s)", stream, (stream == &ssn->client) ?
+                            "ssn->client" : "ssn->server");
                     StreamTcpSetStreamFlagAppProtoDetectionCompleted(stream);
                     f->data_al_so_far[dir] = 0;
                 } else {
                     f->data_al_so_far[dir] = data_len;
+                    SCLogDebug("data_len %u stored in flow for dir %u", data_len, dir);
                 }
             } else {
+
+                SCLogDebug("both unknown FLOW_IS_PM_DONE(f, STREAM_TOSERVER) %s "
+                                        "FLOW_IS_PP_DONE(f, STREAM_TOSERVER) %s "
+                                        "FLOW_IS_PM_DONE(f, STREAM_TOCLIENT) %s "
+                                        "FLOW_IS_PP_DONE(f, STREAM_TOCLIENT) %s,"
+                                        " stream ts %u stream tc %u",
+                        FLOW_IS_PM_DONE(f, STREAM_TOSERVER)?"true":"false",
+                        FLOW_IS_PP_DONE(f, STREAM_TOSERVER)?"true":"false",
+                        FLOW_IS_PM_DONE(f, STREAM_TOCLIENT)?"true":"false",
+                        FLOW_IS_PP_DONE(f, STREAM_TOCLIENT)?"true":"false",
+                        StreamTcpGetStreamSize(&ssn->client), StreamTcpGetStreamSize(&ssn->server));
+
+                int flow_done = 0;
                 if (FLOW_IS_PM_DONE(f, STREAM_TOSERVER) && FLOW_IS_PP_DONE(f, STREAM_TOSERVER) &&
                     FLOW_IS_PM_DONE(f, STREAM_TOCLIENT) && FLOW_IS_PP_DONE(f, STREAM_TOCLIENT)) {
+                    SCLogDebug("proto detection failed for both streams");
+                    flow_done = 1;
+                } else if (FLOW_IS_PM_DONE(f, STREAM_TOSERVER) && FLOW_IS_PP_DONE(f, STREAM_TOSERVER) &&
+                           StreamTcpGetStreamSize(&ssn->server) == 0 &&
+                           StreamTcpGetStreamSize(&ssn->client) > alp_proto_ctx.toserver.async_max) {
+                    SCLogDebug("%u bytes toserver and no proto, no data to "
+                               "client, giving up", alp_proto_ctx.toserver.async_max);
+                    flow_done = 1;
+                } else if (FLOW_IS_PM_DONE(f, STREAM_TOCLIENT) && FLOW_IS_PP_DONE(f, STREAM_TOCLIENT) &&
+                           StreamTcpGetStreamSize(&ssn->client) == 0 &&
+                           StreamTcpGetStreamSize(&ssn->server) > alp_proto_ctx.toclient.async_max) {
+                    SCLogDebug("%u bytes toclient and no proto, no data to "
+                               "server, giving up", alp_proto_ctx.toclient.async_max);
+                    flow_done = 1;
+                }
+
+                if (flow_done) {
                     FlowSetSessionNoApplayerInspectionFlag(f);
                     StreamTcpSetStreamFlagAppProtoDetectionCompleted(&ssn->server);
                     StreamTcpSetStreamFlagAppProtoDetectionCompleted(&ssn->client);

diff --git a/src/stream-tcp.c b/src/stream-tcp.c
index cc3ed39866404f8f30648707b8cc2958d0943b2a..50b2d6536cd86b073346995bb5b55e7cd45b61f3 100644 (file)
--- a/src/stream-tcp.c
+++ b/src/stream-tcp.c
@@ -701,6 +701,19 @@ void StreamTcpSetOSPolicy(TcpStream *stream, Packet *p)
 
 }
 
+/**
+ *  \brief get the size of a stream
+ *
+ *  \note this just calculates the diff between isn and last_ack
+ *        and will not consider sequence wrap arounds (streams
+ *        bigger than 4gb).
+ *
+ *  \retval size stream size
+ */
+uint32_t StreamTcpGetStreamSize(TcpStream *stream) {
+    return (stream->last_ack - stream->isn - 1);
+}
+
 /**
  *  \brief macro to update last_ack only if the new value is higher
  *

diff --git a/src/stream-tcp.h b/src/stream-tcp.h
index 0f4ead0ae0f7d19688a86a9be59500bb8240066c..87bfcca9e187bbd78389c3a75ea02fb3f5e8f579 100644 (file)
--- a/src/stream-tcp.h
+++ b/src/stream-tcp.h
@@ -206,6 +206,7 @@ TmEcode StreamTcpThreadDeinit(ThreadVars *tv, void *data);
 int StreamTcpPacket (ThreadVars *tv, Packet *p, StreamTcpThread *stt,
                      PacketQueue *pq);
 void StreamTcpSessionClear(void *ssnptr);
+uint32_t StreamTcpGetStreamSize(TcpStream *stream);
 
 #endif /* __STREAM_TCP_H__ */