From: Victor Julien Date: Fri, 21 Jan 2022 14:20:51 +0000 (+0100) Subject: frames: support UDP frames X-Git-Tag: suricata-7.0.0-beta1~921 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=c96d22e8a10c9e0b392f75768ea1efd6d24298ad;p=thirdparty%2Fsuricata.git frames: support UDP frames UDP frames point to the UDP packet payloads. The frames are removed after each packet. Ticket: #4983. --- diff --git a/src/app-layer-frames.c b/src/app-layer-frames.c index 87c0fdac50..bc97b2d7de 100644 --- a/src/app-layer-frames.c +++ b/src/app-layer-frames.c @@ -391,20 +391,30 @@ static void FrameFreeSingleFrame(Frames *frames, Frame *r) FrameClean(r); } -void FramesFree(Frames *frames) +static void FramesClear(Frames *frames) { BUG_ON(frames == NULL); + SCLogDebug("frames %u", frames->cnt); for (uint16_t i = 0; i < frames->cnt; i++) { if (i < FRAMES_STATIC_CNT) { Frame *r = &frames->sframes[i]; + SCLogDebug("removing frame %p", r); FrameFreeSingleFrame(frames, r); } else { const uint16_t o = i - FRAMES_STATIC_CNT; Frame *r = &frames->dframes[o]; + SCLogDebug("removing frame %p", r); FrameFreeSingleFrame(frames, r); } } + frames->cnt = 0; +} + +void FramesFree(Frames *frames) +{ + BUG_ON(frames == NULL); + FramesClear(frames); SCFree(frames->dframes); frames->dframes = NULL; } @@ -419,7 +429,7 @@ Frame *AppLayerFrameNewByPointer(Flow *f, const StreamSlice *stream_slice, /* workarounds for many (unit|fuzz)tests not handling TCP data properly */ #if defined(UNITTESTS) || defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) - if (f->protoctx == NULL) + if (f->proto == IPPROTO_TCP && f->protoctx == NULL) return NULL; if (frame_start < stream_slice->input || frame_start >= stream_slice->input + stream_slice->input_len) @@ -427,8 +437,7 @@ Frame *AppLayerFrameNewByPointer(Flow *f, const StreamSlice *stream_slice, #endif BUG_ON(frame_start < stream_slice->input); BUG_ON(stream_slice->input == NULL); - BUG_ON(f->proto != IPPROTO_TCP); - BUG_ON(f->protoctx == NULL); + BUG_ON(f->proto == IPPROTO_TCP && f->protoctx == NULL); ptrdiff_t ptr_offset = frame_start - stream_slice->input; #ifdef DEBUG @@ -464,6 +473,29 @@ Frame *AppLayerFrameNewByPointer(Flow *f, const StreamSlice *stream_slice, return r; } +static Frame *AppLayerFrameUdp(Flow *f, const StreamSlice *stream_slice, + const uint32_t frame_start_rel, const int64_t len, int dir, uint8_t frame_type) +{ + BUG_ON(f->proto != IPPROTO_UDP); + + FramesContainer *frames_container = AppLayerFramesSetupContainer(f); + if (frames_container == NULL) + return NULL; + + Frames *frames; + if (dir == 0) { + frames = &frames_container->toserver; + } else { + frames = &frames_container->toclient; + } + + Frame *r = FrameNew(frames, frame_start_rel, len); + if (r != NULL) { + r->type = frame_type; + } + return r; +} + /** \brief create new frame using a relative offset from the start of the stream slice */ Frame *AppLayerFrameNewByRelativeOffset(Flow *f, const StreamSlice *stream_slice, @@ -471,16 +503,19 @@ Frame *AppLayerFrameNewByRelativeOffset(Flow *f, const StreamSlice *stream_slice { /* workarounds for many (unit|fuzz)tests not handling TCP data properly */ #if defined(UNITTESTS) || defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) - if (f->protoctx == NULL) + if (f->proto == IPPROTO_TCP && f->protoctx == NULL) return NULL; if (stream_slice->input == NULL) return NULL; #endif BUG_ON(stream_slice->input == NULL); - BUG_ON(f->proto != IPPROTO_TCP); - BUG_ON(f->protoctx == NULL); + BUG_ON(f->proto == IPPROTO_TCP && f->protoctx == NULL); BUG_ON(f->alparser == NULL); + if (f->proto == IPPROTO_UDP) { + return AppLayerFrameUdp(f, stream_slice, frame_start_rel, len, dir, frame_type); + } + FramesContainer *frames_container = AppLayerFramesSetupContainer(f); if (frames_container == NULL) return NULL; @@ -545,14 +580,13 @@ Frame *AppLayerFrameNewByAbsoluteOffset(Flow *f, const StreamSlice *stream_slice { /* workarounds for many (unit|fuzz)tests not handling TCP data properly */ #if defined(UNITTESTS) || defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) - if (f->protoctx == NULL) + if (f->proto == IPPROTO_TCP && f->protoctx == NULL) return NULL; if (stream_slice->input == NULL) return NULL; #endif BUG_ON(stream_slice->input == NULL); - BUG_ON(f->proto != IPPROTO_TCP); - BUG_ON(f->protoctx == NULL); + BUG_ON(f->proto == IPPROTO_TCP && f->protoctx == NULL); BUG_ON(f->alparser == NULL); BUG_ON(frame_start < stream_slice->offset); BUG_ON(frame_start - stream_slice->offset >= (uint64_t)INT_MAX); @@ -751,13 +785,25 @@ static void FramePrune(Frames *frames, const TcpStream *stream, const bool eof) void FramesPrune(Flow *f, Packet *p) { - if (f->protoctx == NULL) + if (f->proto == IPPROTO_TCP && f->protoctx == NULL) return; FramesContainer *frames_container = AppLayerFramesGetContainer(f); if (frames_container == NULL) return; Frames *frames; + + if (p->proto == IPPROTO_UDP) { + SCLogDebug("clearing all UDP frames"); + if (PKT_IS_TOSERVER(p)) { + frames = &frames_container->toserver; + } else { + frames = &frames_container->toclient; + } + FramesClear(frames); + return; + } + TcpSession *ssn = f->protoctx; if (ssn->flags & STREAMTCP_FLAG_APP_LAYER_DISABLED) { diff --git a/src/app-layer-parser.c b/src/app-layer-parser.c index 9ba84e0e26..3b04aa94c5 100644 --- a/src/app-layer-parser.c +++ b/src/app-layer-parser.c @@ -197,7 +197,7 @@ FramesContainer *AppLayerFramesGetContainer(Flow *f) FramesContainer *AppLayerFramesSetupContainer(Flow *f) { #ifdef UNITTESTS - if (f == NULL || f->alparser == NULL || f->protoctx == NULL) + if (f == NULL || f->alparser == NULL || (f->proto == IPPROTO_TCP && f->protoctx == NULL)) return NULL; #endif DEBUG_VALIDATE_BUG_ON(f == NULL || f->alparser == NULL); diff --git a/src/detect-engine-frame.c b/src/detect-engine-frame.c index 5ca268aa43..330b084068 100644 --- a/src/detect-engine-frame.c +++ b/src/detect-engine-frame.c @@ -156,6 +156,42 @@ int DetectRunFrameInspectRule(ThreadVars *tv, DetectEngineThreadCtx *det_ctx, co return false; } +/** \internal + * \brief setup buffer based on frame in UDP payload + */ +static InspectionBuffer *DetectFrame2InspectBufferUdp(DetectEngineThreadCtx *det_ctx, + const DetectEngineTransforms *transforms, Packet *p, InspectionBuffer *buffer, + const Frames *frames, const Frame *frame, const int list_id, const uint32_t idx, + const bool first) +{ + DEBUG_VALIDATE_BUG_ON(frame->rel_offset >= p->payload_len); + if (frame->rel_offset >= p->payload_len) + return NULL; + + int frame_len = frame->len != -1 ? frame->len : p->payload_len - frame->rel_offset; + uint8_t ci_flags = DETECT_CI_FLAGS_START; + + if (frame->rel_offset + frame_len > p->payload_len) { + frame_len = p->payload_len - frame->rel_offset; + } else { + ci_flags |= DETECT_CI_FLAGS_END; + } + const uint8_t *data = p->payload + frame->rel_offset; + const uint32_t data_len = frame_len; + + SCLogDebug("packet %" PRIu64 " -> frame %p/%" PRIi64 "/%s rel_offset %" PRIi64 + " type %u len %" PRIi64, + p->pcap_cnt, frame, frame->id, + AppLayerParserGetFrameNameById(p->flow->proto, p->flow->alproto, frame->type), + frame->rel_offset, frame->type, frame->len); + // PrintRawDataFp(stdout, data, MIN(64,data_len)); + + InspectionBufferSetupMulti(buffer, transforms, data, data_len); + buffer->inspect_offset = 0; + buffer->flags = ci_flags; + return buffer; +} + InspectionBuffer *DetectFrame2InspectBuffer(DetectEngineThreadCtx *det_ctx, const DetectEngineTransforms *transforms, Packet *p, const Frames *frames, const Frame *frame, const int list_id, const uint32_t idx, const bool first) @@ -168,6 +204,12 @@ InspectionBuffer *DetectFrame2InspectBuffer(DetectEngineThreadCtx *det_ctx, return buffer; BUG_ON(p->flow == NULL); + + if (p->proto == IPPROTO_UDP) { + return DetectFrame2InspectBufferUdp( + det_ctx, transforms, p, buffer, frames, frame, list_id, idx, first); + } + BUG_ON(p->flow->protoctx == NULL); TcpSession *ssn = p->flow->protoctx; TcpStream *stream; diff --git a/src/detect-frame.c b/src/detect-frame.c index e50a1d0009..6fa75303e6 100644 --- a/src/detect-frame.c +++ b/src/detect-frame.c @@ -64,14 +64,20 @@ static int DetectFrameSetup(DetectEngineCtx *de_ctx, Signature *s, const char *s char value[256] = ""; strlcpy(value, str, sizeof(value)); - if (!(DetectProtoContainsProto(&s->proto, IPPROTO_TCP))) { - SCLogError(SC_ERR_INVALID_RULE_ARGUMENT, "'frame' keyword only supported for TCP"); + const bool is_tcp = DetectProtoContainsProto(&s->proto, IPPROTO_TCP); + const bool is_udp = DetectProtoContainsProto(&s->proto, IPPROTO_UDP); + + if (!(is_tcp || is_udp)) { + SCLogError(SC_ERR_INVALID_RULE_ARGUMENT, "'frame' keyword only supported for TCP and UDP"); return -1; } - int raw_frame_type; + int raw_frame_type = -1; if (AppProtoIsValid(s->alproto)) { - raw_frame_type = AppLayerParserGetFrameIdByName(IPPROTO_TCP, s->alproto, str); + if (is_tcp) + raw_frame_type = AppLayerParserGetFrameIdByName(IPPROTO_TCP, s->alproto, str); + if (is_udp && raw_frame_type < 0) + raw_frame_type = AppLayerParserGetFrameIdByName(IPPROTO_UDP, s->alproto, str); if (raw_frame_type < 0) { char *dot = strchr(value, '.'); if (dot != NULL) @@ -89,7 +95,10 @@ static int DetectFrameSetup(DetectEngineCtx *de_ctx, Signature *s, const char *s if (DetectSignatureSetAppProto(s, keyword_alproto) < 0) return -1; } - raw_frame_type = AppLayerParserGetFrameIdByName(IPPROTO_TCP, s->alproto, val); + if (is_tcp) + raw_frame_type = AppLayerParserGetFrameIdByName(IPPROTO_TCP, s->alproto, val); + if (is_udp && raw_frame_type < 0) + raw_frame_type = AppLayerParserGetFrameIdByName(IPPROTO_UDP, s->alproto, val); if (raw_frame_type < 0) { SCLogError(SC_ERR_INVALID_RULE_ARGUMENT, "unknown frame '%s' for protocol '%s'", val, proto); @@ -115,7 +124,10 @@ static int DetectFrameSetup(DetectEngineCtx *de_ctx, Signature *s, const char *s if (DetectSignatureSetAppProto(s, alproto) < 0) return -1; - raw_frame_type = AppLayerParserGetFrameIdByName(IPPROTO_TCP, s->alproto, val); + if (is_tcp) + raw_frame_type = AppLayerParserGetFrameIdByName(IPPROTO_TCP, s->alproto, val); + if (is_udp && raw_frame_type < 0) + raw_frame_type = AppLayerParserGetFrameIdByName(IPPROTO_UDP, s->alproto, val); if (raw_frame_type < 0) { SCLogError(SC_ERR_INVALID_RULE_ARGUMENT, "unknown frame '%s' for protocol '%s'", val, proto); diff --git a/src/detect.c b/src/detect.c index 51b8de23b3..6a0d034ba9 100644 --- a/src/detect.c +++ b/src/detect.c @@ -145,6 +145,8 @@ static void DetectRun(ThreadVars *th_v, DetectRunFrames(th_v, de_ctx, det_ctx, p, pflow, &scratch); // PACKET_PROFILING_DETECT_END(p, PROF_DETECT_TX); } + } else if (p->proto == IPPROTO_UDP) { + DetectRunFrames(th_v, de_ctx, det_ctx, p, pflow, &scratch); } PACKET_PROFILING_DETECT_START(p, PROF_DETECT_TX); diff --git a/src/flow-worker.c b/src/flow-worker.c index 81c1cdd98c..0a781cca97 100644 --- a/src/flow-worker.c +++ b/src/flow-worker.c @@ -574,6 +574,8 @@ static TmEcode FlowWorker(ThreadVars *tv, Packet *p, void *data) StreamTcpPruneSession(p->flow, p->flowflags & FLOW_PKT_TOSERVER ? STREAM_TOSERVER : STREAM_TOCLIENT); FLOWWORKER_PROFILING_END(p, PROFILE_FLOWWORKER_TCPPRUNE); + } else if (p->proto == IPPROTO_UDP) { + FramesPrune(p->flow, p); } /* run tx cleanup last */ diff --git a/src/output-json-alert.c b/src/output-json-alert.c index a9307921da..c14cfa43ed 100644 --- a/src/output-json-alert.c +++ b/src/output-json-alert.c @@ -592,27 +592,37 @@ static void AlertAddFiles(const Packet *p, JsonBuilder *jb, const uint64_t tx_id static void AlertAddFrame(const Packet *p, JsonBuilder *jb, const int64_t frame_id) { - if (p->flow == NULL || p->flow->protoctx == NULL) + if (p->flow == NULL || (p->proto == IPPROTO_TCP && p->flow->protoctx == NULL)) return; FramesContainer *frames_container = AppLayerFramesGetContainer(p->flow); if (frames_container == NULL) return; - Frames *frames; - TcpSession *ssn = p->flow->protoctx; - TcpStream *stream; - if (PKT_IS_TOSERVER(p)) { - stream = &ssn->client; - frames = &frames_container->toserver; - } else { - stream = &ssn->server; - frames = &frames_container->toclient; + Frames *frames = NULL; + TcpStream *stream = NULL; + if (p->proto == IPPROTO_TCP) { + TcpSession *ssn = p->flow->protoctx; + if (PKT_IS_TOSERVER(p)) { + stream = &ssn->client; + frames = &frames_container->toserver; + } else { + stream = &ssn->server; + frames = &frames_container->toclient; + } + } else if (p->proto == IPPROTO_UDP) { + if (PKT_IS_TOSERVER(p)) { + frames = &frames_container->toserver; + } else { + frames = &frames_container->toclient; + } } - Frame *frame = FrameGetById(frames, frame_id); - if (frame != NULL) { - FrameJsonLogOneFrame(frame, p->flow, stream, p, jb); + if (frames) { + Frame *frame = FrameGetById(frames, frame_id); + if (frame != NULL) { + FrameJsonLogOneFrame(frame, p->flow, stream, p, jb); + } } } diff --git a/src/output-json-frame.c b/src/output-json-frame.c index 8937fbddbc..4cbde9ce00 100644 --- a/src/output-json-frame.c +++ b/src/output-json-frame.c @@ -126,7 +126,7 @@ static void PayloadAsHex(const uint8_t *data, uint32_t data_len, char *str, size } #endif -static void FrameAddPayload(JsonBuilder *js, const TcpStream *stream, const Frame *frame) +static void FrameAddPayloadTCP(JsonBuilder *js, const TcpStream *stream, const Frame *frame) { uint32_t sb_data_len = 0; const uint8_t *data = NULL; @@ -178,44 +178,123 @@ static void FrameAddPayload(JsonBuilder *js, const TcpStream *stream, const Fram #endif } +static void FrameAddPayloadUDP(JsonBuilder *js, const Packet *p, const Frame *frame) +{ + DEBUG_VALIDATE_BUG_ON(frame->rel_offset >= p->payload_len); + if (frame->rel_offset >= p->payload_len) + return; + + int frame_len = frame->len != -1 ? frame->len : p->payload_len - frame->rel_offset; + + if (frame->rel_offset + frame_len > p->payload_len) { + frame_len = p->payload_len - frame->rel_offset; + JB_SET_FALSE(js, "complete"); + } else { + JB_SET_TRUE(js, "complete"); + } + const uint8_t *data = p->payload + frame->rel_offset; + const uint32_t data_len = frame_len; + + const uint32_t log_data_len = MIN(data_len, 256); + jb_set_base64(js, "payload", data, log_data_len); + + uint8_t printable_buf[log_data_len + 1]; + uint32_t o = 0; + PrintStringsToBuffer(printable_buf, &o, log_data_len + 1, data, log_data_len); + printable_buf[log_data_len] = '\0'; + jb_set_string(js, "payload_printable", (char *)printable_buf); +#if 0 + char pretty_buf[data_len * 4 + 1]; + pretty_buf[0] = '\0'; + PayloadAsHex(data, data_len, pretty_buf, data_len * 4 + 1); + jb_set_string(js, "payload_hex", pretty_buf); +#endif +} + // TODO separate between stream_offset and frame_offset void FrameJsonLogOneFrame(const Frame *frame, const Flow *f, const TcpStream *stream, const Packet *p, JsonBuilder *jb) { - int64_t abs_offset = frame->rel_offset + (int64_t)STREAM_BASE_OFFSET(stream); - jb_open_object(jb, "frame"); jb_set_string(jb, "type", AppLayerParserGetFrameNameById(f->proto, f->alproto, frame->type)); jb_set_uint(jb, "id", frame->id); - jb_set_uint(jb, "stream_offset", (uint64_t)abs_offset); + jb_set_string(jb, "direction", PKT_IS_TOSERVER(p) ? "toserver" : "toclient"); + + if (f->proto == IPPROTO_TCP) { + DEBUG_VALIDATE_BUG_ON(stream == NULL); + int64_t abs_offset = frame->rel_offset + (int64_t)STREAM_BASE_OFFSET(stream); + jb_set_uint(jb, "stream_offset", (uint64_t)abs_offset); - if (frame->len < 0) { - uint64_t usable = StreamTcpGetUsable(stream, true); - uint64_t len = usable - abs_offset; - jb_set_uint(jb, "length", len); + if (f->proto == IPPROTO_TCP && frame->len < 0) { + uint64_t usable = StreamTcpGetUsable(stream, true); + uint64_t len = usable - abs_offset; + jb_set_uint(jb, "length", len); + } else { + jb_set_uint(jb, "length", frame->len); + } + FrameAddPayloadTCP(jb, stream, frame); } else { jb_set_uint(jb, "length", frame->len); + FrameAddPayloadUDP(jb, p, frame); } - jb_set_string(jb, "direction", PKT_IS_TOSERVER(p) ? "toserver" : "toclient"); if (frame->flags & FRAME_FLAG_TX_ID_SET) { jb_set_uint(jb, "tx_id", frame->tx_id); } - FrameAddPayload(jb, stream, frame); jb_close(jb); } +static int FrameJsonUdp( + JsonFrameLogThread *aft, const Packet *p, Flow *f, FramesContainer *frames_container) +{ + FrameJsonOutputCtx *json_output_ctx = aft->json_output_ctx; + + Frames *frames; + if (PKT_IS_TOSERVER(p)) { + frames = &frames_container->toserver; + } else { + frames = &frames_container->toclient; + } + + for (uint32_t idx = 0; idx < frames->cnt; idx++) { + Frame *frame = FrameGetByIndex(frames, idx); + if (frame == NULL || frame->flags & FRAME_FLAG_LOGGED) + continue; + + /* First initialize the address info (5-tuple). */ + JsonAddrInfo addr = json_addr_info_zero; + JsonAddrInfoInit(p, LOG_DIR_PACKET, &addr); + + JsonBuilder *jb = + CreateEveHeader(p, LOG_DIR_PACKET, "frame", &addr, json_output_ctx->eve_ctx); + if (unlikely(jb == NULL)) + return TM_ECODE_OK; + + jb_set_string(jb, "app_proto", AppProtoToString(f->alproto)); + FrameJsonLogOneFrame(frame, p->flow, NULL, p, jb); + OutputJsonBuilderBuffer(jb, aft->ctx); + jb_free(jb); + frame->flags |= FRAME_FLAG_LOGGED; + } + return TM_ECODE_OK; +} + static int FrameJson(ThreadVars *tv, JsonFrameLogThread *aft, const Packet *p) { FrameJsonOutputCtx *json_output_ctx = aft->json_output_ctx; - BUG_ON(p->proto != IPPROTO_TCP); BUG_ON(p->flow == NULL); - BUG_ON(p->flow->protoctx == NULL); FramesContainer *frames_container = AppLayerFramesGetContainer(p->flow); if (frames_container == NULL) return TM_ECODE_OK; + if (p->proto == IPPROTO_UDP) { + return FrameJsonUdp(aft, p, p->flow, frames_container); + } + + BUG_ON(p->proto != IPPROTO_TCP); + BUG_ON(p->flow->protoctx == NULL); + /* TODO can we set these EOF flags once per packet? We have them in detect, tx, file, filedata, * etc */ const bool last_pseudo = (p->flowflags & FLOW_PKT_LAST_PSEUDO) != 0; @@ -288,7 +367,7 @@ static int JsonFrameLogCondition(ThreadVars *tv, const Packet *p) if (p->flow == NULL || p->flow->alproto == ALPROTO_UNKNOWN) return FALSE; - if (p->proto == IPPROTO_TCP && p->flow->alparser != NULL) { + if ((p->proto == IPPROTO_TCP || p->proto == IPPROTO_UDP) && p->flow->alparser != NULL) { FramesContainer *frames_container = AppLayerFramesGetContainer(p->flow); if (frames_container == NULL) return FALSE;