1 /* Copyright (C) 2007-2020 Open Information Security Foundation
3 * You can copy, redistribute or modify this Program under the terms of
4 * the GNU General Public License version 2 as published by the Free
7 * This program is distributed in the hope that it will be useful,
8 * but WITHOUT ANY WARRANTY; without even the implied warranty of
9 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10 * GNU General Public License for more details.
12 * You should have received a copy of the GNU General Public License
13 * version 2 along with this program; if not, write to the Free Software
14 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
21 * \author Victor Julien <victor@inliniac.net>
23 * Generic App-layer parsing functions.
26 #include "suricata-common.h"
28 #include "util-unittest.h"
32 #include "util-print.h"
33 #include "util-pool.h"
35 #include "flow-util.h"
36 #include "flow-private.h"
38 #include "detect-engine-state.h"
39 #include "detect-engine-port.h"
41 #include "stream-tcp.h"
42 #include "stream-tcp-private.h"
44 #include "stream-tcp-reassemble.h"
46 #include "app-layer.h"
47 #include "app-layer-protos.h"
48 #include "app-layer-parser.h"
49 #include "app-layer-smb.h"
50 #include "app-layer-htp.h"
51 #include "app-layer-ftp.h"
52 #include "app-layer-ssl.h"
53 #include "app-layer-ssh.h"
54 #include "app-layer-smtp.h"
55 #include "app-layer-modbus.h"
56 #include "app-layer-enip.h"
57 #include "app-layer-dnp3.h"
58 #include "app-layer-nfs-tcp.h"
59 #include "app-layer-nfs-udp.h"
60 #include "app-layer-ntp.h"
61 #include "app-layer-tftp.h"
62 #include "app-layer-ike.h"
63 #include "app-layer-krb5.h"
64 #include "app-layer-snmp.h"
65 #include "app-layer-sip.h"
66 #include "app-layer-rfb.h"
67 #include "app-layer-mqtt.h"
68 #include "app-layer-template.h"
69 #include "app-layer-template-rust.h"
70 #include "app-layer-rdp.h"
71 #include "app-layer-http2.h"
76 #include "util-debug.h"
77 #include "decode-events.h"
78 #include "util-unittest-helper.h"
79 #include "util-validate.h"
85 struct AppLayerParserThreadCtx_
{
86 void *alproto_local_storage
[FLOW_PROTO_MAX
][ALPROTO_MAX
];
91 * \brief App layer protocol parser context.
93 typedef struct AppLayerParserProtoCtx_
95 /* 0 - to_server, 1 - to_client. */
96 AppLayerParserFPtr Parser
[2];
98 uint32_t logger_bits
; /**< registered loggers for this proto */
100 void *(*StateAlloc
)(void *, AppProto
);
101 void (*StateFree
)(void *);
102 void (*StateTransactionFree
)(void *, uint64_t);
103 void *(*LocalStorageAlloc
)(void);
104 void (*LocalStorageFree
)(void *);
106 void (*Truncate
)(void *, uint8_t);
107 FileContainer
*(*StateGetFiles
)(void *, uint8_t);
108 AppLayerDecoderEvents
*(*StateGetEvents
)(void *);
110 int (*StateGetProgress
)(void *alstate
, uint8_t direction
);
111 uint64_t (*StateGetTxCnt
)(void *alstate
);
112 void *(*StateGetTx
)(void *alstate
, uint64_t tx_id
);
113 AppLayerGetTxIteratorFunc StateGetTxIterator
;
116 int (*StateGetEventInfoById
)(int event_id
, const char **event_name
,
117 AppLayerEventType
*event_type
);
118 int (*StateGetEventInfo
)(const char *event_name
,
119 int *event_id
, AppLayerEventType
*event_type
);
121 AppLayerTxData
*(*GetTxData
)(void *tx
);
122 bool (*ApplyTxConfig
)(void *state
, void *tx
, int mode
, AppLayerTxConfig
);
124 void (*SetStreamDepthFlag
)(void *tx
, uint8_t flags
);
126 /* each app-layer has its own value */
127 uint32_t stream_depth
;
129 /* Indicates the direction the parser is ready to see the data
130 * the first time for a flow. Values accepted -
131 * STREAM_TOSERVER, STREAM_TOCLIENT */
132 uint8_t first_data_dir
;
134 /* Option flags such as supporting gaps or not. */
135 uint32_t option_flags
;
136 /* coccinelle: AppLayerParserProtoCtx:option_flags:APP_LAYER_PARSER_OPT_ */
138 uint32_t internal_flags
;
139 /* coccinelle: AppLayerParserProtoCtx:internal_flags:APP_LAYER_PARSER_INT_ */
142 void (*RegisterUnittests
)(void);
144 } AppLayerParserProtoCtx
;
146 typedef struct AppLayerParserCtx_
{
147 AppLayerParserProtoCtx ctxs
[FLOW_PROTO_MAX
][ALPROTO_MAX
];
150 struct AppLayerParserState_
{
151 /* coccinelle: AppLayerParserState:flags:APP_LAYER_PARSER_ */
154 /* Indicates the current transaction that is being inspected.
155 * We have a var per direction. */
156 uint64_t inspect_id
[2];
157 /* Indicates the current transaction being logged. Unlike inspect_id,
158 * we don't need a var per direction since we don't log a transaction
159 * unless we have the entire transaction. */
164 /* Used to store decoder events. */
165 AppLayerDecoderEvents
*decoder_events
;
169 void UTHAppLayerParserStateGetIds(void *ptr
, uint64_t *i1
, uint64_t *i2
, uint64_t *log
, uint64_t *min
)
171 struct AppLayerParserState_
*s
= ptr
;
172 *i1
= s
->inspect_id
[0];
173 *i2
= s
->inspect_id
[1];
179 /* Static global version of the parser context.
180 * Post 2.0 let's look at changing this to move it out to app-layer.c. */
181 static AppLayerParserCtx alp_ctx
;
183 int AppLayerParserProtoIsRegistered(uint8_t ipproto
, AppProto alproto
)
185 uint8_t ipproto_map
= FlowGetProtoMapping(ipproto
);
187 return (alp_ctx
.ctxs
[ipproto_map
][alproto
].StateAlloc
!= NULL
) ? 1 : 0;
190 AppLayerParserState
*AppLayerParserStateAlloc(void)
194 AppLayerParserState
*pstate
= (AppLayerParserState
*)SCMalloc(sizeof(*pstate
));
197 memset(pstate
, 0, sizeof(*pstate
));
200 SCReturnPtr(pstate
, "AppLayerParserState");
203 void AppLayerParserStateFree(AppLayerParserState
*pstate
)
207 if (pstate
->decoder_events
!= NULL
)
208 AppLayerDecoderEventsFreeEvents(&pstate
->decoder_events
);
214 int AppLayerParserSetup(void)
217 memset(&alp_ctx
, 0, sizeof(alp_ctx
));
221 void AppLayerParserPostStreamSetup(void)
223 AppProto alproto
= 0;
226 /* lets set a default value for stream_depth */
227 for (flow_proto
= 0; flow_proto
< FLOW_PROTO_DEFAULT
; flow_proto
++) {
228 for (alproto
= 0; alproto
< ALPROTO_MAX
; alproto
++) {
229 if (!(alp_ctx
.ctxs
[flow_proto
][alproto
].internal_flags
&
230 APP_LAYER_PARSER_INT_STREAM_DEPTH_SET
)) {
231 alp_ctx
.ctxs
[flow_proto
][alproto
].stream_depth
=
232 stream_config
.reassembly_depth
;
238 int AppLayerParserDeSetup(void)
248 AppLayerParserThreadCtx
*AppLayerParserThreadCtxAlloc(void)
252 AppProto alproto
= 0;
254 AppLayerParserThreadCtx
*tctx
;
256 tctx
= SCMalloc(sizeof(*tctx
));
259 memset(tctx
, 0, sizeof(*tctx
));
261 for (flow_proto
= 0; flow_proto
< FLOW_PROTO_DEFAULT
; flow_proto
++) {
262 for (alproto
= 0; alproto
< ALPROTO_MAX
; alproto
++) {
263 uint8_t ipproto
= FlowGetReverseProtoMapping(flow_proto
);
265 tctx
->alproto_local_storage
[flow_proto
][alproto
] =
266 AppLayerParserGetProtocolParserLocalStorage(ipproto
, alproto
);
271 SCReturnPtr(tctx
, "void *");
274 void AppLayerParserThreadCtxFree(AppLayerParserThreadCtx
*tctx
)
278 AppProto alproto
= 0;
281 for (flow_proto
= 0; flow_proto
< FLOW_PROTO_DEFAULT
; flow_proto
++) {
282 for (alproto
= 0; alproto
< ALPROTO_MAX
; alproto
++) {
283 uint8_t ipproto
= FlowGetReverseProtoMapping(flow_proto
);
285 AppLayerParserDestroyProtocolParserLocalStorage(ipproto
, alproto
,
286 tctx
->alproto_local_storage
[flow_proto
][alproto
]);
294 /** \brief check if a parser is enabled in the config
295 * Returns enabled always if: were running unittests
297 int AppLayerParserConfParserEnabled(const char *ipproto
,
298 const char *alproto_name
)
307 if (RunmodeIsUnittests())
310 r
= snprintf(param
, sizeof(param
), "%s%s%s", "app-layer.protocols.",
311 alproto_name
, ".enabled");
313 FatalError(SC_ERR_FATAL
, "snprintf failure.");
314 } else if (r
> (int)sizeof(param
)) {
315 FatalError(SC_ERR_FATAL
, "buffer not big enough to write param.");
318 node
= ConfGetNode(param
);
320 SCLogDebug("Entry for %s not found.", param
);
321 r
= snprintf(param
, sizeof(param
), "%s%s%s%s%s", "app-layer.protocols.",
322 alproto_name
, ".", ipproto
, ".enabled");
324 FatalError(SC_ERR_FATAL
, "snprintf failure.");
325 } else if (r
> (int)sizeof(param
)) {
326 FatalError(SC_ERR_FATAL
, "buffer not big enough to write param.");
329 node
= ConfGetNode(param
);
331 SCLogDebug("Entry for %s not found.", param
);
336 if (ConfValIsTrue(node
->val
)) {
338 } else if (ConfValIsFalse(node
->val
)) {
340 } else if (strcasecmp(node
->val
, "detection-only") == 0) {
343 SCLogError(SC_ERR_FATAL
, "Invalid value found for %s.", param
);
350 SCReturnInt(enabled
);
353 /***** Parser related registration *****/
355 int AppLayerParserRegisterParser(uint8_t ipproto
, AppProto alproto
,
357 AppLayerParserFPtr Parser
)
361 alp_ctx
.ctxs
[FlowGetProtoMapping(ipproto
)][alproto
].
362 Parser
[(direction
& STREAM_TOSERVER
) ? 0 : 1] = Parser
;
367 void AppLayerParserRegisterParserAcceptableDataDirection(uint8_t ipproto
, AppProto alproto
,
372 alp_ctx
.ctxs
[FlowGetProtoMapping(ipproto
)][alproto
].first_data_dir
|=
373 (direction
& (STREAM_TOSERVER
| STREAM_TOCLIENT
));
378 void AppLayerParserRegisterOptionFlags(uint8_t ipproto
, AppProto alproto
,
383 alp_ctx
.ctxs
[FlowGetProtoMapping(ipproto
)][alproto
].option_flags
|= flags
;
388 uint32_t AppLayerParserGetOptionFlags(uint8_t protomap
, AppProto alproto
)
391 SCReturnUInt(alp_ctx
.ctxs
[protomap
][alproto
].option_flags
);
394 void AppLayerParserRegisterStateFuncs(uint8_t ipproto
, AppProto alproto
,
395 void *(*StateAlloc
)(void *, AppProto
), void (*StateFree
)(void *))
399 alp_ctx
.ctxs
[FlowGetProtoMapping(ipproto
)][alproto
].StateAlloc
=
401 alp_ctx
.ctxs
[FlowGetProtoMapping(ipproto
)][alproto
].StateFree
=
407 void AppLayerParserRegisterLocalStorageFunc(uint8_t ipproto
, AppProto alproto
,
408 void *(*LocalStorageAlloc
)(void),
409 void (*LocalStorageFree
)(void *))
413 alp_ctx
.ctxs
[FlowGetProtoMapping(ipproto
)][alproto
].LocalStorageAlloc
=
415 alp_ctx
.ctxs
[FlowGetProtoMapping(ipproto
)][alproto
].LocalStorageFree
=
421 void AppLayerParserRegisterGetFilesFunc(uint8_t ipproto
, AppProto alproto
,
422 FileContainer
*(*StateGetFiles
)(void *, uint8_t))
426 alp_ctx
.ctxs
[FlowGetProtoMapping(ipproto
)][alproto
].StateGetFiles
=
432 void AppLayerParserRegisterGetEventsFunc(uint8_t ipproto
, AppProto alproto
,
433 AppLayerDecoderEvents
*(*StateGetEvents
)(void *))
437 alp_ctx
.ctxs
[FlowGetProtoMapping(ipproto
)][alproto
].StateGetEvents
=
443 void AppLayerParserRegisterLoggerBits(uint8_t ipproto
, AppProto alproto
, LoggerId bits
)
447 alp_ctx
.ctxs
[FlowGetProtoMapping(ipproto
)][alproto
].logger_bits
= bits
;
452 void AppLayerParserRegisterLogger(uint8_t ipproto
, AppProto alproto
)
456 alp_ctx
.ctxs
[FlowGetProtoMapping(ipproto
)][alproto
].logger
= true;
461 void AppLayerParserRegisterTruncateFunc(uint8_t ipproto
, AppProto alproto
,
462 void (*Truncate
)(void *, uint8_t))
466 alp_ctx
.ctxs
[FlowGetProtoMapping(ipproto
)][alproto
].Truncate
= Truncate
;
471 void AppLayerParserRegisterGetStateProgressFunc(uint8_t ipproto
, AppProto alproto
,
472 int (*StateGetProgress
)(void *alstate
, uint8_t direction
))
476 alp_ctx
.ctxs
[FlowGetProtoMapping(ipproto
)][alproto
].
477 StateGetProgress
= StateGetProgress
;
482 void AppLayerParserRegisterTxFreeFunc(uint8_t ipproto
, AppProto alproto
,
483 void (*StateTransactionFree
)(void *, uint64_t))
487 alp_ctx
.ctxs
[FlowGetProtoMapping(ipproto
)][alproto
].
488 StateTransactionFree
= StateTransactionFree
;
493 void AppLayerParserRegisterGetTxCnt(uint8_t ipproto
, AppProto alproto
,
494 uint64_t (*StateGetTxCnt
)(void *alstate
))
498 alp_ctx
.ctxs
[FlowGetProtoMapping(ipproto
)][alproto
].
499 StateGetTxCnt
= StateGetTxCnt
;
504 void AppLayerParserRegisterGetTx(uint8_t ipproto
, AppProto alproto
,
505 void *(StateGetTx
)(void *alstate
, uint64_t tx_id
))
509 alp_ctx
.ctxs
[FlowGetProtoMapping(ipproto
)][alproto
].
510 StateGetTx
= StateGetTx
;
515 void AppLayerParserRegisterGetTxIterator(uint8_t ipproto
, AppProto alproto
,
516 AppLayerGetTxIteratorFunc Func
)
519 alp_ctx
.ctxs
[FlowGetProtoMapping(ipproto
)][alproto
].StateGetTxIterator
= Func
;
523 void AppLayerParserRegisterStateProgressCompletionStatus(
524 AppProto alproto
, const int ts
, const int tc
)
528 BUG_ON(!AppProtoIsValid(alproto
));
529 BUG_ON(alp_ctx
.ctxs
[FLOW_PROTO_DEFAULT
][alproto
].complete_ts
!= 0 &&
530 alp_ctx
.ctxs
[FLOW_PROTO_DEFAULT
][alproto
].complete_ts
!= ts
);
531 BUG_ON(alp_ctx
.ctxs
[FLOW_PROTO_DEFAULT
][alproto
].complete_tc
!= 0 &&
532 alp_ctx
.ctxs
[FLOW_PROTO_DEFAULT
][alproto
].complete_tc
!= tc
);
534 alp_ctx
.ctxs
[FLOW_PROTO_DEFAULT
][alproto
].complete_ts
= ts
;
535 alp_ctx
.ctxs
[FLOW_PROTO_DEFAULT
][alproto
].complete_tc
= tc
;
538 void AppLayerParserRegisterGetEventInfoById(uint8_t ipproto
, AppProto alproto
,
539 int (*StateGetEventInfoById
)(int event_id
, const char **event_name
,
540 AppLayerEventType
*event_type
))
544 alp_ctx
.ctxs
[FlowGetProtoMapping(ipproto
)][alproto
].
545 StateGetEventInfoById
= StateGetEventInfoById
;
550 void AppLayerParserRegisterGetEventInfo(uint8_t ipproto
, AppProto alproto
,
551 int (*StateGetEventInfo
)(const char *event_name
, int *event_id
,
552 AppLayerEventType
*event_type
))
556 alp_ctx
.ctxs
[FlowGetProtoMapping(ipproto
)][alproto
].
557 StateGetEventInfo
= StateGetEventInfo
;
562 void AppLayerParserRegisterTxDataFunc(uint8_t ipproto
, AppProto alproto
,
563 AppLayerTxData
*(*GetTxData
)(void *tx
))
567 alp_ctx
.ctxs
[FlowGetProtoMapping(ipproto
)][alproto
].GetTxData
= GetTxData
;
572 void AppLayerParserRegisterApplyTxConfigFunc(uint8_t ipproto
, AppProto alproto
,
573 bool (*ApplyTxConfig
)(void *state
, void *tx
, int mode
, AppLayerTxConfig
))
577 alp_ctx
.ctxs
[FlowGetProtoMapping(ipproto
)][alproto
].ApplyTxConfig
= ApplyTxConfig
;
582 void AppLayerParserRegisterSetStreamDepthFlag(uint8_t ipproto
, AppProto alproto
,
583 void (*SetStreamDepthFlag
)(void *tx
, uint8_t flags
))
587 alp_ctx
.ctxs
[FlowGetProtoMapping(ipproto
)][alproto
].SetStreamDepthFlag
= SetStreamDepthFlag
;
592 /***** Get and transaction functions *****/
594 void *AppLayerParserGetProtocolParserLocalStorage(uint8_t ipproto
, AppProto alproto
)
599 if (alp_ctx
.ctxs
[FlowGetProtoMapping(ipproto
)][alproto
].
600 LocalStorageAlloc
!= NULL
)
602 r
= alp_ctx
.ctxs
[FlowGetProtoMapping(ipproto
)][alproto
].
606 SCReturnPtr(r
, "void *");
609 void AppLayerParserDestroyProtocolParserLocalStorage(uint8_t ipproto
, AppProto alproto
,
614 if (alp_ctx
.ctxs
[FlowGetProtoMapping(ipproto
)][alproto
].
615 LocalStorageFree
!= NULL
)
617 alp_ctx
.ctxs
[FlowGetProtoMapping(ipproto
)][alproto
].
618 LocalStorageFree(local_data
);
624 /** \brief default tx iterator
626 * Used if the app layer parser doesn't register its own iterator.
627 * Simply walks the tx_id space until it finds a tx. Uses 'state' to
628 * keep track of where it left off.
630 * \retval txptr or NULL if no more txs in list
632 static AppLayerGetTxIterTuple
AppLayerDefaultGetTxIterator(
633 const uint8_t ipproto
, const AppProto alproto
,
634 void *alstate
, uint64_t min_tx_id
, uint64_t max_tx_id
,
635 AppLayerGetTxIterState
*state
)
637 uint64_t ustate
= *(uint64_t *)state
;
638 uint64_t tx_id
= MAX(min_tx_id
, ustate
);
639 for ( ; tx_id
< max_tx_id
; tx_id
++) {
640 void *tx_ptr
= AppLayerParserGetTx(ipproto
, alproto
, alstate
, tx_id
);
641 if (tx_ptr
!= NULL
) {
643 *state
= *(AppLayerGetTxIterState
*)&ustate
;
644 AppLayerGetTxIterTuple tuple
= {
647 .has_next
= (tx_id
+ 1 < max_tx_id
),
649 SCLogDebug("tuple: %p/%"PRIu64
"/%s", tuple
.tx_ptr
, tuple
.tx_id
,
650 tuple
.has_next
? "true" : "false");
655 AppLayerGetTxIterTuple no_tuple
= { NULL
, 0, false };
659 AppLayerGetTxIteratorFunc
AppLayerGetTxIterator(const uint8_t ipproto
,
660 const AppProto alproto
)
662 AppLayerGetTxIteratorFunc Func
=
663 alp_ctx
.ctxs
[FlowGetProtoMapping(ipproto
)][alproto
].StateGetTxIterator
;
664 return Func
? Func
: AppLayerDefaultGetTxIterator
;
667 uint64_t AppLayerParserGetTransactionLogId(AppLayerParserState
*pstate
)
671 SCReturnCT((pstate
== NULL
) ? 0 : pstate
->log_id
, "uint64_t");
674 void AppLayerParserSetTransactionLogId(AppLayerParserState
*pstate
, uint64_t tx_id
)
679 pstate
->log_id
= tx_id
;
684 uint64_t AppLayerParserGetTransactionInspectId(AppLayerParserState
*pstate
, uint8_t direction
)
689 SCReturnCT(0ULL, "uint64_t");
691 SCReturnCT(pstate
->inspect_id
[direction
& STREAM_TOSERVER
? 0 : 1], "uint64_t");
694 static inline uint64_t GetTxDetectFlags(AppLayerTxData
*txd
, const uint8_t dir
)
696 uint64_t detect_flags
=
697 (dir
& STREAM_TOSERVER
) ? txd
->detect_flags_ts
: txd
->detect_flags_tc
;
701 static inline void SetTxDetectFlags(AppLayerTxData
*txd
, const uint8_t dir
, const uint64_t detect_flags
)
703 if (dir
& STREAM_TOSERVER
) {
704 txd
->detect_flags_ts
= detect_flags
;
706 txd
->detect_flags_tc
= detect_flags
;
710 static inline uint32_t GetTxLogged(AppLayerTxData
*txd
)
712 return txd
->logged
.flags
;
715 void AppLayerParserSetTransactionInspectId(const Flow
*f
, AppLayerParserState
*pstate
,
716 void *alstate
, const uint8_t flags
,
717 bool tag_txs_as_inspected
)
721 const int direction
= (flags
& STREAM_TOSERVER
) ? 0 : 1;
722 const uint64_t total_txs
= AppLayerParserGetTxCnt(f
, alstate
);
723 uint64_t idx
= AppLayerParserGetTransactionInspectId(pstate
, flags
);
724 const int state_done_progress
= AppLayerParserGetStateProgressCompletionStatus(f
->alproto
, flags
);
725 const uint8_t ipproto
= f
->proto
;
726 const AppProto alproto
= f
->alproto
;
728 AppLayerGetTxIteratorFunc IterFunc
= AppLayerGetTxIterator(ipproto
, alproto
);
729 AppLayerGetTxIterState state
;
730 memset(&state
, 0, sizeof(state
));
732 SCLogDebug("called: %s, tag_txs_as_inspected %s",direction
==0?"toserver":"toclient",
733 tag_txs_as_inspected
?"true":"false");
735 /* mark all txs as inspected if the applayer progress is
736 * at the 'end state'. */
738 AppLayerGetTxIterTuple ires
= IterFunc(ipproto
, alproto
, alstate
, idx
, total_txs
, &state
);
739 if (ires
.tx_ptr
== NULL
)
742 void *tx
= ires
.tx_ptr
;
745 int state_progress
= AppLayerParserGetStateProgress(ipproto
, alproto
, tx
, flags
);
746 if (state_progress
< state_done_progress
)
749 AppLayerTxData
*txd
= AppLayerParserGetTxData(ipproto
, alproto
, tx
);
750 if (txd
&& tag_txs_as_inspected
) {
751 uint64_t detect_flags
= GetTxDetectFlags(txd
, flags
);
752 if ((detect_flags
& APP_LAYER_TX_INSPECTED_FLAG
) == 0) {
753 detect_flags
|= APP_LAYER_TX_INSPECTED_FLAG
;
754 SetTxDetectFlags(txd
, flags
, detect_flags
);
755 SCLogDebug("%p/%"PRIu64
" in-order tx is done for direction %s. Flag %016"PRIx64
,
756 tx
, idx
, flags
& STREAM_TOSERVER
? "toserver" : "toclient", detect_flags
);
763 pstate
->inspect_id
[direction
] = idx
;
764 SCLogDebug("inspect_id now %"PRIu64
, pstate
->inspect_id
[direction
]);
766 /* if necessary we flag all txs that are complete as 'inspected'
767 * also move inspect_id forward. */
768 if (tag_txs_as_inspected
) {
769 /* continue at idx */
771 AppLayerGetTxIterTuple ires
= IterFunc(ipproto
, alproto
, alstate
, idx
, total_txs
, &state
);
772 if (ires
.tx_ptr
== NULL
)
775 void *tx
= ires
.tx_ptr
;
776 /* if we got a higher id than the minimum we requested, we
777 * skipped a bunch of 'null-txs'. Lets see if we can up the
779 if (ires
.tx_id
> idx
&& pstate
->inspect_id
[direction
] == idx
) {
780 pstate
->inspect_id
[direction
] = ires
.tx_id
;
784 const int state_progress
= AppLayerParserGetStateProgress(ipproto
, alproto
, tx
, flags
);
785 if (state_progress
< state_done_progress
)
788 /* txd can be NULL for HTTP sessions where the user data alloc failed */
789 AppLayerTxData
*txd
= AppLayerParserGetTxData(ipproto
, alproto
, tx
);
791 uint64_t detect_flags
= GetTxDetectFlags(txd
, flags
);
792 if ((detect_flags
& APP_LAYER_TX_INSPECTED_FLAG
) == 0) {
793 detect_flags
|= APP_LAYER_TX_INSPECTED_FLAG
;
794 SetTxDetectFlags(txd
, flags
, detect_flags
);
795 SCLogDebug("%p/%"PRIu64
" out of order tx is done for direction %s. Flag %016"PRIx64
,
796 tx
, idx
, flags
& STREAM_TOSERVER
? "toserver" : "toclient", detect_flags
);
798 SCLogDebug("%p/%"PRIu64
" out of order tx. Update inspect_id? %"PRIu64
,
799 tx
, idx
, pstate
->inspect_id
[direction
]);
800 if (pstate
->inspect_id
[direction
]+1 == idx
)
801 pstate
->inspect_id
[direction
] = idx
;
804 if (pstate
->inspect_id
[direction
]+1 == idx
)
805 pstate
->inspect_id
[direction
] = idx
;
816 AppLayerDecoderEvents
*AppLayerParserGetDecoderEvents(AppLayerParserState
*pstate
)
820 SCReturnPtr(pstate
->decoder_events
,
821 "AppLayerDecoderEvents *");
824 void AppLayerParserSetDecoderEvents(AppLayerParserState
*pstate
, AppLayerDecoderEvents
*devents
)
826 pstate
->decoder_events
= devents
;
829 AppLayerDecoderEvents
*AppLayerParserGetEventsByTx(uint8_t ipproto
, AppProto alproto
,
834 AppLayerDecoderEvents
*ptr
= NULL
;
836 if (alp_ctx
.ctxs
[FlowGetProtoMapping(ipproto
)][alproto
].
837 StateGetEvents
!= NULL
)
839 ptr
= alp_ctx
.ctxs
[FlowGetProtoMapping(ipproto
)][alproto
].
843 SCReturnPtr(ptr
, "AppLayerDecoderEvents *");
846 FileContainer
*AppLayerParserGetFiles(const Flow
*f
, const uint8_t direction
)
850 FileContainer
*ptr
= NULL
;
852 if (alp_ctx
.ctxs
[f
->protomap
][f
->alproto
].StateGetFiles
!= NULL
)
854 ptr
= alp_ctx
.ctxs
[f
->protomap
][f
->alproto
].
855 StateGetFiles(f
->alstate
, direction
);
858 SCReturnPtr(ptr
, "FileContainer *");
861 #define IS_DISRUPTED(flags) ((flags) & (STREAM_DEPTH | STREAM_GAP))
863 extern int g_detect_disabled
;
864 extern bool g_file_logger_enabled
;
865 extern bool g_filedata_logger_enabled
;
868 * \brief remove obsolete (inspected and logged) transactions
870 void AppLayerParserTransactionsCleanup(Flow
*f
)
873 DEBUG_ASSERT_FLOW_LOCKED(f
);
875 AppLayerParserProtoCtx
*p
= &alp_ctx
.ctxs
[f
->protomap
][f
->alproto
];
876 if (unlikely(p
->StateTransactionFree
== NULL
))
879 const bool has_tx_detect_flags
= !g_detect_disabled
;
880 const uint8_t ipproto
= f
->proto
;
881 const AppProto alproto
= f
->alproto
;
882 void * const alstate
= f
->alstate
;
883 AppLayerParserState
* const alparser
= f
->alparser
;
885 if (alstate
== NULL
|| alparser
== NULL
)
888 const uint64_t min
= alparser
->min_id
;
889 const uint64_t total_txs
= AppLayerParserGetTxCnt(f
, alstate
);
890 const LoggerId logger_expectation
= AppLayerParserProtocolGetLoggerBits(ipproto
, alproto
);
891 const int tx_end_state_ts
= AppLayerParserGetStateProgressCompletionStatus(alproto
, STREAM_TOSERVER
);
892 const int tx_end_state_tc
= AppLayerParserGetStateProgressCompletionStatus(alproto
, STREAM_TOCLIENT
);
893 const uint8_t ts_disrupt_flags
= FlowGetDisruptionFlags(f
, STREAM_TOSERVER
);
894 const uint8_t tc_disrupt_flags
= FlowGetDisruptionFlags(f
, STREAM_TOCLIENT
);
896 AppLayerGetTxIteratorFunc IterFunc
= AppLayerGetTxIterator(ipproto
, alproto
);
897 AppLayerGetTxIterState state
;
898 memset(&state
, 0, sizeof(state
));
900 uint64_t new_min
= min
;
901 SCLogDebug("start min %"PRIu64
, min
);
902 bool skipped
= false;
903 const bool is_unidir
=
904 AppLayerParserGetOptionFlags(f
->protomap
, f
->alproto
) & APP_LAYER_PARSER_OPT_UNIDIR_TXS
;
907 AppLayerGetTxIterTuple ires
= IterFunc(ipproto
, alproto
, alstate
, i
, total_txs
, &state
);
908 if (ires
.tx_ptr
== NULL
)
911 bool tx_skipped
= false;
912 void *tx
= ires
.tx_ptr
;
913 i
= ires
.tx_id
; // actual tx id for the tx the IterFunc returned
915 SCLogDebug("%p/%"PRIu64
" checking", tx
, i
);
917 const int tx_progress_tc
=
918 AppLayerParserGetStateProgress(ipproto
, alproto
, tx
, tc_disrupt_flags
);
919 if (tx_progress_tc
< tx_end_state_tc
) {
920 SCLogDebug("%p/%"PRIu64
" skipping: tc parser not done", tx
, i
);
924 const int tx_progress_ts
=
925 AppLayerParserGetStateProgress(ipproto
, alproto
, tx
, ts_disrupt_flags
);
926 if (tx_progress_ts
< tx_end_state_ts
) {
927 SCLogDebug("%p/%"PRIu64
" skipping: ts parser not done", tx
, i
);
932 AppLayerTxData
*txd
= AppLayerParserGetTxData(ipproto
, alproto
, tx
);
933 bool inspected
= false;
934 if (txd
&& has_tx_detect_flags
) {
935 if (!IS_DISRUPTED(ts_disrupt_flags
) && f
->sgh_toserver
!= NULL
) {
936 uint64_t detect_flags_ts
= GetTxDetectFlags(txd
, STREAM_TOSERVER
);
937 if (!(detect_flags_ts
& APP_LAYER_TX_INSPECTED_FLAG
)) {
938 SCLogDebug("%p/%"PRIu64
" skipping: TS inspect not done: ts:%"PRIx64
,
939 tx
, i
, detect_flags_ts
);
945 if (!IS_DISRUPTED(tc_disrupt_flags
) && f
->sgh_toclient
!= NULL
) {
946 uint64_t detect_flags_tc
= GetTxDetectFlags(txd
, STREAM_TOCLIENT
);
947 if (!(detect_flags_tc
& APP_LAYER_TX_INSPECTED_FLAG
)) {
948 SCLogDebug("%p/%"PRIu64
" skipping: TC inspect not done: tc:%"PRIx64
,
949 tx
, i
, detect_flags_tc
);
957 // If not a unidirectional transaction both sides are required to have
959 if (!is_unidir
&& tx_skipped
) {
960 SCLogDebug("%p/%" PRIu64
" !is_unidir && tx_skipped", tx
, i
);
965 // If this is a unidirectional transaction require only one side to be
966 // inspected, which the inspected flag tells us. This is also guarded
967 // with skip to limit this check to transactions that actually had the
968 // tx inspected flag checked.
969 if (is_unidir
&& tx_skipped
&& !inspected
) {
970 SCLogDebug("%p/%" PRIu64
" is_unidir && tx_skipped && !inspected", tx
, i
);
975 if (txd
&& logger_expectation
!= 0) {
976 LoggerId tx_logged
= GetTxLogged(txd
);
977 if (tx_logged
!= logger_expectation
) {
978 SCLogDebug("%p/%"PRIu64
" skipping: logging not done: want:%"PRIx32
", have:%"PRIx32
,
979 tx
, i
, logger_expectation
, tx_logged
);
985 /* if file logging is enabled, we keep a tx active while some of the files aren't
987 if (txd
&& txd
->files_opened
) {
988 if (g_file_logger_enabled
&& txd
->files_opened
!= txd
->files_logged
) {
992 if (g_filedata_logger_enabled
&& txd
->files_opened
!= txd
->files_stored
) {
998 /* if we are here, the tx can be freed. */
999 p
->StateTransactionFree(alstate
, i
);
1000 SCLogDebug("%p/%"PRIu64
" freed", tx
, i
);
1002 /* if we didn't skip any tx so far, up the minimum */
1003 SCLogDebug("skipped? %s i %"PRIu64
", new_min %"PRIu64
, skipped
? "true" : "false", i
, new_min
);
1006 SCLogDebug("final i %"PRIu64
", new_min %"PRIu64
, i
, new_min
);
1009 if (!ires
.has_next
) {
1010 /* this was the last tx. See if we skipped any. If not
1011 * we removed all and can update the minimum to the max
1013 SCLogDebug("no next: cur tx i %"PRIu64
", total %"PRIu64
, i
, total_txs
);
1015 new_min
= total_txs
;
1016 SCLogDebug("no next: cur tx i %"PRIu64
", total %"PRIu64
": "
1017 "new_min updated to %"PRIu64
, i
, total_txs
, new_min
);
1024 /* see if we need to bring all trackers up to date. */
1025 SCLogDebug("update f->alparser->min_id? %"PRIu64
" vs %"PRIu64
, new_min
, alparser
->min_id
);
1026 if (new_min
> alparser
->min_id
) {
1027 const uint64_t next_id
= new_min
;
1028 alparser
->min_id
= next_id
;
1029 alparser
->inspect_id
[0] = MAX(alparser
->inspect_id
[0], next_id
);
1030 alparser
->inspect_id
[1] = MAX(alparser
->inspect_id
[1], next_id
);
1031 alparser
->log_id
= MAX(alparser
->log_id
, next_id
);
1032 SCLogDebug("updated f->alparser->min_id %"PRIu64
, alparser
->min_id
);
1037 static inline int StateGetProgressCompletionStatus(const AppProto alproto
, const uint8_t flags
)
1039 if (flags
& STREAM_TOSERVER
) {
1040 return alp_ctx
.ctxs
[FLOW_PROTO_DEFAULT
][alproto
].complete_ts
;
1041 } else if (flags
& STREAM_TOCLIENT
) {
1042 return alp_ctx
.ctxs
[FLOW_PROTO_DEFAULT
][alproto
].complete_tc
;
1044 DEBUG_VALIDATE_BUG_ON(1);
1050 * \brief get the progress value for a tx/protocol
1052 * If the stream is disrupted, we return the 'completion' value.
1054 int AppLayerParserGetStateProgress(uint8_t ipproto
, AppProto alproto
,
1055 void *alstate
, uint8_t flags
)
1059 if (unlikely(IS_DISRUPTED(flags
))) {
1060 r
= StateGetProgressCompletionStatus(alproto
, flags
);
1062 r
= alp_ctx
.ctxs
[FlowGetProtoMapping(ipproto
)][alproto
].
1063 StateGetProgress(alstate
, flags
);
1068 uint64_t AppLayerParserGetTxCnt(const Flow
*f
, void *alstate
)
1071 uint64_t r
= alp_ctx
.ctxs
[f
->protomap
][f
->alproto
].StateGetTxCnt(alstate
);
1072 SCReturnCT(r
, "uint64_t");
1075 void *AppLayerParserGetTx(uint8_t ipproto
, AppProto alproto
, void *alstate
, uint64_t tx_id
)
1078 void *r
= alp_ctx
.ctxs
[FlowGetProtoMapping(ipproto
)][alproto
].StateGetTx(alstate
, tx_id
);
1079 SCReturnPtr(r
, "void *");
1082 int AppLayerParserGetStateProgressCompletionStatus(AppProto alproto
,
1086 int r
= StateGetProgressCompletionStatus(alproto
, direction
);
1090 int AppLayerParserGetEventInfo(uint8_t ipproto
, AppProto alproto
, const char *event_name
,
1091 int *event_id
, AppLayerEventType
*event_type
)
1094 const int ipproto_map
= FlowGetProtoMapping(ipproto
);
1095 int r
= (alp_ctx
.ctxs
[ipproto_map
][alproto
].StateGetEventInfo
== NULL
) ?
1096 -1 : alp_ctx
.ctxs
[ipproto_map
][alproto
].StateGetEventInfo(event_name
, event_id
, event_type
);
1100 int AppLayerParserGetEventInfoById(uint8_t ipproto
, AppProto alproto
, int event_id
,
1101 const char **event_name
, AppLayerEventType
*event_type
)
1104 const int ipproto_map
= FlowGetProtoMapping(ipproto
);
1105 *event_name
= (const char *)NULL
;
1106 int r
= (alp_ctx
.ctxs
[ipproto_map
][alproto
].StateGetEventInfoById
== NULL
) ?
1107 -1 : alp_ctx
.ctxs
[ipproto_map
][alproto
].StateGetEventInfoById(event_id
, event_name
, event_type
);
1111 uint8_t AppLayerParserGetFirstDataDir(uint8_t ipproto
, AppProto alproto
)
1114 uint8_t r
= alp_ctx
.ctxs
[FlowGetProtoMapping(ipproto
)][alproto
].first_data_dir
;
1115 SCReturnCT(r
, "uint8_t");
1118 uint64_t AppLayerParserGetTransactionActive(const Flow
*f
,
1119 AppLayerParserState
*pstate
, uint8_t direction
)
1124 uint64_t log_id
= pstate
->log_id
;
1125 uint64_t inspect_id
= pstate
->inspect_id
[direction
& STREAM_TOSERVER
? 0 : 1];
1126 if (alp_ctx
.ctxs
[f
->protomap
][f
->alproto
].logger
== true) {
1127 active_id
= MIN(log_id
, inspect_id
);
1129 active_id
= inspect_id
;
1132 SCReturnCT(active_id
, "uint64_t");
1135 int AppLayerParserSupportsFiles(uint8_t ipproto
, AppProto alproto
)
1137 // Custom case for only signature-only protocol so far
1138 if (alproto
== ALPROTO_HTTP
) {
1139 return AppLayerParserSupportsFiles(ipproto
, ALPROTO_HTTP1
) ||
1140 AppLayerParserSupportsFiles(ipproto
, ALPROTO_HTTP2
);
1142 if (alp_ctx
.ctxs
[FlowGetProtoMapping(ipproto
)][alproto
].StateGetFiles
!= NULL
)
1147 AppLayerTxData
*AppLayerParserGetTxData(uint8_t ipproto
, AppProto alproto
, void *tx
)
1150 AppLayerTxData
*d
= alp_ctx
.ctxs
[FlowGetProtoMapping(ipproto
)][alproto
].GetTxData(tx
);
1151 SCReturnPtr(d
, "AppLayerTxData");
1154 void AppLayerParserApplyTxConfig(uint8_t ipproto
, AppProto alproto
,
1155 void *state
, void *tx
, enum ConfigAction mode
, AppLayerTxConfig config
)
1158 const int ipproto_map
= FlowGetProtoMapping(ipproto
);
1159 if (alp_ctx
.ctxs
[ipproto_map
][alproto
].ApplyTxConfig
) {
1160 alp_ctx
.ctxs
[ipproto_map
][alproto
].ApplyTxConfig(state
, tx
, mode
, config
);
1165 /***** General *****/
1167 static inline void SetEOFFlags(AppLayerParserState
*pstate
, const uint8_t flags
)
1169 if ((flags
& (STREAM_EOF
|STREAM_TOSERVER
)) == (STREAM_EOF
|STREAM_TOSERVER
)) {
1170 SCLogDebug("setting APP_LAYER_PARSER_EOF_TS");
1171 AppLayerParserStateSetFlag(pstate
, APP_LAYER_PARSER_EOF_TS
);
1172 } else if ((flags
& (STREAM_EOF
|STREAM_TOCLIENT
)) == (STREAM_EOF
|STREAM_TOCLIENT
)) {
1173 SCLogDebug("setting APP_LAYER_PARSER_EOF_TC");
1174 AppLayerParserStateSetFlag(pstate
, APP_LAYER_PARSER_EOF_TC
);
1178 /** \retval int -1 in case of unrecoverable error. App-layer tracking stops for this flow.
1179 * \retval int 0 ok: we did not update app_progress
1180 * \retval int 1 ok: we updated app_progress */
1181 int AppLayerParserParse(ThreadVars
*tv
, AppLayerParserThreadCtx
*alp_tctx
, Flow
*f
, AppProto alproto
,
1182 uint8_t flags
, const uint8_t *input
, uint32_t input_len
)
1185 #ifdef DEBUG_VALIDATION
1186 BUG_ON(f
->protomap
!= FlowGetProtoMapping(f
->proto
));
1188 AppLayerParserState
*pstate
= f
->alparser
;
1189 AppLayerParserProtoCtx
*p
= &alp_ctx
.ctxs
[f
->protomap
][alproto
];
1190 void *alstate
= NULL
;
1191 uint64_t p_tx_cnt
= 0;
1192 uint32_t consumed
= input_len
;
1193 const int direction
= (flags
& STREAM_TOSERVER
) ? 0 : 1;
1195 /* we don't have the parser registered for this protocol */
1196 if (p
->StateAlloc
== NULL
)
1199 if (flags
& STREAM_GAP
) {
1200 if (!(p
->option_flags
& APP_LAYER_PARSER_OPT_ACCEPT_GAPS
)) {
1201 SCLogDebug("app-layer parser does not accept gaps");
1202 if (f
->alstate
!= NULL
&& !FlowChangeProto(f
)) {
1203 AppLayerParserStreamTruncated(f
->proto
, alproto
, f
->alstate
,
1210 /* Get the parser state (if any) */
1211 if (pstate
== NULL
) {
1212 f
->alparser
= pstate
= AppLayerParserStateAlloc();
1217 SetEOFFlags(pstate
, flags
);
1219 alstate
= f
->alstate
;
1220 if (alstate
== NULL
|| FlowChangeProto(f
)) {
1221 f
->alstate
= alstate
= p
->StateAlloc(alstate
, f
->alproto_orig
);
1222 if (alstate
== NULL
)
1224 SCLogDebug("alloced new app layer state %p (name %s)",
1225 alstate
, AppLayerGetProtoName(f
->alproto
));
1227 SCLogDebug("using existing app layer state %p (name %s))",
1228 alstate
, AppLayerGetProtoName(f
->alproto
));
1231 p_tx_cnt
= AppLayerParserGetTxCnt(f
, f
->alstate
);
1233 /* invoke the recursive parser, but only on data. We may get empty msgs on EOF */
1234 if (input_len
> 0 || (flags
& STREAM_EOF
)) {
1235 /* invoke the parser */
1236 AppLayerResult res
= p
->Parser
[direction
](f
, alstate
, pstate
,
1238 alp_tctx
->alproto_local_storage
[f
->protomap
][alproto
],
1240 if (res
.status
< 0) {
1242 } else if (res
.status
> 0) {
1243 DEBUG_VALIDATE_BUG_ON(res
.consumed
> input_len
);
1244 DEBUG_VALIDATE_BUG_ON(res
.needed
+ res
.consumed
< input_len
);
1245 DEBUG_VALIDATE_BUG_ON(res
.needed
== 0);
1246 /* incomplete is only supported for TCP */
1247 DEBUG_VALIDATE_BUG_ON(f
->proto
!= IPPROTO_TCP
);
1249 /* put protocol in error state on improper use of the
1251 if (res
.consumed
> input_len
|| res
.needed
+ res
.consumed
< input_len
) {
1255 if (f
->proto
== IPPROTO_TCP
&& f
->protoctx
!= NULL
) {
1256 TcpSession
*ssn
= f
->protoctx
;
1257 SCLogDebug("direction %d/%s", direction
,
1258 (flags
& STREAM_TOSERVER
) ? "toserver" : "toclient");
1259 if (direction
== 0) {
1260 /* parser told us how much data it needs on top of what it
1261 * consumed. So we need tell stream engine how much we need
1262 * before the next call */
1263 ssn
->client
.data_required
= res
.needed
;
1264 SCLogDebug("setting data_required %u", ssn
->client
.data_required
);
1266 /* parser told us how much data it needs on top of what it
1267 * consumed. So we need tell stream engine how much we need
1268 * before the next call */
1269 ssn
->server
.data_required
= res
.needed
;
1270 SCLogDebug("setting data_required %u", ssn
->server
.data_required
);
1273 consumed
= res
.consumed
;
1277 /* set the packets to no inspection and reassembly if required */
1278 if (pstate
->flags
& APP_LAYER_PARSER_NO_INSPECTION
) {
1279 AppLayerParserSetEOF(pstate
);
1280 FlowSetNoPayloadInspectionFlag(f
);
1282 if (f
->proto
== IPPROTO_TCP
) {
1283 StreamTcpDisableAppLayer(f
);
1285 /* Set the no reassembly flag for both the stream in this TcpSession */
1286 if (pstate
->flags
& APP_LAYER_PARSER_NO_REASSEMBLY
) {
1287 /* Used only if it's TCP */
1288 TcpSession
*ssn
= f
->protoctx
;
1290 StreamTcpSetSessionNoReassemblyFlag(ssn
, 0);
1291 StreamTcpSetSessionNoReassemblyFlag(ssn
, 1);
1294 /* Set the bypass flag for both the stream in this TcpSession */
1295 if (pstate
->flags
& APP_LAYER_PARSER_BYPASS_READY
) {
1296 /* Used only if it's TCP */
1297 TcpSession
*ssn
= f
->protoctx
;
1299 StreamTcpSetSessionBypassFlag(ssn
);
1305 /* In cases like HeartBleed for TLS we need to inspect AppLayer but not Payload */
1306 if (!(f
->flags
& FLOW_NOPAYLOAD_INSPECTION
) && pstate
->flags
& APP_LAYER_PARSER_NO_INSPECTION_PAYLOAD
) {
1307 FlowSetNoPayloadInspectionFlag(f
);
1308 /* Set the no reassembly flag for both the stream in this TcpSession */
1309 if (f
->proto
== IPPROTO_TCP
) {
1310 /* Used only if it's TCP */
1311 TcpSession
*ssn
= f
->protoctx
;
1313 StreamTcpSetDisableRawReassemblyFlag(ssn
, 0);
1314 StreamTcpSetDisableRawReassemblyFlag(ssn
, 1);
1319 /* get the diff in tx cnt for stats keeping */
1320 uint64_t cur_tx_cnt
= AppLayerParserGetTxCnt(f
, f
->alstate
);
1321 if (cur_tx_cnt
> p_tx_cnt
&& tv
) {
1322 AppLayerIncTxCounter(tv
, f
, cur_tx_cnt
- p_tx_cnt
);
1325 /* stream truncated, inform app layer */
1326 if (flags
& STREAM_DEPTH
)
1327 AppLayerParserStreamTruncated(f
->proto
, alproto
, alstate
, flags
);
1330 /* update app progress */
1331 if (consumed
!= input_len
&& f
->proto
== IPPROTO_TCP
&& f
->protoctx
!= NULL
) {
1332 TcpSession
*ssn
= f
->protoctx
;
1333 StreamTcpUpdateAppLayerProgress(ssn
, direction
, consumed
);
1339 /* Set the no app layer inspection flag for both
1340 * the stream in this Flow */
1341 if (f
->proto
== IPPROTO_TCP
) {
1342 StreamTcpDisableAppLayer(f
);
1344 AppLayerParserSetEOF(pstate
);
1348 void AppLayerParserSetEOF(AppLayerParserState
*pstate
)
1355 SCLogDebug("setting APP_LAYER_PARSER_EOF_TC and APP_LAYER_PARSER_EOF_TS");
1356 AppLayerParserStateSetFlag(pstate
, (APP_LAYER_PARSER_EOF_TS
|APP_LAYER_PARSER_EOF_TC
));
1362 /* return true if there are app parser decoder events. These are
1363 * only the ones that are set during protocol detection. */
1364 bool AppLayerParserHasDecoderEvents(AppLayerParserState
*pstate
)
1371 const AppLayerDecoderEvents
*decoder_events
= AppLayerParserGetDecoderEvents(pstate
);
1372 if (decoder_events
&& decoder_events
->cnt
)
1375 /* if we have reached here, we don't have events */
1379 /** \brief simple way to globally test if a alproto is registered
1380 * and fully enabled in the configuration.
1382 int AppLayerParserIsEnabled(AppProto alproto
)
1384 for (int i
= 0; i
< FLOW_PROTO_APPLAYER_MAX
; i
++) {
1385 if (alp_ctx
.ctxs
[i
][alproto
].StateGetProgress
!= NULL
) {
1392 int AppLayerParserProtocolHasLogger(uint8_t ipproto
, AppProto alproto
)
1395 int ipproto_map
= FlowGetProtoMapping(ipproto
);
1396 int r
= (alp_ctx
.ctxs
[ipproto_map
][alproto
].logger
== false) ? 0 : 1;
1400 LoggerId
AppLayerParserProtocolGetLoggerBits(uint8_t ipproto
, AppProto alproto
)
1403 const int ipproto_map
= FlowGetProtoMapping(ipproto
);
1404 LoggerId r
= alp_ctx
.ctxs
[ipproto_map
][alproto
].logger_bits
;
1408 void AppLayerParserTriggerRawStreamReassembly(Flow
*f
, int direction
)
1412 SCLogDebug("f %p tcp %p direction %d", f
, f
? f
->protoctx
: NULL
, direction
);
1413 if (f
!= NULL
&& f
->protoctx
!= NULL
)
1414 StreamTcpReassembleTriggerRawReassembly(f
->protoctx
, direction
);
1419 void AppLayerParserSetStreamDepth(uint8_t ipproto
, AppProto alproto
, uint32_t stream_depth
)
1423 alp_ctx
.ctxs
[FlowGetProtoMapping(ipproto
)][alproto
].stream_depth
= stream_depth
;
1424 alp_ctx
.ctxs
[FlowGetProtoMapping(ipproto
)][alproto
].internal_flags
|=
1425 APP_LAYER_PARSER_INT_STREAM_DEPTH_SET
;
1430 uint32_t AppLayerParserGetStreamDepth(const Flow
*f
)
1432 SCReturnInt(alp_ctx
.ctxs
[f
->protomap
][f
->alproto
].stream_depth
);
1435 void AppLayerParserSetStreamDepthFlag(uint8_t ipproto
, AppProto alproto
, void *state
, uint64_t tx_id
, uint8_t flags
)
1439 if (state
!= NULL
) {
1440 if ((tx
= AppLayerParserGetTx(ipproto
, alproto
, state
, tx_id
)) != NULL
) {
1441 if (alp_ctx
.ctxs
[FlowGetProtoMapping(ipproto
)][alproto
].SetStreamDepthFlag
!= NULL
) {
1442 alp_ctx
.ctxs
[FlowGetProtoMapping(ipproto
)][alproto
].SetStreamDepthFlag(tx
, flags
);
1449 /***** Cleanup *****/
1451 void AppLayerParserStateProtoCleanup(
1452 uint8_t protomap
, AppProto alproto
, void *alstate
, AppLayerParserState
*pstate
)
1456 AppLayerParserProtoCtx
*ctx
= &alp_ctx
.ctxs
[protomap
][alproto
];
1458 if (ctx
->StateFree
!= NULL
&& alstate
!= NULL
)
1459 ctx
->StateFree(alstate
);
1461 /* free the app layer parser api state */
1463 AppLayerParserStateFree(pstate
);
1468 void AppLayerParserStateCleanup(const Flow
*f
, void *alstate
, AppLayerParserState
*pstate
)
1470 AppLayerParserStateProtoCleanup(f
->protomap
, f
->alproto
, alstate
, pstate
);
1473 static void ValidateParserProtoDump(AppProto alproto
, uint8_t ipproto
)
1475 uint8_t map
= FlowGetProtoMapping(ipproto
);
1476 const AppLayerParserProtoCtx
*ctx
= &alp_ctx
.ctxs
[map
][alproto
];
1477 printf("ERROR: incomplete app-layer registration\n");
1478 printf("AppLayer protocol %s ipproto %u\n", AppProtoToString(alproto
), ipproto
);
1479 printf("- option flags %"PRIx32
"\n", ctx
->option_flags
);
1480 printf("- first_data_dir %"PRIx8
"\n", ctx
->first_data_dir
);
1481 printf("Mandatory:\n");
1482 printf("- Parser[0] %p Parser[1] %p\n", ctx
->Parser
[0], ctx
->Parser
[1]);
1483 printf("- StateAlloc %p StateFree %p\n", ctx
->StateAlloc
, ctx
->StateFree
);
1484 printf("- StateGetTx %p StateGetTxCnt %p StateTransactionFree %p\n",
1485 ctx
->StateGetTx
, ctx
->StateGetTxCnt
, ctx
->StateTransactionFree
);
1486 printf("- GetTxData %p\n", ctx
->GetTxData
);
1487 printf("- StateGetProgress %p\n", ctx
->StateGetProgress
);
1488 printf("Optional:\n");
1489 printf("- LocalStorageAlloc %p LocalStorageFree %p\n", ctx
->LocalStorageAlloc
, ctx
->LocalStorageFree
);
1490 printf("- StateGetEvents %p StateGetEventInfo %p StateGetEventInfoById %p\n", ctx
->StateGetEvents
, ctx
->StateGetEventInfo
,
1491 ctx
->StateGetEventInfoById
);
1494 #define BOTH_SET(a, b) ((a) != NULL && (b) != NULL)
1495 #define BOTH_SET_OR_BOTH_UNSET(a, b) (((a) == NULL && (b) == NULL) || ((a) != NULL && (b) != NULL))
1496 #define THREE_SET_OR_THREE_UNSET(a, b, c) (((a) == NULL && (b) == NULL && (c) == NULL) || ((a) != NULL && (b) != NULL && (c) != NULL))
1497 #define THREE_SET(a, b, c) ((a) != NULL && (b) != NULL && (c) != NULL)
1499 static void ValidateParserProto(AppProto alproto
, uint8_t ipproto
)
1501 uint8_t map
= FlowGetProtoMapping(ipproto
);
1502 const AppLayerParserProtoCtx
*ctx
= &alp_ctx
.ctxs
[map
][alproto
];
1504 if (ctx
->Parser
[0] == NULL
&& ctx
->Parser
[1] == NULL
)
1507 if (!(BOTH_SET(ctx
->Parser
[0], ctx
->Parser
[1]))) {
1510 if (!(BOTH_SET(ctx
->StateFree
, ctx
->StateAlloc
))) {
1513 if (!(THREE_SET(ctx
->StateGetTx
, ctx
->StateGetTxCnt
, ctx
->StateTransactionFree
))) {
1516 if (ctx
->StateGetProgress
== NULL
) {
1519 /* local storage is optional, but needs both set if used */
1520 if (!(BOTH_SET_OR_BOTH_UNSET(ctx
->LocalStorageAlloc
, ctx
->LocalStorageFree
))) {
1523 if (ctx
->GetTxData
== NULL
) {
1528 ValidateParserProtoDump(alproto
, ipproto
);
1532 #undef BOTH_SET_OR_BOTH_UNSET
1533 #undef THREE_SET_OR_THREE_UNSET
1536 static void ValidateParser(AppProto alproto
)
1538 ValidateParserProto(alproto
, IPPROTO_TCP
);
1539 ValidateParserProto(alproto
, IPPROTO_UDP
);
1542 static void ValidateParsers(void)
1545 for ( ; p
< ALPROTO_MAX
; p
++) {
1550 void AppLayerParserRegisterProtocolParsers(void)
1554 RegisterHTPParsers();
1555 RegisterSSLParsers();
1556 rs_dcerpc_register_parser();
1557 rs_dcerpc_udp_register_parser();
1558 RegisterSMBParsers();
1559 RegisterFTPParsers();
1560 RegisterSSHParsers();
1561 RegisterSMTPParsers();
1562 rs_dns_udp_register_parser();
1563 rs_dns_tcp_register_parser();
1564 RegisterModbusParsers();
1565 RegisterENIPUDPParsers();
1566 RegisterENIPTCPParsers();
1567 RegisterDNP3Parsers();
1568 RegisterNFSTCPParsers();
1569 RegisterNFSUDPParsers();
1570 RegisterNTPParsers();
1571 RegisterTFTPParsers();
1572 RegisterIKEParsers();
1573 RegisterKRB5Parsers();
1574 rs_dhcp_register_parser();
1575 RegisterSNMPParsers();
1576 RegisterSIPParsers();
1577 RegisterTemplateRustParsers();
1578 RegisterRFBParsers();
1579 RegisterMQTTParsers();
1580 RegisterTemplateParsers();
1581 RegisterRdpParsers();
1582 RegisterHTTP2Parsers();
1585 AppLayerProtoDetectRegisterProtocol(ALPROTO_IMAP
, "imap");
1586 if (AppLayerProtoDetectConfProtoDetectionEnabled("tcp", "imap")) {
1587 if (AppLayerProtoDetectPMRegisterPatternCS(IPPROTO_TCP
, ALPROTO_IMAP
,
1588 "1|20|capability", 12, 0, STREAM_TOSERVER
) < 0)
1590 SCLogInfo("imap proto registration failure");
1594 SCLogInfo("Protocol detection and parser disabled for %s protocol.",
1603 /* coccinelle: AppLayerParserStateSetFlag():2,2:APP_LAYER_PARSER_ */
1604 void AppLayerParserStateSetFlag(AppLayerParserState
*pstate
, uint8_t flag
)
1607 pstate
->flags
|= flag
;
1611 /* coccinelle: AppLayerParserStateIssetFlag():2,2:APP_LAYER_PARSER_ */
1612 int AppLayerParserStateIssetFlag(AppLayerParserState
*pstate
, uint8_t flag
)
1615 SCReturnInt(pstate
->flags
& flag
);
1619 void AppLayerParserStreamTruncated(uint8_t ipproto
, AppProto alproto
, void *alstate
,
1624 if (alp_ctx
.ctxs
[FlowGetProtoMapping(ipproto
)][alproto
].Truncate
!= NULL
)
1625 alp_ctx
.ctxs
[FlowGetProtoMapping(ipproto
)][alproto
].Truncate(alstate
, direction
);
1631 void AppLayerParserStatePrintDetails(AppLayerParserState
*pstate
)
1638 AppLayerParserState
*p
= pstate
;
1639 SCLogDebug("AppLayerParser parser state information for parser state p(%p). "
1640 "p->inspect_id[0](%"PRIu64
"), "
1641 "p->inspect_id[1](%"PRIu64
"), "
1642 "p->log_id(%"PRIu64
"), "
1643 "p->decoder_events(%p).",
1644 pstate
, p
->inspect_id
[0], p
->inspect_id
[1], p
->log_id
,
1651 /***** Unittests *****/
1655 static AppLayerParserCtx alp_ctx_backup_unittest
;
1657 typedef struct TestState_
{
1662 * \brief Test parser function to test the memory deallocation of app layer
1663 * parser of occurence of an error.
1665 static AppLayerResult
TestProtocolParser(Flow
*f
, void *test_state
, AppLayerParserState
*pstate
,
1666 const uint8_t *input
, uint32_t input_len
,
1667 void *local_data
, const uint8_t flags
)
1670 SCReturnStruct(APP_LAYER_ERROR
);
1673 /** \brief Function to allocates the Test protocol state memory
1675 static void *TestProtocolStateAlloc(void *orig_state
, AppProto proto_orig
)
1678 void *s
= SCMalloc(sizeof(TestState
));
1679 if (unlikely(s
== NULL
))
1681 memset(s
, 0, sizeof(TestState
));
1683 SCReturnPtr(s
, "TestState");
1686 /** \brief Function to free the Test Protocol state memory
1688 static void TestProtocolStateFree(void *s
)
1693 static uint64_t TestGetTxCnt(void *state
)
1699 static void TestStateTransactionFree(void *state
, uint64_t tx_id
)
1704 static void *TestGetTx(void *state
, uint64_t tx_id
)
1706 TestState
*test_state
= (TestState
*)state
;
1710 void AppLayerParserRegisterProtocolUnittests(uint8_t ipproto
, AppProto alproto
,
1711 void (*RegisterUnittests
)(void))
1714 alp_ctx
.ctxs
[FlowGetProtoMapping(ipproto
)][alproto
].
1715 RegisterUnittests
= RegisterUnittests
;
1719 void AppLayerParserBackupParserTable(void)
1722 alp_ctx_backup_unittest
= alp_ctx
;
1723 memset(&alp_ctx
, 0, sizeof(alp_ctx
));
1727 void AppLayerParserRestoreParserTable(void)
1730 alp_ctx
= alp_ctx_backup_unittest
;
1731 memset(&alp_ctx_backup_unittest
, 0, sizeof(alp_ctx_backup_unittest
));
1736 * \test Test the deallocation of app layer parser memory on occurance of
1737 * error in the parsing process.
1739 static int AppLayerParserTest01(void)
1741 AppLayerParserBackupParserTable();
1744 uint8_t testbuf
[] = { 0x11 };
1745 uint32_t testlen
= sizeof(testbuf
);
1747 AppLayerParserThreadCtx
*alp_tctx
= AppLayerParserThreadCtxAlloc();
1749 memset(&ssn
, 0, sizeof(ssn
));
1751 /* Register the Test protocol state and parser functions */
1752 AppLayerParserRegisterParser(IPPROTO_TCP
, ALPROTO_TEST
, STREAM_TOSERVER
,
1753 TestProtocolParser
);
1754 AppLayerParserRegisterStateFuncs(IPPROTO_TCP
, ALPROTO_TEST
,
1755 TestProtocolStateAlloc
, TestProtocolStateFree
);
1756 AppLayerParserRegisterTxFreeFunc(IPPROTO_TCP
, ALPROTO_TEST
, TestStateTransactionFree
);
1757 AppLayerParserRegisterGetTx(IPPROTO_TCP
, ALPROTO_TEST
, TestGetTx
);
1758 AppLayerParserRegisterGetTxCnt(IPPROTO_TCP
, ALPROTO_TEST
, TestGetTxCnt
);
1760 f
= UTHBuildFlow(AF_INET
, "1.2.3.4", "4.3.2.1", 20, 40);
1763 f
->alproto
= ALPROTO_TEST
;
1764 f
->proto
= IPPROTO_TCP
;
1766 StreamTcpInitConfig(true);
1768 int r
= AppLayerParserParse(NULL
, alp_tctx
, f
, ALPROTO_TEST
,
1769 STREAM_TOSERVER
| STREAM_EOF
, testbuf
,
1773 FAIL_IF(!(ssn
.flags
& STREAMTCP_FLAG_APP_LAYER_DISABLED
));
1775 AppLayerParserRestoreParserTable();
1776 StreamTcpFreeConfig(true);
1782 * \test Test the deallocation of app layer parser memory on occurance of
1783 * error in the parsing process for UDP.
1785 static int AppLayerParserTest02(void)
1787 AppLayerParserBackupParserTable();
1790 uint8_t testbuf
[] = { 0x11 };
1791 uint32_t testlen
= sizeof(testbuf
);
1792 AppLayerParserThreadCtx
*alp_tctx
= AppLayerParserThreadCtxAlloc();
1794 /* Register the Test protocol state and parser functions */
1795 AppLayerParserRegisterParser(IPPROTO_UDP
, ALPROTO_TEST
, STREAM_TOSERVER
,
1796 TestProtocolParser
);
1797 AppLayerParserRegisterStateFuncs(IPPROTO_UDP
, ALPROTO_TEST
,
1798 TestProtocolStateAlloc
, TestProtocolStateFree
);
1799 AppLayerParserRegisterTxFreeFunc(IPPROTO_UDP
, ALPROTO_TEST
, TestStateTransactionFree
);
1800 AppLayerParserRegisterGetTx(IPPROTO_UDP
, ALPROTO_TEST
, TestGetTx
);
1801 AppLayerParserRegisterGetTxCnt(IPPROTO_UDP
, ALPROTO_TEST
, TestGetTxCnt
);
1803 f
= UTHBuildFlow(AF_INET
, "1.2.3.4", "4.3.2.1", 20, 40);
1805 f
->alproto
= ALPROTO_TEST
;
1806 f
->proto
= IPPROTO_UDP
;
1807 f
->protomap
= FlowGetProtoMapping(f
->proto
);
1809 StreamTcpInitConfig(true);
1811 int r
= AppLayerParserParse(NULL
, alp_tctx
, f
, ALPROTO_TEST
,
1812 STREAM_TOSERVER
| STREAM_EOF
, testbuf
,
1816 AppLayerParserRestoreParserTable();
1817 StreamTcpFreeConfig(true);
1823 void AppLayerParserRegisterUnittests(void)
1829 AppLayerParserProtoCtx
*ctx
;
1831 for (ip
= 0; ip
< FLOW_PROTO_DEFAULT
; ip
++) {
1832 for (alproto
= 0; alproto
< ALPROTO_MAX
; alproto
++) {
1833 ctx
= &alp_ctx
.ctxs
[ip
][alproto
];
1834 if (ctx
->RegisterUnittests
== NULL
)
1836 ctx
->RegisterUnittests();
1840 UtRegisterTest("AppLayerParserTest01", AppLayerParserTest01
);
1841 UtRegisterTest("AppLayerParserTest02", AppLayerParserTest02
);