From: Jeff Lucovsky Date: Fri, 31 Jan 2020 15:21:57 +0000 (-0500) Subject: detect/transform: Support transform options X-Git-Tag: suricata-6.0.0-beta1~345 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=5d10db88bc823e594afbb742a4f482548d0a982f;p=thirdparty%2Fsuricata.git detect/transform: Support transform options This commit adds support for transform-specific options. During Setup, transforms have the signature string available for options detection. When a transform detects an option, it should convert the option into an internal format and supply a pointer to this format as the last argument to DetectSignatureAddTransform. Transforms that support options must provide a function in their Sigmatch table entry. When the transform is freed, a pointer to the internal format of the option is passed to this function. --- diff --git a/src/detect-engine-mpm.c b/src/detect-engine-mpm.c index 00d043a1a7..009aeea004 100644 --- a/src/detect-engine-mpm.c +++ b/src/detect-engine-mpm.c @@ -173,7 +173,7 @@ void DetectAppLayerMpmRegisterByParentId(DetectEngineCtx *de_ctx, for (int i = 0; i < transforms->cnt; i++) { char ttstr[64]; (void)snprintf(ttstr,sizeof(ttstr), "%s,", - sigmatch_table[transforms->transforms[i]].name); + sigmatch_table[transforms->transforms[i].transform].name); strlcat(xforms, ttstr, sizeof(xforms)); } xforms[strlen(xforms)-1] = '\0'; diff --git a/src/detect-engine.c b/src/detect-engine.c index 3d38b5799c..326060bf0c 100644 --- a/src/detect-engine.c +++ b/src/detect-engine.c @@ -730,7 +730,10 @@ static HashListTable *g_buffer_type_hash = NULL; static int g_buffer_type_id = DETECT_SM_LIST_DYNAMIC_START; static int g_buffer_type_reg_closed = 0; -static DetectEngineTransforms no_transforms = { .transforms = { 0 }, .cnt = 0, }; +static DetectEngineTransforms no_transforms = { + .transforms[0] = {0, NULL}, + .cnt = 0, +}; int DetectBufferTypeMaxId(void) { @@ -763,6 +766,19 @@ static char DetectBufferTypeCompareFunc(void *data1, uint16_t len1, void *data2, static void DetectBufferTypeFreeFunc(void *data) { DetectBufferType *map = (DetectBufferType *)data; + + /* Release transformation option memory, if any */ + for (int i = 0; i < map->transforms.cnt; i++) { + if (map->transforms.transforms[i].options == NULL) + continue; + if (sigmatch_table[map->transforms.transforms[i].transform].Free == NULL) { + SCLogError(SC_ERR_UNIMPLEMENTED, + "%s allocates transform option memory but has no free routine", + sigmatch_table[map->transforms.transforms[i].transform].name); + continue; + } + sigmatch_table[map->transforms.transforms[i].transform].Free(NULL, map->transforms.transforms[i].options); + } if (map != NULL) { SCFree(map); } @@ -973,7 +989,7 @@ int DetectBufferSetActiveList(Signature *s, const int list) { BUG_ON(s->init_data == NULL); - if (s->init_data->list && s->init_data->transform_cnt) { + if (s->init_data->list && s->init_data->transforms.cnt) { return -1; } s->init_data->list = list; @@ -986,19 +1002,19 @@ int DetectBufferGetActiveList(DetectEngineCtx *de_ctx, Signature *s) { BUG_ON(s->init_data == NULL); - if (s->init_data->transform_cnt) { + if (s->init_data->list && s->init_data->transforms.cnt) { if (s->init_data->list == DETECT_SM_LIST_NOTSET || s->init_data->list < DETECT_SM_LIST_DYNAMIC_START) { SCLogError(SC_ERR_INVALID_SIGNATURE, "previous transforms not consumed " "(list: %u, transform_cnt %u)", s->init_data->list, - s->init_data->transform_cnt); + s->init_data->transforms.cnt); SCReturnInt(-1); } SCLogDebug("buffer %d has transform(s) registered: %d", - s->init_data->list, s->init_data->transforms[0]); + s->init_data->list, s->init_data->transforms.cnt); int new_list = DetectBufferTypeGetByIdTransforms(de_ctx, s->init_data->list, - s->init_data->transforms, s->init_data->transform_cnt); + s->init_data->transforms.transforms, s->init_data->transforms.cnt); if (new_list == -1) { SCReturnInt(-1); } @@ -1006,7 +1022,7 @@ int DetectBufferGetActiveList(DetectEngineCtx *de_ctx, Signature *s) s->init_data->list = new_list; s->init_data->list_set = false; // reset transforms now that we've set up the list - s->init_data->transform_cnt = 0; + s->init_data->transforms.cnt = 0; } SCReturnInt(0); @@ -1150,11 +1166,11 @@ void InspectionBufferApplyTransforms(InspectionBuffer *buffer, { if (transforms) { for (int i = 0; i < DETECT_TRANSFORMS_MAX; i++) { - const int id = transforms->transforms[i]; + const int id = transforms->transforms[i].transform; if (id == 0) break; BUG_ON(sigmatch_table[id].Transform == NULL); - sigmatch_table[id].Transform(buffer); + sigmatch_table[id].Transform(buffer, transforms->transforms[i].options); SCLogDebug("applied transform %s", sigmatch_table[id].name); } } @@ -1242,7 +1258,7 @@ void DetectBufferTypeCloseRegistration(void) } int DetectBufferTypeGetByIdTransforms(DetectEngineCtx *de_ctx, const int id, - int *transforms, int transform_cnt) + TransformData *transforms, int transform_cnt) { const DetectBufferType *base_map = DetectBufferTypeGetById(de_ctx, id); if (!base_map) { diff --git a/src/detect-engine.h b/src/detect-engine.h index 7bb232a206..6ef796134d 100644 --- a/src/detect-engine.h +++ b/src/detect-engine.h @@ -55,7 +55,7 @@ void DetectBufferTypeRegisterValidateCallback(const char *name, bool (*ValidateCallback)(const Signature *, const char **sigerror)); int DetectBufferTypeGetByIdTransforms(DetectEngineCtx *de_ctx, const int id, - int *transforms, int transform_cnt); + TransformData *transforms, int transform_cnt); const char *DetectBufferTypeGetNameById(const DetectEngineCtx *de_ctx, const int id); bool DetectBufferTypeSupportsMpmGetById(const DetectEngineCtx *de_ctx, const int id); bool DetectBufferTypeSupportsPacketGetById(const DetectEngineCtx *de_ctx, const int id); diff --git a/src/detect-parse.c b/src/detect-parse.c index fae001172e..c4705de37b 100644 --- a/src/detect-parse.c +++ b/src/detect-parse.c @@ -1386,6 +1386,15 @@ void SigFree(DetectEngineCtx *de_ctx, Signature *s) IPOnlyCIDRListFree(s->CidrSrc); int i; + + if (s->init_data && s->init_data->transforms.cnt) { + for(i = 0; i < s->init_data->transforms.cnt; i++) { + if (s->init_data->transforms.transforms[i].options) { + SCFree(s->init_data->transforms.transforms[i].options); + s->init_data->transforms.transforms[i].options = NULL; + } + } + } if (s->init_data) { const int nlists = s->init_data->smlists_array_size; for (i = 0; i < nlists; i++) { @@ -1439,7 +1448,7 @@ void SigFree(DetectEngineCtx *de_ctx, Signature *s) SCFree(s); } -int DetectSignatureAddTransform(Signature *s, int transform) +int DetectSignatureAddTransform(Signature *s, int transform, void *options) { /* we only support buffers */ if (s->init_data->list == 0) { @@ -1449,10 +1458,18 @@ int DetectSignatureAddTransform(Signature *s, int transform) SCLogError(SC_ERR_INVALID_SIGNATURE, "transforms must directly follow stickybuffers"); SCReturnInt(-1); } - if (s->init_data->transform_cnt >= DETECT_TRANSFORMS_MAX) { + if (s->init_data->transforms.cnt >= DETECT_TRANSFORMS_MAX) { SCReturnInt(-1); } - s->init_data->transforms[s->init_data->transform_cnt++] = transform; + + s->init_data->transforms.transforms[s->init_data->transforms.cnt].transform = transform; + s->init_data->transforms.transforms[s->init_data->transforms.cnt].options = options; + + s->init_data->transforms.cnt++; + SCLogDebug("Added transform #%d [%s]", + s->init_data->transforms.cnt, + s->sig_str); + SCReturnInt(0); } diff --git a/src/detect-parse.h b/src/detect-parse.h index 2e9d354658..be6a6a16af 100644 --- a/src/detect-parse.h +++ b/src/detect-parse.h @@ -83,7 +83,7 @@ SigMatch *DetectGetLastSMFromLists(const Signature *s, ...); SigMatch *DetectGetLastSMByListPtr(const Signature *s, SigMatch *sm_list, ...); SigMatch *DetectGetLastSMByListId(const Signature *s, int list_id, ...); -int DetectSignatureAddTransform(Signature *s, int transform); +int DetectSignatureAddTransform(Signature *s, int transform, void *options); int WARN_UNUSED DetectSignatureSetAppProto(Signature *s, AppProto alproto); /* parse regex setup and free util funcs */ diff --git a/src/detect-pkt-data.c b/src/detect-pkt-data.c index 72d5d59123..792597e9d8 100644 --- a/src/detect-pkt-data.c +++ b/src/detect-pkt-data.c @@ -71,7 +71,7 @@ void DetectPktDataRegister(void) static int DetectPktDataSetup (DetectEngineCtx *de_ctx, Signature *s, const char *unused) { SCEnter(); - if (s->init_data->transform_cnt) { + if (s->init_data->transforms.cnt) { SCLogError(SC_ERR_INVALID_SIGNATURE, "previous transforms not consumed before 'pkt_data'"); SCReturnInt(-1); diff --git a/src/detect-transform-compress-whitespace.c b/src/detect-transform-compress-whitespace.c index 97594f598a..4087b12111 100644 --- a/src/detect-transform-compress-whitespace.c +++ b/src/detect-transform-compress-whitespace.c @@ -37,7 +37,7 @@ static int DetectTransformCompressWhitespaceSetup (DetectEngineCtx *, Signature *, const char *); static void DetectTransformCompressWhitespaceRegisterTests(void); -static void TransformCompressWhitespace(InspectionBuffer *buffer); +static void TransformCompressWhitespace(InspectionBuffer *buffer, void *options); void DetectTransformCompressWhitespaceRegister(void) { @@ -69,11 +69,11 @@ void DetectTransformCompressWhitespaceRegister(void) static int DetectTransformCompressWhitespaceSetup (DetectEngineCtx *de_ctx, Signature *s, const char *nullstr) { SCEnter(); - int r = DetectSignatureAddTransform(s, DETECT_TRANSFORM_COMPRESS_WHITESPACE); + int r = DetectSignatureAddTransform(s, DETECT_TRANSFORM_COMPRESS_WHITESPACE, NULL); SCReturnInt(r); } -static void TransformCompressWhitespace(InspectionBuffer *buffer) +static void TransformCompressWhitespace(InspectionBuffer *buffer, void *options) { const uint8_t *input = buffer->inspect; const uint32_t input_len = buffer->inspect_len; @@ -133,7 +133,7 @@ static int DetectTransformCompressWhitespaceTest01(void) InspectionBufferInit(&buffer, 8); InspectionBufferSetup(&buffer, input, input_len); PrintRawDataFp(stdout, buffer.inspect, buffer.inspect_len); - TransformCompressWhitespace(&buffer); + TransformCompressWhitespace(&buffer, NULL); PrintRawDataFp(stdout, buffer.inspect, buffer.inspect_len); InspectionBufferFree(&buffer); PASS; @@ -152,7 +152,7 @@ static int DetectTransformCompressWhitespaceTest02(void) PrintRawDataFp(stdout, buffer.inspect, buffer.inspect_len); TransformDoubleWhitespace(&buffer); PrintRawDataFp(stdout, buffer.inspect, buffer.inspect_len); - TransformCompressWhitespace(&buffer); + TransformCompressWhitespace(&buffer, NULL); PrintRawDataFp(stdout, buffer.inspect, buffer.inspect_len); InspectionBufferFree(&buffer); PASS; diff --git a/src/detect-transform-dotprefix.c b/src/detect-transform-dotprefix.c index f490eef6b6..b0fa5c6835 100644 --- a/src/detect-transform-dotprefix.c +++ b/src/detect-transform-dotprefix.c @@ -39,7 +39,7 @@ static int DetectTransformDotPrefixSetup (DetectEngineCtx *, Signature *, const char *); static void DetectTransformDotPrefixRegisterTests(void); -static void TransformDotPrefix(InspectionBuffer *buffer); +static void TransformDotPrefix(InspectionBuffer *buffer, void *options); void DetectTransformDotPrefixRegister(void) { @@ -68,7 +68,7 @@ void DetectTransformDotPrefixRegister(void) static int DetectTransformDotPrefixSetup (DetectEngineCtx *de_ctx, Signature *s, const char *nullstr) { SCEnter(); - int r = DetectSignatureAddTransform(s, DETECT_TRANSFORM_DOTPREFIX); + int r = DetectSignatureAddTransform(s, DETECT_TRANSFORM_DOTPREFIX, NULL); SCReturnInt(r); } @@ -102,7 +102,7 @@ static int DetectTransformDotPrefixSetup (DetectEngineCtx *de_ctx, Signature *s, * 4. something.google.co.uk --> match * 5. google.com --> no match */ -static void TransformDotPrefix(InspectionBuffer *buffer) +static void TransformDotPrefix(InspectionBuffer *buffer, void *options) { const size_t input_len = buffer->inspect_len; @@ -128,7 +128,7 @@ static int DetectTransformDotPrefixTest01(void) InspectionBufferInit(&buffer, input_len); InspectionBufferSetup(&buffer, input, input_len); PrintRawDataFp(stdout, buffer.inspect, buffer.inspect_len); - TransformDotPrefix(&buffer); + TransformDotPrefix(&buffer, NULL); PrintRawDataFp(stdout, buffer.inspect, buffer.inspect_len); FAIL_IF_NOT(buffer.inspect_len == result_len); FAIL_IF_NOT(strncmp(result, (const char *)buffer.inspect, result_len) == 0); @@ -148,7 +148,7 @@ static int DetectTransformDotPrefixTest02(void) InspectionBufferInit(&buffer, input_len); InspectionBufferSetup(&buffer, input, input_len); PrintRawDataFp(stdout, buffer.inspect, buffer.inspect_len); - TransformDotPrefix(&buffer); + TransformDotPrefix(&buffer, NULL); PrintRawDataFp(stdout, buffer.inspect, buffer.inspect_len); FAIL_IF_NOT(buffer.inspect_len == result_len); FAIL_IF_NOT(strncmp(result, (const char *)buffer.inspect, result_len) == 0); diff --git a/src/detect-transform-md5.c b/src/detect-transform-md5.c index 871da01b59..a2af21efdd 100644 --- a/src/detect-transform-md5.c +++ b/src/detect-transform-md5.c @@ -37,7 +37,7 @@ static int DetectTransformToMd5Setup (DetectEngineCtx *, Signature *, const char *); #ifdef HAVE_NSS static void DetectTransformToMd5RegisterTests(void); -static void TransformToMd5(InspectionBuffer *buffer); +static void TransformToMd5(InspectionBuffer *buffer, void *options); #endif void DetectTransformMd5Register(void) @@ -78,11 +78,11 @@ static int DetectTransformToMd5Setup (DetectEngineCtx *de_ctx, Signature *s, con static int DetectTransformToMd5Setup (DetectEngineCtx *de_ctx, Signature *s, const char *nullstr) { SCEnter(); - int r = DetectSignatureAddTransform(s, DETECT_TRANSFORM_MD5); + int r = DetectSignatureAddTransform(s, DETECT_TRANSFORM_MD5, NULL); SCReturnInt(r); } -static void TransformToMd5(InspectionBuffer *buffer) +static void TransformToMd5(InspectionBuffer *buffer, void *options) { const uint8_t *input = buffer->inspect; const uint32_t input_len = buffer->inspect_len; @@ -112,7 +112,7 @@ static int DetectTransformToMd5Test01(void) InspectionBufferInit(&buffer, 8); InspectionBufferSetup(&buffer, input, input_len); PrintRawDataFp(stdout, buffer.inspect, buffer.inspect_len); - TransformToMd5(&buffer); + TransformToMd5(&buffer, NULL); PrintRawDataFp(stdout, buffer.inspect, buffer.inspect_len); InspectionBufferFree(&buffer); PASS; diff --git a/src/detect-transform-sha1.c b/src/detect-transform-sha1.c index a9127ae31f..cd42151733 100644 --- a/src/detect-transform-sha1.c +++ b/src/detect-transform-sha1.c @@ -37,7 +37,7 @@ static int DetectTransformToSha1Setup (DetectEngineCtx *, Signature *, const char *); #ifdef HAVE_NSS static void DetectTransformToSha1RegisterTests(void); -static void TransformToSha1(InspectionBuffer *buffer); +static void TransformToSha1(InspectionBuffer *buffer, void *options); #endif void DetectTransformSha1Register(void) @@ -78,11 +78,11 @@ static int DetectTransformToSha1Setup (DetectEngineCtx *de_ctx, Signature *s, co static int DetectTransformToSha1Setup (DetectEngineCtx *de_ctx, Signature *s, const char *nullstr) { SCEnter(); - int r = DetectSignatureAddTransform(s, DETECT_TRANSFORM_SHA1); + int r = DetectSignatureAddTransform(s, DETECT_TRANSFORM_SHA1, NULL); SCReturnInt(r); } -static void TransformToSha1(InspectionBuffer *buffer) +static void TransformToSha1(InspectionBuffer *buffer, void *options) { const uint8_t *input = buffer->inspect; const uint32_t input_len = buffer->inspect_len; @@ -112,7 +112,7 @@ static int DetectTransformToSha1Test01(void) InspectionBufferInit(&buffer, 8); InspectionBufferSetup(&buffer, input, input_len); PrintRawDataFp(stdout, buffer.inspect, buffer.inspect_len); - TransformToSha1(&buffer); + TransformToSha1(&buffer, NULL); PrintRawDataFp(stdout, buffer.inspect, buffer.inspect_len); InspectionBufferFree(&buffer); PASS; diff --git a/src/detect-transform-sha256.c b/src/detect-transform-sha256.c index 45c3920efb..b28b5492df 100644 --- a/src/detect-transform-sha256.c +++ b/src/detect-transform-sha256.c @@ -37,7 +37,7 @@ static int DetectTransformToSha256Setup (DetectEngineCtx *, Signature *, const char *); #ifdef HAVE_NSS static void DetectTransformToSha256RegisterTests(void); -static void TransformToSha256(InspectionBuffer *buffer); +static void TransformToSha256(InspectionBuffer *buffer, void *options); #endif void DetectTransformSha256Register(void) @@ -78,11 +78,11 @@ static int DetectTransformToSha256Setup (DetectEngineCtx *de_ctx, Signature *s, static int DetectTransformToSha256Setup (DetectEngineCtx *de_ctx, Signature *s, const char *nullstr) { SCEnter(); - int r = DetectSignatureAddTransform(s, DETECT_TRANSFORM_SHA256); + int r = DetectSignatureAddTransform(s, DETECT_TRANSFORM_SHA256, NULL); SCReturnInt(r); } -static void TransformToSha256(InspectionBuffer *buffer) +static void TransformToSha256(InspectionBuffer *buffer, void *options) { const uint8_t *input = buffer->inspect; const uint32_t input_len = buffer->inspect_len; @@ -112,7 +112,7 @@ static int DetectTransformToSha256Test01(void) InspectionBufferInit(&buffer, 8); InspectionBufferSetup(&buffer, input, input_len); PrintRawDataFp(stdout, buffer.inspect, buffer.inspect_len); - TransformToSha256(&buffer); + TransformToSha256(&buffer, NULL); PrintRawDataFp(stdout, buffer.inspect, buffer.inspect_len); InspectionBufferFree(&buffer); PASS; diff --git a/src/detect-transform-strip-whitespace.c b/src/detect-transform-strip-whitespace.c index 6622925136..6574fdbc5f 100644 --- a/src/detect-transform-strip-whitespace.c +++ b/src/detect-transform-strip-whitespace.c @@ -37,7 +37,7 @@ static int DetectTransformStripWhitespaceSetup (DetectEngineCtx *, Signature *, const char *); static void DetectTransformStripWhitespaceRegisterTests(void); -static void TransformStripWhitespace(InspectionBuffer *buffer); +static void TransformStripWhitespace(InspectionBuffer *buffer, void *options); void DetectTransformStripWhitespaceRegister(void) { @@ -68,11 +68,11 @@ void DetectTransformStripWhitespaceRegister(void) static int DetectTransformStripWhitespaceSetup (DetectEngineCtx *de_ctx, Signature *s, const char *nullstr) { SCEnter(); - int r = DetectSignatureAddTransform(s, DETECT_TRANSFORM_STRIP_WHITESPACE); + int r = DetectSignatureAddTransform(s, DETECT_TRANSFORM_STRIP_WHITESPACE, NULL); SCReturnInt(r); } -static void TransformStripWhitespace(InspectionBuffer *buffer) +static void TransformStripWhitespace(InspectionBuffer *buffer, void *options) { const uint8_t *input = buffer->inspect; const uint32_t input_len = buffer->inspect_len; @@ -124,7 +124,7 @@ static int DetectTransformStripWhitespaceTest01(void) InspectionBufferInit(&buffer, 8); InspectionBufferSetup(&buffer, input, input_len); PrintRawDataFp(stdout, buffer.inspect, buffer.inspect_len); - TransformStripWhitespace(&buffer); + TransformStripWhitespace(&buffer, NULL); PrintRawDataFp(stdout, buffer.inspect, buffer.inspect_len); InspectionBufferFree(&buffer); PASS; @@ -143,7 +143,7 @@ static int DetectTransformStripWhitespaceTest02(void) PrintRawDataFp(stdout, buffer.inspect, buffer.inspect_len); TransformDoubleWhitespace(&buffer); PrintRawDataFp(stdout, buffer.inspect, buffer.inspect_len); - TransformStripWhitespace(&buffer); + TransformStripWhitespace(&buffer, NULL); PrintRawDataFp(stdout, buffer.inspect, buffer.inspect_len); InspectionBufferFree(&buffer); PASS; diff --git a/src/detect.h b/src/detect.h index 8d6d665591..dbb8091840 100644 --- a/src/detect.h +++ b/src/detect.h @@ -366,8 +366,13 @@ typedef struct InspectionBufferMultipleForList { uint32_t init:1; /**< first time used this run. Used for clean logic */ } InspectionBufferMultipleForList; +typedef struct TransformData_ { + int transform; + void *options; +} TransformData; + typedef struct DetectEngineTransforms { - int transforms[DETECT_TRANSFORMS_MAX]; + TransformData transforms[DETECT_TRANSFORMS_MAX]; int cnt; } DetectEngineTransforms; @@ -498,8 +503,7 @@ typedef struct SignatureInitData_ { int list; bool list_set; - int transforms[DETECT_TRANSFORMS_MAX]; - int transform_cnt; + DetectEngineTransforms transforms; /** score to influence rule grouping. A higher value leads to a higher * likelihood of a rulegroup with this sig ending up as a contained @@ -1183,7 +1187,7 @@ typedef struct SigTableElmt_ { uint8_t flags, File *, const Signature *, const SigMatchCtx *); /** InspectionBuffer transformation callback */ - void (*Transform)(InspectionBuffer *); + void (*Transform)(InspectionBuffer *, void *context); /** keyword setup function pointer */ int (*Setup)(DetectEngineCtx *, Signature *, const char *);