#include "detect-engine-mpm.h"
#include "detect-engine-state.h"
#include "detect-engine-build.h"
+#include "detect-engine-prefilter.h"
+
+#include "tree.h"
#include "util-var-name.h"
#include "util-unittest.h"
#ifdef UNITTESTS
void FlowBitsRegisterTests(void);
#endif
+static bool PrefilterFlowbitIsPrefilterable(const Signature *s);
+static int PrefilterSetupFlowbits(DetectEngineCtx *de_ctx, SigGroupHead *sgh);
void DetectFlowbitsRegister (void)
{
/* this is compatible to ip-only signatures */
sigmatch_table[DETECT_FLOWBITS].flags |= SIGMATCH_IPONLY_COMPAT;
+ sigmatch_table[DETECT_FLOWBITS].SupportsPrefilter = PrefilterFlowbitIsPrefilterable;
+ sigmatch_table[DETECT_FLOWBITS].SetupPrefilter = PrefilterSetupFlowbits;
DetectSetupParseRegexes(PARSE_REGEX, &parse_regex);
}
static int DetectFlowbitMatchToggle (Packet *p, const DetectFlowbitsData *fd)
{
if (p->flow == NULL)
- return 0;
-
- FlowBitToggle(p->flow,fd->idx);
+ return -1;
- return 1;
+ return FlowBitToggle(p->flow, fd->idx);
}
static int DetectFlowbitMatchUnset (Packet *p, const DetectFlowbitsData *fd)
static int DetectFlowbitMatchSet (Packet *p, const DetectFlowbitsData *fd)
{
if (p->flow == NULL)
- return 0;
-
- FlowBitSet(p->flow,fd->idx);
+ return -1;
- return 1;
+ int r = FlowBitSet(p->flow, fd->idx);
+ SCLogDebug("set %u", fd->idx);
+ return r;
}
static int DetectFlowbitMatchIsset (Packet *p, const DetectFlowbitsData *fd)
}
/*
- * returns 0: no match
+ * returns 0: no match (or error)
* 1: match
- * -1: error
*/
int DetectFlowbitMatch (DetectEngineThreadCtx *det_ctx, Packet *p,
return DetectFlowbitMatchIsset(p,fd);
case DETECT_FLOWBITS_CMD_ISNOTSET:
return DetectFlowbitMatchIsnotset(p,fd);
- case DETECT_FLOWBITS_CMD_SET:
- return DetectFlowbitMatchSet(p,fd);
+ case DETECT_FLOWBITS_CMD_SET: {
+ int r = DetectFlowbitMatchSet(p, fd);
+ /* only on a new "set" invoke the prefilter */
+ if (r == 1 && fd->post_rule_match_prefilter) {
+ SCLogDebug("flowbit set, appending to work queue");
+ PostRuleMatchWorkQueueAppend(det_ctx, s, DETECT_FLOWBITS, fd->idx);
+ }
+ return (r != -1);
+ }
case DETECT_FLOWBITS_CMD_UNSET:
return DetectFlowbitMatchUnset(p,fd);
- case DETECT_FLOWBITS_CMD_TOGGLE:
- return DetectFlowbitMatchToggle(p,fd);
+ case DETECT_FLOWBITS_CMD_TOGGLE: {
+ int r = DetectFlowbitMatchToggle(p, fd);
+ if (r == 1 && fd->post_rule_match_prefilter) {
+ SCLogDebug("flowbit set (by toggle), appending to work queue");
+ PostRuleMatchWorkQueueAppend(det_ctx, s, DETECT_FLOWBITS, fd->idx);
+ }
+ return (r != -1);
+ }
default:
SCLogError("unknown cmd %" PRIu32 "", fd->cmd);
return 0;
SCFree(fd);
}
+struct FBAnalyzer {
+ struct FBAnalyze *array;
+ uint32_t array_size;
+};
+
struct FBAnalyze {
uint16_t cnts[DETECT_FLOWBITS_CMD_MAX];
uint16_t state_cnts[DETECT_FLOWBITS_CMD_MAX];
static void DetectFlowbitsAnalyzeDump(const DetectEngineCtx *de_ctx,
struct FBAnalyze *array, uint32_t elements);
+static void FBAnalyzerArrayFree(struct FBAnalyze *array, const uint32_t array_size)
+{
+ if (array) {
+ for (uint32_t i = 0; i < array_size; i++) {
+ SCFree(array[i].set_sids);
+ SCFree(array[i].unset_sids);
+ SCFree(array[i].isset_sids);
+ SCFree(array[i].isnotset_sids);
+ SCFree(array[i].toggle_sids);
+ }
+ SCFree(array);
+ }
+}
+
+static void FBAnalyzerFree(struct FBAnalyzer *fba)
+{
+ if (fba && fba->array) {
+ FBAnalyzerArrayFree(fba->array, fba->array_size);
+ fba->array = NULL;
+ fba->array_size = 0;
+ }
+}
+
+#define MAX_SIDS 8
+static bool CheckExpand(const uint32_t sids_idx, uint32_t **sids, uint32_t *sids_size)
+{
+ if (sids_idx >= *sids_size) {
+ const uint32_t old_size = *sids_size;
+ const uint32_t new_size = MAX(2 * old_size, MAX_SIDS);
+
+ void *ptr = SCRealloc(*sids, new_size * sizeof(uint32_t));
+ if (ptr == NULL)
+ return false;
+ *sids_size = new_size;
+ *sids = ptr;
+ }
+ return true;
+}
+
+static int DetectFlowbitsAnalyzeSignature(const Signature *s, struct FBAnalyzer *fba)
+{
+ struct FBAnalyze *array = fba->array;
+ if (array == NULL)
+ return -1;
+
+ /* see if the signature uses stateful matching TODO is there not a flag? */
+ bool has_state = (s->init_data->buffer_index != 0);
+
+ for (const SigMatch *sm = s->init_data->smlists[DETECT_SM_LIST_MATCH]; sm != NULL;
+ sm = sm->next) {
+ if (sm->type != DETECT_FLOWBITS)
+ continue;
+ /* figure out the flowbit action */
+ const DetectFlowbitsData *fb = (DetectFlowbitsData *)sm->ctx;
+ // Handle flowbit array in case of ORed flowbits
+ for (uint8_t k = 0; k < fb->or_list_size; k++) {
+ struct FBAnalyze *fa = &array[fb->or_list[k]];
+ fa->cnts[fb->cmd]++;
+ fa->state_cnts[fb->cmd] += has_state;
+
+ if (fb->cmd == DETECT_FLOWBITS_CMD_ISSET) {
+ if (!CheckExpand(fa->isset_sids_idx, &fa->isset_sids, &fa->isset_sids_size))
+ return -1;
+ fa->isset_sids[fa->isset_sids_idx] = s->num;
+ fa->isset_sids_idx++;
+ } else if (fb->cmd == DETECT_FLOWBITS_CMD_ISNOTSET) {
+ if (!CheckExpand(
+ fa->isnotset_sids_idx, &fa->isnotset_sids, &fa->isnotset_sids_size))
+ return -1;
+ fa->isnotset_sids[fa->isnotset_sids_idx] = s->num;
+ fa->isnotset_sids_idx++;
+ }
+ }
+ if (fb->or_list_size == 0) {
+ struct FBAnalyze *fa = &array[fb->idx];
+ fa->cnts[fb->cmd]++;
+ fa->state_cnts[fb->cmd] += has_state;
+
+ if (fb->cmd == DETECT_FLOWBITS_CMD_ISSET) {
+ if (!CheckExpand(fa->isset_sids_idx, &fa->isset_sids, &fa->isset_sids_size))
+ return -1;
+ fa->isset_sids[fa->isset_sids_idx] = s->num;
+ fa->isset_sids_idx++;
+ } else if (fb->cmd == DETECT_FLOWBITS_CMD_ISNOTSET) {
+ if (!CheckExpand(
+ fa->isnotset_sids_idx, &fa->isnotset_sids, &fa->isnotset_sids_size))
+ return -1;
+ fa->isnotset_sids[fa->isnotset_sids_idx] = s->num;
+ fa->isnotset_sids_idx++;
+ }
+ }
+ }
+ for (const SigMatch *sm = s->init_data->smlists[DETECT_SM_LIST_POSTMATCH]; sm != NULL;
+ sm = sm->next) {
+ if (sm->type != DETECT_FLOWBITS)
+ continue;
+ /* figure out what flowbit action */
+ const DetectFlowbitsData *fb = (DetectFlowbitsData *)sm->ctx;
+ struct FBAnalyze *fa = &array[fb->idx];
+ fa->cnts[fb->cmd]++;
+ fa->state_cnts[fb->cmd] += has_state;
+
+ if (fb->cmd == DETECT_FLOWBITS_CMD_SET) {
+ if (!CheckExpand(fa->set_sids_idx, &fa->set_sids, &fa->set_sids_size))
+ return -1;
+ fa->set_sids[fa->set_sids_idx] = s->num;
+ fa->set_sids_idx++;
+ } else if (fb->cmd == DETECT_FLOWBITS_CMD_UNSET) {
+ if (!CheckExpand(fa->unset_sids_idx, &fa->unset_sids, &fa->unset_sids_size))
+ return -1;
+ fa->unset_sids[fa->unset_sids_idx] = s->num;
+ fa->unset_sids_idx++;
+ } else if (fb->cmd == DETECT_FLOWBITS_CMD_TOGGLE) {
+ if (!CheckExpand(fa->toggle_sids_idx, &fa->toggle_sids, &fa->toggle_sids_size))
+ return -1;
+ fa->toggle_sids[fa->toggle_sids_idx] = s->num;
+ fa->toggle_sids_idx++;
+ }
+ }
+ return 0;
+}
+
int DetectFlowbitsAnalyze(DetectEngineCtx *de_ctx)
{
const uint32_t max_fb_id = de_ctx->max_fb_id;
if (max_fb_id == 0)
return 0;
-#define MAX_SIDS 8
- uint32_t array_size = max_fb_id + 1;
+ struct FBAnalyzer fba = { .array = NULL, .array_size = 0 };
+ const uint32_t array_size = max_fb_id + 1;
struct FBAnalyze *array = SCCalloc(array_size, sizeof(struct FBAnalyze));
-
if (array == NULL) {
SCLogError("Unable to allocate flowbit analyze array");
return -1;
}
+ fba.array = array;
+ fba.array_size = array_size;
SCLogDebug("fb analyzer array size: %"PRIu64,
(uint64_t)(array_size * sizeof(struct FBAnalyze)));
for (uint32_t i = 0; i < de_ctx->sig_array_len; i++) {
const Signature *s = de_ctx->sig_array[i];
- /* see if the signature uses stateful matching */
- bool has_state = (s->init_data->buffer_index != 0);
-
- for (const SigMatch *sm = s->init_data->smlists[DETECT_SM_LIST_MATCH] ; sm != NULL; sm = sm->next) {
- switch (sm->type) {
- case DETECT_FLOWBITS:
- {
- /* figure out the flowbit action */
- const DetectFlowbitsData *fb = (DetectFlowbitsData *)sm->ctx;
- // Handle flowbit array in case of ORed flowbits
- for (uint8_t k = 0; k < fb->or_list_size; k++) {
- array[fb->or_list[k]].cnts[fb->cmd]++;
- if (has_state)
- array[fb->or_list[k]].state_cnts[fb->cmd]++;
- if (fb->cmd == DETECT_FLOWBITS_CMD_ISSET) {
- if (array[fb->or_list[k]].isset_sids_idx >= array[fb->or_list[k]].isset_sids_size) {
- uint32_t old_size = array[fb->or_list[k]].isset_sids_size;
- uint32_t new_size = MAX(2 * old_size, MAX_SIDS);
-
- void *ptr = SCRealloc(array[fb->or_list[k]].isset_sids, new_size * sizeof(uint32_t));
- if (ptr == NULL)
- goto end;
- array[fb->or_list[k]].isset_sids_size = new_size;
- array[fb->or_list[k]].isset_sids = ptr;
- }
-
- array[fb->or_list[k]].isset_sids[array[fb->or_list[k]].isset_sids_idx] = s->num;
- array[fb->or_list[k]].isset_sids_idx++;
- } else if (fb->cmd == DETECT_FLOWBITS_CMD_ISNOTSET) {
- if (array[fb->or_list[k]].isnotset_sids_idx >= array[fb->or_list[k]].isnotset_sids_size) {
- uint32_t old_size = array[fb->or_list[k]].isnotset_sids_size;
- uint32_t new_size = MAX(2 * old_size, MAX_SIDS);
-
- void *ptr = SCRealloc(array[fb->or_list[k]].isnotset_sids, new_size * sizeof(uint32_t));
- if (ptr == NULL)
- goto end;
- array[fb->or_list[k]].isnotset_sids_size = new_size;
- array[fb->or_list[k]].isnotset_sids = ptr;
- }
-
- array[fb->or_list[k]].isnotset_sids[array[fb->or_list[k]].isnotset_sids_idx] = s->num;
- array[fb->or_list[k]].isnotset_sids_idx++;
- }
- }
- if (fb->or_list_size == 0) {
- array[fb->idx].cnts[fb->cmd]++;
- if (has_state)
- array[fb->idx].state_cnts[fb->cmd]++;
- if (fb->cmd == DETECT_FLOWBITS_CMD_ISSET) {
- if (array[fb->idx].isset_sids_idx >= array[fb->idx].isset_sids_size) {
- uint32_t old_size = array[fb->idx].isset_sids_size;
- uint32_t new_size = MAX(2 * old_size, MAX_SIDS);
-
- void *ptr = SCRealloc(array[fb->idx].isset_sids, new_size * sizeof(uint32_t));
- if (ptr == NULL)
- goto end;
- array[fb->idx].isset_sids_size = new_size;
- array[fb->idx].isset_sids = ptr;
- }
-
- array[fb->idx].isset_sids[array[fb->idx].isset_sids_idx] = s->num;
- array[fb->idx].isset_sids_idx++;
- } else if (fb->cmd == DETECT_FLOWBITS_CMD_ISNOTSET) {
- if (array[fb->idx].isnotset_sids_idx >= array[fb->idx].isnotset_sids_size) {
- uint32_t old_size = array[fb->idx].isnotset_sids_size;
- uint32_t new_size = MAX(2 * old_size, MAX_SIDS);
-
- void *ptr = SCRealloc(array[fb->idx].isnotset_sids, new_size * sizeof(uint32_t));
- if (ptr == NULL)
- goto end;
- array[fb->idx].isnotset_sids_size = new_size;
- array[fb->idx].isnotset_sids = ptr;
- }
-
- array[fb->idx].isnotset_sids[array[fb->idx].isnotset_sids_idx] = s->num;
- array[fb->idx].isnotset_sids_idx++;
- }
- }
- }
- }
- }
- for (const SigMatch *sm = s->init_data->smlists[DETECT_SM_LIST_POSTMATCH] ; sm != NULL; sm = sm->next) {
- switch (sm->type) {
- case DETECT_FLOWBITS:
- {
- /* figure out what flowbit action */
- const DetectFlowbitsData *fb = (DetectFlowbitsData *)sm->ctx;
- array[fb->idx].cnts[fb->cmd]++;
- if (has_state)
- array[fb->idx].state_cnts[fb->cmd]++;
- if (fb->cmd == DETECT_FLOWBITS_CMD_SET) {
- if (array[fb->idx].set_sids_idx >= array[fb->idx].set_sids_size) {
- uint32_t old_size = array[fb->idx].set_sids_size;
- uint32_t new_size = MAX(2 * old_size, MAX_SIDS);
-
- void *ptr = SCRealloc(array[fb->idx].set_sids, new_size * sizeof(uint32_t));
- if (ptr == NULL)
- goto end;
- array[fb->idx].set_sids_size = new_size;
- array[fb->idx].set_sids = ptr;
- }
-
- array[fb->idx].set_sids[array[fb->idx].set_sids_idx] = s->num;
- array[fb->idx].set_sids_idx++;
- }
- else if (fb->cmd == DETECT_FLOWBITS_CMD_UNSET) {
- if (array[fb->idx].unset_sids_idx >= array[fb->idx].unset_sids_size) {
- uint32_t old_size = array[fb->idx].unset_sids_size;
- uint32_t new_size = MAX(2 * old_size, MAX_SIDS);
-
- void *ptr = SCRealloc(array[fb->idx].unset_sids, new_size * sizeof(uint32_t));
- if (ptr == NULL)
- goto end;
- array[fb->idx].unset_sids_size = new_size;
- array[fb->idx].unset_sids = ptr;
- }
-
- array[fb->idx].unset_sids[array[fb->idx].unset_sids_idx] = s->num;
- array[fb->idx].unset_sids_idx++;
- }
- else if (fb->cmd == DETECT_FLOWBITS_CMD_TOGGLE) {
- if (array[fb->idx].toggle_sids_idx >= array[fb->idx].toggle_sids_size) {
- uint32_t old_size = array[fb->idx].toggle_sids_size;
- uint32_t new_size = MAX(2 * old_size, MAX_SIDS);
-
- void *ptr = SCRealloc(array[fb->idx].toggle_sids, new_size * sizeof(uint32_t));
- if (ptr == NULL)
- goto end;
- array[fb->idx].toggle_sids_size = new_size;
- array[fb->idx].toggle_sids = ptr;
- }
-
- array[fb->idx].toggle_sids[array[fb->idx].toggle_sids_idx] = s->num;
- array[fb->idx].toggle_sids_idx++;
- }
- }
- }
+ int r = DetectFlowbitsAnalyzeSignature(s, &fba);
+ if (r < 0) {
+ FBAnalyzerFree(&fba);
+ return -1;
}
}
SCCalloc(sids_array_size, sizeof(uint32_t));
if (s->init_data->rule_state_dependant_sids_array == NULL) {
SCLogError("Failed to allocate memory for rule_state_dependant_ids");
- goto end;
+ goto error;
}
s->init_data->rule_state_flowbits_ids_size = 1;
s->init_data->rule_state_flowbits_ids_array =
SCCalloc(s->init_data->rule_state_flowbits_ids_size, sizeof(uint32_t));
if (s->init_data->rule_state_flowbits_ids_array == NULL) {
SCLogError("Failed to allocate memory for rule_state_variable_idx");
- goto end;
+ goto error;
}
s->init_data->rule_state_dependant_sids_size = sids_array_size;
SCLogDebug("alloc'ed array for rule dependency and fbs idx array, sid %u, "
new_array_size * sizeof(uint32_t));
if (tmp_ptr == NULL) {
SCLogError("Failed to allocate memory for rule_state_variable_idx");
- goto end;
+ goto error;
}
s->init_data->rule_state_dependant_sids_array = tmp_ptr;
s->init_data->rule_state_dependant_sids_size = new_array_size;
s->init_data->rule_state_flowbits_ids_array = tmp_fb_ptr;
if (s->init_data->rule_state_flowbits_ids_array == NULL) {
SCLogError("Failed to reallocate memory for rule_state_variable_idx");
- goto end;
+ goto error;
}
SCLogDebug(
"realloc'ed array for flowbits ids, new size is %u", new_fb_array_size);
DetectFlowbitsAnalyzeDump(de_ctx, array, array_size);
}
-end:
+ FBAnalyzerFree(&fba);
+ return 0;
+error:
+ FBAnalyzerFree(&fba);
+ return -1;
+}
+
+// TODO misses IPOnly rules. IPOnly flowbit rules are set only though.
+static struct FBAnalyzer DetectFlowbitsAnalyzeForGroup(
+ const DetectEngineCtx *de_ctx, SigGroupHead *sgh)
+{
+ struct FBAnalyzer fba = { .array = NULL, .array_size = 0 };
+
+ const uint32_t max_fb_id = de_ctx->max_fb_id;
+ if (max_fb_id == 0)
+ return fba;
+
+ uint32_t array_size = max_fb_id + 1;
+ struct FBAnalyze *array = SCCalloc(array_size, sizeof(struct FBAnalyze));
+ if (array == NULL) {
+ SCLogError("Unable to allocate flowbit analyze array");
+ return fba;
+ }
+ SCLogDebug(
+ "fb analyzer array size: %" PRIu64, (uint64_t)(array_size * sizeof(struct FBAnalyze)));
+ fba.array = array;
+ fba.array_size = array_size;
+
+ /* fill flowbit array, updating counters per sig */
+ for (uint32_t i = 0; i < sgh->init->sig_cnt; i++) {
+ const Signature *s = sgh->init->match_array[i];
+ SCLogDebug("sgh %p: s->id %u", sgh, s->id);
+
+ int r = DetectFlowbitsAnalyzeSignature(s, &fba);
+ if (r < 0) {
+ FBAnalyzerFree(&fba);
+ return fba;
+ }
+ }
+
+ /* walk array to see if all bits make sense */
for (uint32_t i = 0; i < array_size; i++) {
- SCFree(array[i].set_sids);
- SCFree(array[i].unset_sids);
- SCFree(array[i].isset_sids);
- SCFree(array[i].isnotset_sids);
- SCFree(array[i].toggle_sids);
+ const char *varname = VarNameStoreSetupLookup(i, VAR_TYPE_FLOW_BIT);
+ if (varname == NULL)
+ continue;
+
+ bool to_state = false;
+ if (array[i].state_cnts[DETECT_FLOWBITS_CMD_ISSET] &&
+ array[i].state_cnts[DETECT_FLOWBITS_CMD_SET] == 0) {
+ SCLogDebug("flowbit %s/%u: isset in state, set not in state", varname, i);
+ }
+
+ /* if signature depends on 'stateful' flowbits, then turn the
+ * sig into a stateful sig itself */
+ if (array[i].cnts[DETECT_FLOWBITS_CMD_ISSET] > 0 &&
+ array[i].state_cnts[DETECT_FLOWBITS_CMD_ISSET] == 0 &&
+ array[i].state_cnts[DETECT_FLOWBITS_CMD_SET]) {
+ SCLogDebug("flowbit %s/%u: isset not in state, set in state", varname, i);
+ to_state = true;
+ }
+
+ SCLogDebug("ALL flowbit %s/%u: sets %u toggles %u unsets %u isnotsets %u issets %u",
+ varname, i, array[i].cnts[DETECT_FLOWBITS_CMD_SET],
+ array[i].cnts[DETECT_FLOWBITS_CMD_TOGGLE], array[i].cnts[DETECT_FLOWBITS_CMD_UNSET],
+ array[i].cnts[DETECT_FLOWBITS_CMD_ISNOTSET],
+ array[i].cnts[DETECT_FLOWBITS_CMD_ISSET]);
+ SCLogDebug("STATE flowbit %s/%u: sets %u toggles %u unsets %u isnotsets %u issets %u",
+ varname, i, array[i].state_cnts[DETECT_FLOWBITS_CMD_SET],
+ array[i].state_cnts[DETECT_FLOWBITS_CMD_TOGGLE],
+ array[i].state_cnts[DETECT_FLOWBITS_CMD_UNSET],
+ array[i].state_cnts[DETECT_FLOWBITS_CMD_ISNOTSET],
+ array[i].state_cnts[DETECT_FLOWBITS_CMD_ISSET]);
+ for (uint32_t x = 0; x < array[i].set_sids_idx; x++) {
+ SCLogDebug("SET flowbit %s/%u: SID %u", varname, i,
+ de_ctx->sig_array[array[i].set_sids[x]]->id);
+ }
+ for (uint32_t x = 0; x < array[i].isset_sids_idx; x++) {
+ Signature *s = de_ctx->sig_array[array[i].isset_sids[x]];
+ SCLogDebug("GET flowbit %s/%u: SID %u", varname, i, s->id);
+
+ if (to_state) {
+ s->init_data->init_flags |= SIG_FLAG_INIT_STATE_MATCH;
+ SCLogDebug("made SID %u stateful because it depends on "
+ "stateful rules that set flowbit %s",
+ s->id, varname);
+ }
+ }
}
- SCFree(array);
- return 0;
+ return fba;
}
SCMutex g_flowbits_dump_write_m = SCMUTEX_INITIALIZER;
jb_free(js);
}
+static bool PrefilterFlowbitIsPrefilterable(const Signature *s)
+{
+ SCLogDebug("sid:%u: checking", s->id);
+
+ for (const SigMatch *sm = s->init_data->smlists[DETECT_SM_LIST_MATCH]; sm != NULL;
+ sm = sm->next) {
+ switch (sm->type) {
+ case DETECT_FLOWBITS: {
+ const DetectFlowbitsData *fb = (DetectFlowbitsData *)sm->ctx;
+ if (fb->cmd == DETECT_FLOWBITS_CMD_ISSET) {
+ SCLogDebug("sid:%u: FLOWBITS ISSET can prefilter", s->id);
+ return true;
+ }
+ break;
+ }
+ }
+ }
+ SCLogDebug("sid:%u: no flowbit prefilter", s->id);
+ return false;
+}
+
+/** core flowbit data structure: map a flowbit id to the signatures that need inspecting after it is
+ * found. Part of a rb-tree. */
+typedef struct PrefilterFlowbit {
+ uint32_t id; /**< flowbit id */
+ uint32_t rule_id_size; /**< size in elements of `rule_id` */
+ uint32_t rule_id_cnt; /**< usage in elements of `rule_id` */
+ uint32_t *rule_id; /**< array of signature iid that are part of this prefilter */
+ RB_ENTRY(PrefilterFlowbit) __attribute__((__packed__)) rb;
+} __attribute__((__packed__)) PrefilterFlowbit;
+
+static int PrefilterFlowbitCompare(const PrefilterFlowbit *a, const PrefilterFlowbit *b)
+{
+ if (a->id > b->id)
+ return 1;
+ else if (a->id < b->id)
+ return -1;
+ else
+ return 0;
+}
+
+/** red-black tree prototype for PFB (Prefilter Flow Bits) */
+RB_HEAD(PFB, PrefilterFlowbit);
+RB_PROTOTYPE(PFB, PrefilterFlowbit, rb, PrefilterFlowbitCompare);
+RB_GENERATE(PFB, PrefilterFlowbit, rb, PrefilterFlowbitCompare);
+
+struct PrefilterEngineFlowbits {
+ struct PFB fb_tree;
+};
+
+static void PrefilterFlowbitFree(void *vctx)
+{
+ struct PrefilterEngineFlowbits *ctx = vctx;
+ struct PrefilterFlowbit *rec, *safe = NULL;
+ RB_FOREACH_SAFE (rec, PFB, &ctx->fb_tree, safe) {
+ PFB_RB_REMOVE(&ctx->fb_tree, rec);
+ SCFree(rec->rule_id);
+ SCFree(rec);
+ }
+
+ SCFree(ctx);
+}
+
+static void PrefilterFlowbitMatch(DetectEngineThreadCtx *det_ctx, Packet *p, const void *pectx)
+{
+ struct PrefilterEngineFlowbits *ctx = (struct PrefilterEngineFlowbits *)pectx;
+ SCLogDebug("%" PRIu64 ": ctx %p", p->pcap_cnt, ctx);
+
+ if (p->flow == NULL) {
+ SCReturn;
+ }
+
+ for (GenericVar *gv = p->flow->flowvar; gv != NULL; gv = gv->next) {
+ if (gv->type != DETECT_FLOWBITS)
+ continue;
+
+ PrefilterFlowbit lookup;
+ memset(&lookup, 0, sizeof(lookup));
+ lookup.id = gv->idx;
+ SCLogDebug("flowbit %u", gv->idx);
+
+ PrefilterFlowbit *b = PFB_RB_FIND(&ctx->fb_tree, &lookup);
+ if (b == NULL) {
+ SCLogDebug("flowbit %u not in the tree", lookup.id);
+ } else {
+ SCLogDebug("flowbit %u found in the tree: %u", lookup.id, b->id);
+
+ PrefilterAddSids(&det_ctx->pmq, b->rule_id, b->rule_id_cnt);
+#ifdef DEBUG
+ for (uint32_t x = 0; x < b->rule_id_cnt; x++) {
+ const Signature *s = det_ctx->de_ctx->sig_array[b->rule_id[x]];
+ SCLogDebug("flowbit %u -> sig %u", gv->idx, s->id);
+ }
+#endif
+ }
+ }
+}
+
+static void PrefilterFlowbitPostRuleMatch(
+ DetectEngineThreadCtx *det_ctx, const void *pectx, Packet *p, Flow *f)
+{
+ struct PrefilterEngineFlowbits *ctx = (struct PrefilterEngineFlowbits *)pectx;
+ SCLogDebug("%" PRIu64 ": ctx %p", p->pcap_cnt, ctx);
+
+ if (p->flow == NULL) {
+ SCReturn;
+ }
+
+ for (uint32_t i = 0; i < det_ctx->post_rule_work_queue.len; i++) {
+ const PostRuleMatchWorkQueueItem *w = &det_ctx->post_rule_work_queue.q[i];
+ if (w->sm_type != DETECT_FLOWBITS)
+ continue;
+
+ PrefilterFlowbit lookup;
+ memset(&lookup, 0, sizeof(lookup));
+ lookup.id = w->value;
+
+ PrefilterFlowbit *b = PFB_RB_FIND(&ctx->fb_tree, &lookup);
+ if (b == NULL) {
+ SCLogDebug("flowbit %u not in the tree", lookup.id);
+ } else {
+ SCLogDebug("flowbit %u found in the tree: %u. Adding %u sids", lookup.id, b->id,
+ b->rule_id_cnt);
+ PrefilterAddSids(&det_ctx->pmq, b->rule_id, b->rule_id_cnt);
+#ifdef DEBUG
+ // SCLogDebug("b %u", b->rule_id_cnt);
+ for (uint32_t x = 0; x < b->rule_id_cnt; x++) {
+ Signature *s = det_ctx->de_ctx->sig_array[b->rule_id[x]];
+ SCLogDebug("flowbit %u -> sig %u (triggered by %u)", w->value, s->id,
+ det_ctx->de_ctx->sig_array[w->id]->id);
+ }
+#endif
+ }
+ }
+}
+
+#define BLOCK_SIZE 8
+
+static int AddBitAndSid(
+ struct PrefilterEngineFlowbits *ctx, const Signature *s, const uint32_t flowbit_id)
+{
+ PrefilterFlowbit x;
+ memset(&x, 0, sizeof(x));
+ x.id = flowbit_id;
+
+ PrefilterFlowbit *pfb = PFB_RB_FIND(&ctx->fb_tree, &x);
+ if (pfb == NULL) {
+ PrefilterFlowbit *add = SCCalloc(1, sizeof(*add));
+ if (add == NULL)
+ return -1;
+
+ add->id = flowbit_id;
+ add->rule_id = SCCalloc(1, BLOCK_SIZE * sizeof(uint32_t));
+ if (add->rule_id == NULL) {
+ SCFree(add);
+ return -1;
+ }
+ add->rule_id_size = BLOCK_SIZE;
+ add->rule_id_cnt = 1;
+ add->rule_id[0] = s->num;
+
+ PrefilterFlowbit *res = PFB_RB_INSERT(&ctx->fb_tree, add);
+ SCLogDebug("not found, so added (res %p)", res);
+ if (res != NULL) {
+ // duplicate, shouldn't be possible after the FIND above
+ BUG_ON(1);
+ return -1;
+ }
+ } else {
+ SCLogDebug("found! pfb %p id %u", pfb, pfb->id);
+
+ if (pfb->rule_id_cnt < pfb->rule_id_size) {
+ pfb->rule_id[pfb->rule_id_cnt++] = s->num;
+ } else {
+ uint32_t *ptr =
+ SCRealloc(pfb->rule_id, (pfb->rule_id_size + BLOCK_SIZE) * sizeof(uint32_t));
+ if (ptr == NULL) {
+ // memory stays in the tree
+ return -1;
+ }
+ pfb->rule_id = ptr;
+ pfb->rule_id_size += BLOCK_SIZE;
+ pfb->rule_id[pfb->rule_id_cnt++] = s->num;
+ }
+ }
+ return 0;
+}
+
+static int AddBitsAndSid(const DetectEngineCtx *de_ctx, struct PrefilterEngineFlowbits *ctx,
+ const DetectFlowbitsData *fb, const Signature *s)
+{
+ if (fb->or_list_size == 0) {
+ if (AddBitAndSid(ctx, s, fb->idx) < 0) {
+ return -1;
+ }
+ } else {
+ for (uint8_t i = 0; i < fb->or_list_size; i++) {
+ SCLogDebug("flowbit OR: bit %u", fb->or_list[i]);
+ if (AddBitAndSid(ctx, s, fb->or_list[i]) < 0) {
+ return -1;
+ }
+ }
+ }
+ return 0;
+}
+
+static uint32_t NextMultiple(const uint32_t v, const uint32_t m)
+{
+ return v + (m - v % m);
+}
+
+/** \internal
+ * \brief adds sids for 'isset' prefilter flowbits
+ * \retval int 1 if we added sid(s), 0 if we didn't, -1 on error */
+// TODO skip sids that aren't set by this sgh
+// TODO skip sids that doesn't have a isset in the same direction
+static int AddIssetSidsForBit(const DetectEngineCtx *de_ctx, const struct FBAnalyzer *fba,
+ const DetectFlowbitsData *fb, PrefilterFlowbit *add)
+{
+ int added = 0;
+ for (uint32_t i = 0; i < fba->array[fb->idx].isset_sids_idx; i++) {
+ const uint32_t sig_iid = fba->array[fb->idx].isset_sids[i];
+ const Signature *s = de_ctx->sig_array[sig_iid];
+ SCLogDebug("flowbit: %u => considering sid %u (iid:%u)", fb->idx, s->id, s->num);
+
+ /* Skip sids that aren't prefilter. These would just run all the time. */
+ if (s->init_data->prefilter_sm == NULL ||
+ s->init_data->prefilter_sm->type != DETECT_FLOWBITS) {
+#ifdef DEBUG
+ const char *name = s->init_data->prefilter_sm
+ ? sigmatch_table[s->init_data->prefilter_sm->type].name
+ : "none";
+ SCLogDebug("flowbit: %u => rejected sid %u (iid:%u). No prefilter or prefilter not "
+ "flowbits (%p, %s, %d)",
+ fb->idx, s->id, sig_iid, s->init_data->prefilter_sm, name,
+ s->init_data->prefilter_sm ? s->init_data->prefilter_sm->type : -1);
+#endif
+ continue;
+ }
+
+ /* only add sids that match our bit */
+ const DetectFlowbitsData *fs_fb =
+ (const DetectFlowbitsData *)s->init_data->prefilter_sm->ctx;
+ if (fs_fb->idx != fb->idx) {
+ SCLogDebug(
+ "flowbit: %u => rejected sid %u (iid:%u). Sig prefilters on different bit %u",
+ fb->idx, s->id, sig_iid, fs_fb->idx);
+ continue;
+ }
+
+ bool dup = false;
+ for (uint32_t x = 0; x < add->rule_id_cnt; x++) {
+ if (add->rule_id[x] == sig_iid) {
+ dup = true;
+ }
+ }
+
+ if (!dup) {
+ if (add->rule_id_cnt < add->rule_id_size) {
+ add->rule_id[add->rule_id_cnt++] = sig_iid;
+ } else {
+ uint32_t *ptr = SCRealloc(
+ add->rule_id, (add->rule_id_size + BLOCK_SIZE) * sizeof(uint32_t));
+ if (ptr == NULL) {
+ return -1;
+ }
+ add->rule_id = ptr;
+ add->rule_id_size += BLOCK_SIZE;
+ add->rule_id[add->rule_id_cnt++] = sig_iid;
+ }
+ added = 1;
+ SCLogDebug("flowbit: %u => accepted sid %u (iid:%u)", fb->idx, s->id, sig_iid);
+ }
+ }
+ return added;
+}
+
+/* TODO shouldn't add sids for which Signature::num is < our num. Is this possible after sorting? */
+
+/** \brief For set/toggle flowbits, build "set" post-rule-match engine
+ *
+ * For set/toggle flowbits, a special post-rule-match engine is constructed
+ * to update the running match array during rule matching.
+ */
+static int AddBitSetToggle(const DetectEngineCtx *de_ctx, struct FBAnalyzer *fba,
+ struct PrefilterEngineFlowbits *ctx, const DetectFlowbitsData *fb, const Signature *s)
+{
+ PrefilterFlowbit x;
+ memset(&x, 0, sizeof(x));
+ x.id = fb->idx;
+ PrefilterFlowbit *pfb = PFB_RB_FIND(&ctx->fb_tree, &x);
+ if (pfb == NULL) {
+ PrefilterFlowbit *add = SCCalloc(1, sizeof(*add));
+ if (add == NULL)
+ return -1;
+
+ add->id = fb->idx;
+ add->rule_id_size = NextMultiple(fba->array[fb->idx].isset_sids_idx, BLOCK_SIZE);
+ add->rule_id = SCCalloc(1, add->rule_id_size * sizeof(uint32_t));
+ if (add->rule_id == NULL) {
+ SCFree(add);
+ return -1;
+ }
+
+ if (AddIssetSidsForBit(de_ctx, fba, fb, add) != 1) {
+ SCLogDebug("no sids added");
+ SCFree(add->rule_id);
+ SCFree(add);
+ return 0;
+ }
+ PrefilterFlowbit *res = PFB_RB_INSERT(&ctx->fb_tree, add);
+ SCLogDebug("not found, so added (res %p)", res);
+ BUG_ON(res != NULL); // TODO if res != NULL we have a duplicate which should be impossible
+ } else {
+ SCLogDebug("found! pfb %p id %u", pfb, pfb->id);
+
+ int r = AddIssetSidsForBit(de_ctx, fba, fb, pfb);
+ if (r < 0) {
+ return -1;
+ } else if (r == 0) {
+ SCLogDebug("no sids added");
+ return 0;
+ }
+ }
+ return 1;
+}
+
+/** \brief build flowbit prefilter state(s)
+ *
+ * Build "set" and "isset" states.
+ *
+ * For each flowbit "isset" in the sgh, we need to check:
+ * 1. is it supported
+ * 2. is prefilter enabled
+ * 3. does it match in the same dir or only opposing dir
+ */
+static int PrefilterSetupFlowbits(DetectEngineCtx *de_ctx, SigGroupHead *sgh)
+{
+ if (sgh == NULL)
+ return 0;
+
+ SCLogDebug("sgh %p: setting up prefilter", sgh);
+ struct PrefilterEngineFlowbits *isset_ctx = NULL;
+ struct PrefilterEngineFlowbits *set_ctx = NULL;
+
+ struct FBAnalyzer fb_analysis = DetectFlowbitsAnalyzeForGroup(de_ctx, sgh);
+ if (fb_analysis.array == NULL)
+ goto error;
+
+ for (uint32_t i = 0; i < sgh->init->sig_cnt; i++) {
+ Signature *s = sgh->init->match_array[i];
+ if (s == NULL)
+ continue;
+
+ SCLogDebug("checking sid %u", s->id);
+
+ /* first build the 'set' state */
+ for (SigMatch *sm = s->init_data->smlists[DETECT_SM_LIST_POSTMATCH]; sm != NULL;
+ sm = sm->next) {
+ if (sm->type != DETECT_FLOWBITS) {
+ SCLogDebug("skip non flowbits sm");
+ continue;
+ }
+
+ DetectFlowbitsData *fb = (DetectFlowbitsData *)sm->ctx;
+ if (fb->cmd == DETECT_FLOWBITS_CMD_SET) {
+ SCLogDebug(
+ "DETECT_SM_LIST_POSTMATCH: sid %u DETECT_FLOWBITS set %u", s->id, fb->idx);
+ } else if (fb->cmd == DETECT_FLOWBITS_CMD_TOGGLE) {
+ SCLogDebug("DETECT_SM_LIST_POSTMATCH: sid %u DETECT_FLOWBITS toggle %u", s->id,
+ fb->idx);
+ } else {
+ SCLogDebug("unsupported flowbits setting");
+ continue;
+ }
+
+ if (fb_analysis.array[fb->idx].isnotset_sids_idx ||
+ fb_analysis.array[fb->idx].unset_sids_idx) {
+ SCLogDebug("flowbit %u not supported: unset in use", fb->idx);
+ continue;
+ }
+
+ if (set_ctx == NULL) {
+ set_ctx = SCCalloc(1, sizeof(*set_ctx));
+ if (set_ctx == NULL)
+ goto error;
+ }
+
+ SCLogDebug("setting up sets/toggles for sid %u", s->id);
+ if (AddBitSetToggle(de_ctx, &fb_analysis, set_ctx, fb, s) == 1) {
+ // flag the set/toggle to trigger the post-rule match logic
+ SCLogDebug("set up sets/toggles for sid %u", s->id);
+ fb->post_rule_match_prefilter = true;
+ }
+
+ // TODO don't add for sigs that don't have isset in this sgh. Reasoning:
+ // prefilter post match logic only makes sense in the same dir as otherwise
+ // the regular 'isset' logic can simply run with the regular prefilters
+ // before the rule loop
+ }
+
+ /* next, build the 'isset' state */
+ if (s->init_data->prefilter_sm == NULL ||
+ s->init_data->prefilter_sm->type != DETECT_FLOWBITS) {
+ SCLogDebug("no prefilter or prefilter not flowbits");
+ continue;
+ }
+
+ const DetectFlowbitsData *fb = (DetectFlowbitsData *)s->init_data->prefilter_sm->ctx;
+ if (fb_analysis.array[fb->idx].isnotset_sids_idx ||
+ fb_analysis.array[fb->idx].unset_sids_idx) {
+ SCLogDebug("flowbit %u not supported: toggle or unset in use", fb->idx);
+ s->init_data->prefilter_sm = NULL;
+ s->flags &= ~SIG_FLAG_PREFILTER;
+ continue;
+ }
+
+ SCLogDebug("isset: adding sid %u, flowbit %u", s->id, fb->idx);
+
+ if (isset_ctx == NULL) {
+ isset_ctx = SCCalloc(1, sizeof(*isset_ctx));
+ if (isset_ctx == NULL)
+ goto error;
+ }
+ if (AddBitsAndSid(de_ctx, isset_ctx, fb, s) < 0) {
+ goto error;
+ }
+ }
+
+ /* finally, register the states with their engines */
+ static const char *g_prefilter_flowbits_isset = "flowbits:isset";
+ if (isset_ctx != NULL) {
+ PrefilterAppendEngine(de_ctx, sgh, PrefilterFlowbitMatch, SIG_MASK_REQUIRE_FLOW, isset_ctx,
+ PrefilterFlowbitFree, g_prefilter_flowbits_isset);
+ SCLogDebug("isset: added prefilter engine");
+
+ if (set_ctx != NULL && !RB_EMPTY(&set_ctx->fb_tree)) {
+ static const char *g_prefilter_flowbits_set = "flowbits:set";
+ PrefilterAppendPostRuleEngine(de_ctx, sgh, PrefilterFlowbitPostRuleMatch, set_ctx,
+ PrefilterFlowbitFree, g_prefilter_flowbits_set);
+ SCLogDebug("set/toggle: added prefilter engine");
+ } else {
+ if (set_ctx) {
+ PrefilterFlowbitFree(set_ctx);
+ }
+ SCLogDebug("set/toggle: NO prefilter engine added");
+ }
+ } else if (set_ctx != NULL) {
+ PrefilterFlowbitFree(set_ctx);
+ }
+ FBAnalyzerFree(&fb_analysis);
+ return 0;
+
+error:
+ if (set_ctx) {
+ PrefilterFlowbitFree(set_ctx);
+ }
+ if (isset_ctx) {
+ PrefilterFlowbitFree(isset_ctx);
+ }
+ FBAnalyzerFree(&fb_analysis);
+ return -1;
+}
+
#ifdef UNITTESTS
static int FlowBitsTestParse01(void)
projects / thirdparty / suricata.git / commitdiff
