From: Russ Combs (rucombs) Date: Thu, 17 Aug 2017 17:59:29 +0000 (-0400) Subject: Merge pull request #990 in SNORT/snort3 from port_reload_performance_fixes_2 to master X-Git-Tag: 3.0.0-240~65 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=b4ec5e9e2660a2b5db91ae5299fc7bde7c4292ab;p=thirdparty%2Fsnort3.git Merge pull request #990 in SNORT/snort3 from port_reload_performance_fixes_2 to master Squashed commit of the following: commit 8c1d83e9188cc38480fbfc99c363608ecd4ca93b Author: Victor Roemer Date: Thu Aug 17 08:58:02 2017 -0400 snort: Use Debug::enable for runtime check commit 1c5791908d3e286db0d25b57a1c9d556aad14883 Author: Victor Roemer Date: Wed Aug 16 14:36:05 2017 -0400 snort: more review comments commit b6c48d7096add2a9cc0df8af0b7877996b7ab470 Author: Victor Roemer Date: Wed Aug 16 09:20:15 2017 -0400 snort3: update based on review comments commit e1bd664d7cfd6766e98ddb39e715ceab2879d678 Author: Victor Roemer Date: Tue Aug 15 15:57:05 2017 -0400 snort: move debug code into the ifdef DEBUG commit 1b1350a638cac3355ba0bb80d99298e8f7fd59e6 Author: Victor Roemer Date: Fri Aug 11 15:45:54 2017 -0400 search_engines: only add state to queue once commit 16069bfce5ffe09d5a52531cf32da305cf28d8d1 Author: Victor Roemer Date: Thu Aug 10 13:31:09 2017 -0400 snort: remove duplicate function declaration commit 42a2156b3b08232fd6032a7c27d3558fb29dcee2 Author: Victor Roemer Date: Mon Aug 7 18:21:54 2017 -0400 snort: port the optimized port table compilation from 2.9.12 Incomplete port of port, some things require reworking since PortObjectItem changed so much in Snort3 --- diff --git a/src/ports/port_table.cc b/src/ports/port_table.cc index 25041b407..fb0ba6242 100644 --- a/src/ports/port_table.cc +++ b/src/ports/port_table.cc @@ -513,6 +513,99 @@ static PortObject2* PortTableCompileMergePortObjectList2( return ponew; } +static inline void add_port_object(Port port, PortObject* po, SF_LIST** parray) +{ + if ( !parray[port] ) + { + parray[port] = sflist_new(); + assert(parray[port]); + } + + if ( parray[port]->tail && parray[port]->tail->ndata == po ) + return; + + sflist_add_tail(parray[port], po); +} + +// Update port object lists +static inline void update_port_lists(PortObject* po, SF_LIST** parray) +{ + bool not_flag_set = false; + + PortObjectItem* poi; + SF_LNODE* lpos; + for ( poi = (PortObjectItem*)sflist_first(po->item_list, &lpos); + poi; + poi = (PortObjectItem*)sflist_next(&lpos) ) + { + if( poi->any()) + return; + + else if( poi->one() ) + { + if (poi->negate ) + { + not_flag_set = true; + break; + } + + add_port_object(poi->lport, po, parray); + } + else + { + if (poi->negate ) + { + not_flag_set = true; + break; + } + + for( int port = poi->lport; port <= poi->hport; port++ ) + { + add_port_object(port, po, parray); + } + } + + add_port_object(poi->lport, po, parray); + } + + if (not_flag_set) + { + for( int port = 0; port < SFPO_MAX_PORTS; port++ ) + { + add_port_object(port, po, parray); + } + } +} + +// Create optimized port lists per port +static inline SF_LIST** create_port_lists(PortTable* p) +{ + SF_LIST** parray = (SF_LIST**)snort_calloc(sizeof(SF_LIST*), SFPO_MAX_PORTS); + assert(parray); + + PortObject* po; + SF_LNODE* lpos; + for ( po = (PortObject*)sflist_first(p->pt_polist, &lpos); + po; + po = (PortObject*)sflist_next(&lpos) ) + { + update_port_lists(po, parray); + } + + return parray; +} + +static inline void delete_port_lists(SF_LIST** parray) +{ + for ( int port = 0; port < SFPO_MAX_PORTS; port++ ) + { + SF_LIST* list = parray[port]; + if (list) + sflist_free(list); + } +} + + static int PortTableCompileMergePortObjects(PortTable* p) { DebugMessage(DEBUG_PORTLISTS, "***\n***Merging PortObjects->PortObjects2\n***\n"); @@ -541,6 +634,8 @@ static int PortTableCompileMergePortObjects(PortTable* p) SF_LIST* plx_list = sflist_new(); + SF_LIST** optimized_pl = create_port_lists(p); + /* * For each port, merge rules from all port objects that touch the port * into an optimal object, that may be shared with other ports. @@ -554,16 +649,13 @@ static int PortTableCompileMergePortObjects(PortTable* p) PortObject* po; SF_LNODE* lpos; - for (po = (PortObject*)sflist_first(p->pt_polist, &lpos); + for (po = (PortObject*)sflist_first(optimized_pl[i], &lpos); po; po = (PortObject*)sflist_next(&lpos) ) { - if ( PortObjectHasPort (po, i) ) + if (pol_cnt < SFPO_MAX_LPORTS ) { - if ( pol_cnt < SFPO_MAX_LPORTS ) - { - pol[ pol_cnt++ ] = po; - } + pol[ pol_cnt++ ] = po; } } p->pt_port_object[i] = 0; @@ -593,6 +685,9 @@ static int PortTableCompileMergePortObjects(PortTable* p) DEBUG_WRAP(DebugMessage(DEBUG_PORTLISTS, "\n"); fflush(stdout); ); } + delete_port_lists(optimized_pl); + snort_free(optimized_pl); + /* * Normalize the Ports so they indicate only the ports that * reference the composite port object @@ -689,6 +784,31 @@ static int PortTableCompileMergePortObjects(PortTable* p) } #ifdef DEBUG +/* Verify all rules in 'po' list are in 'po2' hash + * + * return 0 - OK + * !0 - a rule in po is not in po2 + */ +static int _po2_include_po_rules(PortObject2* po2, PortObject* po) +{ + SF_LNODE* rpos; + + /* get each rule in po */ + for ( int* pid = (int*)sflist_first(po->rule_list, &rpos); + pid; + pid = (int*)sflist_next(&rpos) ) + { + /* find it in po2 */ + int* id = (int*)sfghash_find(po2->rule_hash, pid); + + /* make sure it's in po2 */ + if ( !id ) + return 1; /* error */ + } + + return 0; +} + // consistency check - part 1 // make sure each port is only in one composite port object static bool PortTableConsistencyCheck(PortTable* p) @@ -730,30 +850,6 @@ static bool PortTableConsistencyCheck(PortTable* p) return true; } -/* - * Verify all rules in 'po' list are in 'po2' hash - * return 0 - OK; !0 - a rule in po is not in po2 - */ -static int _po2_include_po_rules(PortObject2* po2, PortObject* po) -{ - SF_LNODE* rpos; - - /* get each rule in po */ - for ( int* pid = (int*)sflist_first(po->rule_list, &rpos); - pid; - pid = (int*)sflist_next(&rpos) ) - { - /* find it in po2 */ - int* id = (int*)sfghash_find(po2->rule_hash, pid); - - /* make sure it's in po2 */ - if ( !id ) - return 1; /* error */ - } - - return 0; -} - // consistency check - part 2 /* * This phase checks the Input port object rules/ports against @@ -937,6 +1033,11 @@ int PortTableCompile(PortTable* p) assert(PortTableConsistencyCheck2(p)); #endif +#ifdef DEBUG_MSGS + if ( Debug::enabled(DEBUG_PORTLISTS) ) + PortTablePrintPortGroups(p); +#endif + return 0; } diff --git a/src/ports/port_utils.cc b/src/ports/port_utils.cc index 68d939884..4e8fdab37 100644 --- a/src/ports/port_utils.cc +++ b/src/ports/port_utils.cc @@ -91,14 +91,9 @@ int PortObjectBits(PortBitSet& parray, PortObject* po) /* A pure Not list */ if ( po->item_list->count == not_cnt ) { - int i; - /* enable all of the ports */ - for (i=0; iitem_list,&pos); @@ -111,11 +106,11 @@ int PortObjectBits(PortBitSet& parray, PortObject* po) if ( poi->any() ) continue; - for ( int k = poi->lport; k <= poi->hport; k++ ) + for ( int i = poi->lport; i <= poi->hport; i++ ) { - if ( parray[k] ) + if ( parray[i] ) cnt--; - parray[k] = 0; + parray[i] = 0; } } } diff --git a/src/search_engines/acsmx2.cc b/src/search_engines/acsmx2.cc index 1e1f8eff1..5a2f5cc7f 100644 --- a/src/search_engines/acsmx2.cc +++ b/src/search_engines/acsmx2.cc @@ -343,6 +343,26 @@ static void AC_FREE_DFA(void* p, int n, int sizeofstate) } } + +/* + * Get Next State-NFA, using direct index to speed up search + */ +static int List_GetNextStateOpt( ACSM_STRUCT2 * acsm, + trans_node_t **acsmTransTableOpt, int state, int input ) +{ + int index = state * acsm->acsmAlphabetSize + input; + trans_node_t * t = acsmTransTableOpt[index]; + + if ( t ) + return t->next_state; + + if( state == 0 ) + return 0; + + return ACSM_FAIL_STATE2; /* Fail state ??? */ +} + + /* * Get Next State-NFA */ @@ -384,6 +404,40 @@ static int List_GetNextState2(ACSM_STRUCT2* acsm, int state, int input) return 0; /* default state */ } +/* + * Put Next State - Head insertion, and transition updates + */ +static int List_PutNextStateOpt( ACSM_STRUCT2 * acsm, trans_node_t **acsmTransTableOpt, + int state, int input, int next_state ) +{ + int index = state * acsm->acsmAlphabetSize + input; + + trans_node_t *t = acsmTransTableOpt[index]; + + if ( t ) + { + t->next_state = next_state; + return 0; + } + + /* Definitely not an existing transition - add it */ + trans_node_t * tnew = (trans_node_t*)AC_MALLOC(sizeof(trans_node_t), + ACSM2_MEMORY_TYPE__TRANSTABLE); + + if( !tnew ) + return -1; + + tnew->key = input; + tnew->next_state = next_state; + tnew->next = acsm->acsmTransTable[state]; + acsm->acsmTransTable[state] = tnew; + acsm->acsmNumTrans++; + + acsmTransTableOpt[index] = tnew; + + return 0; +} + /* * Put Next State - Head insertion, and transition updates */ @@ -452,39 +506,15 @@ static int List_FreeTransTable(ACSM_STRUCT2* acsm) return 0; } -/* -* -*/ -/* -static int List_FreeList( trans_node_t * t ) -{ - int tcnt=0; - - trans_node_t *p; - - while( t ) - { - p = t->next; - snort_free(t); - t = p; - acsm2_total_memory -= sizeof(trans_node_t); - tcnt++; - } - - return tcnt; -} -*/ /* * Converts row of states from list to a full vector format */ -static int List_ConvToFull(ACSM_STRUCT2* acsm, acstate_t state, acstate_t* full) +static inline int List_ConvToFull(ACSM_STRUCT2* acsm, acstate_t state, acstate_t* full) { int tcnt = 0; trans_node_t* t = acsm->acsmTransTable[state]; - memset(full, 0, acsm->sizeofstate * acsm->acsmAlphabetSize); - if (t == NULL) return 0; @@ -606,10 +636,12 @@ static void Build_NFA(ACSM_STRUCT2* acsm) { acstate_t* FailState = acsm->acsmFailState; ACSM_PATTERN2** MatchList = acsm->acsmMatchList; - ACSM_PATTERN2* mlist,* px; + ACSM_PATTERN2* mlist, * px; std::list queue; + bool* queue_array = (bool*) snort_calloc( acsm->acsmNumStates, sizeof(bool) ); + /* Add the state 0 transitions 1st, the states at depth 1, fail to state 0 */ for (int i = 0; i < acsm->acsmAlphabetSize; i++) { @@ -617,7 +649,11 @@ static void Build_NFA(ACSM_STRUCT2* acsm) if ( s ) { - queue.push_back(s); + if ( !queue_array[s] ) + { + queue.push_back(s); + queue_array[s] = true; + } FailState[s] = 0; } } @@ -625,6 +661,8 @@ static void Build_NFA(ACSM_STRUCT2* acsm) /* Build the fail state successive layer of transitions */ for ( auto r : queue ) { + queue_array[r] = false; + /* Find Final States for any Failure */ for (int i = 0; i < acsm->acsmAlphabetSize; i++) { @@ -633,14 +671,17 @@ static void Build_NFA(ACSM_STRUCT2* acsm) if ( (acstate_t)s != ACSM_FAIL_STATE2 ) { - queue.push_back(s); + if ( !queue_array[s] ) + { + queue.push_back(s); + queue_array[s] = true; + } int fs = FailState[r]; /* * Locate the next valid state for 'i' starting at fs */ - while ((acstate_t)(next = List_GetNextState(acsm,fs,i)) - == ACSM_FAIL_STATE2 ) + while ((acstate_t)(next = List_GetNextState(acsm,fs,i)) == ACSM_FAIL_STATE2 ) { fs = FailState[fs]; } @@ -668,6 +709,8 @@ static void Build_NFA(ACSM_STRUCT2* acsm) } } } + + snort_free(queue_array); } /* @@ -679,37 +722,66 @@ static void Convert_NFA_To_DFA(ACSM_STRUCT2* acsm) acstate_t* FailState = acsm->acsmFailState; std::list queue; + bool* queue_array = (bool*) snort_calloc( acsm->acsmNumStates, sizeof(bool) ); + trans_node_t** acsmTransTableOpt = (trans_node_t**) + snort_calloc( acsm->acsmAlphabetSize * acsm->acsmNumStates, sizeof(trans_node_t*) ); + + for ( int i = 0; i < acsm->acsmNumStates; i++ ) + { + trans_node_t* t = acsm->acsmTransTable[i]; + while ( t ) + { + int index = i * acsm->acsmAlphabetSize + t->key; + acsmTransTableOpt[index] = t; + t = t->next; + } + } /* Add the state 0 transitions 1st */ for (int i=0; iacsmAlphabetSize; i++) { - if ( int s = List_GetNextState(acsm,0,i) ) - queue.push_back(s); + if ( int s = List_GetNextStateOpt(acsm, acsmTransTableOpt, 0, i) ) + { + if ( !queue_array[s] ) + { + queue.push_back(s); + queue_array[s] = true; + } + } } /* Start building the next layer of transitions */ for ( auto r : queue ) { + queue_array[r] = false; + /* Process this states layer */ for (int i = 0; i < acsm->acsmAlphabetSize; i++) { - int s = List_GetNextState(acsm,r,i); + int s = List_GetNextStateOpt(acsm, acsmTransTableOpt, r, i); - if ( (acstate_t)s != ACSM_FAIL_STATE2 && s!= 0) + if ( (acstate_t)s != ACSM_FAIL_STATE2 && s != 0 ) { - queue.push_back(s); + if ( !queue_array[s] ) + { + queue.push_back(s); + queue_array[s] = true; + } } else { - cFailState = List_GetNextState(acsm,FailState[r],i); + cFailState = List_GetNextStateOpt(acsm, acsmTransTableOpt, FailState[r], i); if ( cFailState != 0 && (acstate_t)cFailState != ACSM_FAIL_STATE2 ) { - List_PutNextState(acsm,r,i,cFailState); + List_PutNextStateOpt(acsm, acsmTransTableOpt, r, i, cFailState); } } } } + + snort_free(queue_array); + snort_free(acsmTransTableOpt); } /* @@ -794,6 +866,7 @@ static int Conv_Full_DFA_To_Sparse(ACSM_STRUCT2* acsm) { cnt=0; + memset(full, 0, acsm->sizeofstate * acsm->acsmAlphabetSize); List_ConvToFull(acsm, (acstate_t)k, full); for (i = 0; i < acsm->acsmAlphabetSize; i++) @@ -862,6 +935,7 @@ static int Conv_Full_DFA_To_Banded(ACSM_STRUCT2* acsm) for (k=0; kacsmNumStates; k++) { + memset(full, 0, acsm->sizeofstate * acsm->acsmAlphabetSize); List_ConvToFull(acsm, (acstate_t)k, full); first=-1; @@ -981,6 +1055,7 @@ static int Conv_Full_DFA_To_SparseBands(ACSM_STRUCT2* acsm) for (k=0; kacsmNumStates; k++) { + memset(full, 0, acsm->sizeofstate * acsm->acsmAlphabetSize); List_ConvToFull(acsm, (acstate_t)k, full); nbands = calcSparseBands(full, band_begin, band_end, acsm->acsmAlphabetSize, zcnt);