From: Anoop Saldanha Date: Sat, 13 Oct 2012 18:13:20 +0000 (+0530) Subject: update client/server/http_header to use a different form of X-Git-Tag: suricata-1.4beta3~76 X-Git-Url: http://git.ipfire.org/gitweb.cgi?a=commitdiff_plain;h=b0e20a486c52b949f6c92cdcd6f3279670a84fa6;p=thirdparty%2Fsuricata.git update client/server/http_header to use a different form of buffering/buffer_retrieval. Now it happens per tx, based on tx id. Also notice a perf improvement with this. --- diff --git a/src/detect-engine-hcbd.c b/src/detect-engine-hcbd.c index a8d6b50668..de326f21e5 100644 --- a/src/detect-engine-hcbd.c +++ b/src/detect-engine-hcbd.c @@ -58,6 +58,161 @@ #define BODY_SCAN_WINDOW 4096 #define BODY_MINIMAL_SIZE 32768 +#define BUFFER_STEP 50 + +static uint8_t *DetectEngineHCBDGetBufferForTX(int tx_id, + DetectEngineCtx *de_ctx, + DetectEngineThreadCtx *det_ctx, + Flow *f, HtpState *htp_state, + uint8_t flags, + uint32_t *buffer_len) +{ +#define HCBDCreateSpace(det_ctx, size) do { \ + if (size > det_ctx->hcbd_buffers_size) { \ + det_ctx->hcbd = SCRealloc(det_ctx->hcbd, (det_ctx->hcbd_buffers_size + BUFFER_STEP) * sizeof(HttpReassembledBody)); \ + if (det_ctx->hcbd == NULL) { \ + det_ctx->hcbd_buffers_size = 0; \ + det_ctx->hcbd_buffers_list_len = 0; \ + goto end; \ + } \ + memset(det_ctx->hcbd + det_ctx->hcbd_buffers_size, 0, BUFFER_STEP * sizeof(HttpReassembledBody)); \ + det_ctx->hcbd_buffers_size += BUFFER_STEP; \ + } \ + for (int i = det_ctx->hcbd_buffers_list_len; i < (size); i++) { \ + det_ctx->hcbd[i].buffer_len = 0; \ + det_ctx->hcbd[i].offset = 0; \ + } \ + } while (0) + + int index = 0; + uint8_t *buffer = NULL; + *buffer_len = 0; + + if (det_ctx->hcbd_buffers_list_len == 0) { + HCBDCreateSpace(det_ctx, 1); + index = 0; + } else { + if ((tx_id - det_ctx->hcbd_start_tx_id) < det_ctx->hcbd_buffers_list_len) { + if (det_ctx->hcbd[(tx_id - det_ctx->hcbd_start_tx_id)].buffer_len != 0) { + *buffer_len = det_ctx->hcbd[(tx_id - det_ctx->hcbd_start_tx_id)].buffer_len; + return det_ctx->hcbd[(tx_id - det_ctx->hcbd_start_tx_id)].buffer; + } + } else { + HCBDCreateSpace(det_ctx, (tx_id - det_ctx->hcbd_start_tx_id) + 1); + } + index = (tx_id - det_ctx->hcbd_start_tx_id); + } + + if (det_ctx->hcbd_buffers_list_len == 0) { + det_ctx->hcbd_start_tx_id = tx_id; + } + det_ctx->hcbd_buffers_list_len++; + + htp_tx_t *tx = list_get(htp_state->connp->conn->transactions, tx_id); + if (tx == NULL) { + SCLogDebug("no tx"); + goto end; + } + + HtpTxUserData *htud = (HtpTxUserData *)htp_tx_get_user_data(tx); + if (htud == NULL) { + SCLogDebug("no htud"); + goto end; + } + + /* no new data */ + if (htud->request_body.body_inspected == htud->request_body.content_len_so_far) { + SCLogDebug("no new data"); + goto end; + } + + HtpBodyChunk *cur = htud->request_body.first; + if (cur == NULL) { + SCLogDebug("No http chunks to inspect for this transacation"); + goto end; + } + + /* in case of chunked transfer encoding, we don't have the length + * of the request body until we see a chunk with length 0. This + * doesn't let us use the request body callback function to + * figure out the end of request body. Instead we do it here. If + * the length is 0, and we have already seen content, it indicates + * chunked transfer. We also check if the parser has truly seen + * the last chunk by checking the progress state for the + * transaction. If we are done parsing all the chunks, we would + * have it set to something other than TX_PROGRESS_REQ_BODY. + * Either ways we should be moving away from buffering in the end + * and running content validation on this buffer type of architecture + * to a stateful inspection, where we can inspect body chunks as and + * when they come */ + if (htud->request_body.content_len == 0) { + if ((htud->request_body.content_len_so_far > 0) && + tx->progress != TX_PROGRESS_REQ_BODY) { + /* final length of the body */ + htud->tsflags |= HTP_REQ_BODY_COMPLETE; + } + } + + if (flags & STREAM_EOF) { + htud->tsflags |= HTP_REQ_BODY_COMPLETE; + } + + /* inspect the body if the transfer is complete or we have hit + * our body size limit */ + if (htud->request_body.content_len_so_far < BODY_MINIMAL_SIZE && + !(htud->tsflags & HTP_REQ_BODY_COMPLETE)) { + SCLogDebug("we still haven't seen the entire request body. " + "Let's defer body inspection till we see the " + "entire body."); + goto end; + } + + int first = 1; + while (cur != NULL) { + /* see if we can filter out chunks */ + if (htud->request_body.body_inspected > 0) { + if (cur->stream_offset < htud->request_body.body_inspected) { + if ((htud->request_body.body_inspected - cur->stream_offset) > BODY_SCAN_WINDOW) { + cur = cur->next; + continue; + } else { + /* include this one */ + } + } else { + /* include this one */ + } + } + + if (first) { + det_ctx->hcbd[index].offset = cur->stream_offset; + first = 0; + } + + /* see if we need to grow the buffer */ + if (det_ctx->hcbd[index].buffer == NULL || (det_ctx->hcbd[index].buffer_len + cur->len) > det_ctx->hcbd[index].buffer_size) { + det_ctx->hcbd[index].buffer_size += cur->len * 2; + + if ((det_ctx->hcbd[index].buffer = SCRealloc(det_ctx->hcbd[index].buffer, det_ctx->hcbd[index].buffer_size)) == NULL) { + det_ctx->hcbd[index].buffer_size = 0; + det_ctx->hcbd[index].buffer_len = 0; + goto end; + } + } + memcpy(det_ctx->hcbd[index].buffer + det_ctx->hcbd[index].buffer_len, cur->data, cur->len); + det_ctx->hcbd[index].buffer_len += cur->len; + + cur = cur->next; + } + + /* update inspected tracker */ + htud->request_body.body_inspected = htud->request_body.last->stream_offset + htud->request_body.last->len; + + buffer = det_ctx->hcbd[index].buffer; + *buffer_len = det_ctx->hcbd[index].buffer_len; + end: + return buffer; +} + /** * \brief Helps buffer request bodies for different transactions and stores them * away in detection code. @@ -215,6 +370,49 @@ end: return; } +int DetectEngineRunHttpClientBodyMpmV2(DetectEngineCtx *de_ctx, + DetectEngineThreadCtx *det_ctx, Flow *f, + HtpState *htp_state, uint8_t flags) +{ + uint32_t cnt = 0; + + if (htp_state == NULL) { + SCLogDebug("no HTTP state"); + goto end; + } + + FLOWLOCK_WRLOCK(f); + + if (htp_state->connp == NULL || htp_state->connp->conn == NULL) { + SCLogDebug("HTP state has no conn(p)"); + goto end; + } + + /* get the transaction id */ + int idx = AppLayerTransactionGetInspectId(f); + /* error! get out of here */ + if (idx == -1) + goto end; + + int size = (int)list_size(htp_state->connp->conn->transactions); + for (; idx < size; idx++) { + uint32_t buffer_len = 0; + uint8_t *buffer = DetectEngineHCBDGetBufferForTX(idx, + de_ctx, det_ctx, + f, htp_state, + flags, + &buffer_len); + if (buffer_len == 0) + continue; + + cnt += HttpClientBodyPatternSearch(det_ctx, buffer, buffer_len, flags); + } + + end: + FLOWLOCK_UNLOCK(f); + return cnt; +} + int DetectEngineRunHttpClientBodyMpm(DetectEngineCtx *de_ctx, DetectEngineThreadCtx *det_ctx, Flow *f, HtpState *htp_state, uint8_t flags) @@ -241,6 +439,62 @@ int DetectEngineRunHttpClientBodyMpm(DetectEngineCtx *de_ctx, return cnt; } +int DetectEngineInspectHttpClientBodyV2(DetectEngineCtx *de_ctx, + DetectEngineThreadCtx *det_ctx, + Signature *s, Flow *f, uint8_t flags, + void *alstate) +{ + int r = 0; + + HtpState *htp_state = (HtpState *)alstate; + + if (htp_state == NULL) { + SCLogDebug("no HTTP state"); + goto end; + } + + FLOWLOCK_WRLOCK(f); + + if (htp_state->connp == NULL || htp_state->connp->conn == NULL) { + SCLogDebug("HTP state has no conn(p)"); + goto end; + } + + /* get the transaction id */ + int idx = AppLayerTransactionGetInspectId(f); + /* error! get out of here */ + if (idx == -1) + goto end; + + int size = (int)list_size(htp_state->connp->conn->transactions); + for (; idx < size; idx++) { + det_ctx->buffer_offset = 0; + det_ctx->discontinue_matching = 0; + det_ctx->inspection_recursion_counter = 0; + + uint32_t buffer_len = 0; + uint8_t *buffer = DetectEngineHCBDGetBufferForTX(idx, + de_ctx, det_ctx, + f, htp_state, + flags, + &buffer_len); + if (buffer_len == 0) + continue; + + r = DetectEngineContentInspection(de_ctx, det_ctx, s, s->sm_lists[DETECT_SM_LIST_HCBDMATCH], + f, + buffer, + buffer_len, + DETECT_ENGINE_CONTENT_INSPECTION_MODE_HCBD, NULL); + if (r == 1) { + break; + } + } + + end: + FLOWLOCK_UNLOCK(f); + return r; +} /** * \brief Do the http_client_body content inspection for a signature. @@ -293,6 +547,20 @@ int DetectEngineInspectHttpClientBody(DetectEngineCtx *de_ctx, SCReturnInt(r); } +void DetectEngineCleanHCBDBuffersV2(DetectEngineThreadCtx *det_ctx) +{ + if (det_ctx->hcbd_buffers_list_len > 0) { + for (int i = 0; i < det_ctx->hcbd_buffers_list_len; i++) { + det_ctx->hcbd[i].buffer_len = 0; + det_ctx->hcbd[i].offset = 0; + } + } + det_ctx->hcbd_buffers_list_len = 0; + det_ctx->hcbd_start_tx_id = 0; + + return; +} + /** * \brief Clean the hcbd buffers. * diff --git a/src/detect-engine-hcbd.h b/src/detect-engine-hcbd.h index 438a69f2a7..e8034feacb 100644 --- a/src/detect-engine-hcbd.h +++ b/src/detect-engine-hcbd.h @@ -36,5 +36,16 @@ int DetectEngineInspectHttpClientBody(DetectEngineCtx *, void DetectEngineCleanHCBDBuffers(DetectEngineThreadCtx *); void DetectEngineHttpClientBodyRegisterTests(void); + +int DetectEngineRunHttpClientBodyMpmV2(DetectEngineCtx *, + DetectEngineThreadCtx *, Flow *f, + HtpState *, uint8_t); +int DetectEngineInspectHttpClientBodyV2(DetectEngineCtx *, + DetectEngineThreadCtx *, + Signature *, Flow *, + uint8_t, void *); +void DetectEngineCleanHCBDBuffersV2(DetectEngineThreadCtx *); + + #endif /* __DETECT_ENGINE_HCBD_H__ */ diff --git a/src/detect-engine-hhd.c b/src/detect-engine-hhd.c index 9a8649d1ac..e04e746a4b 100644 --- a/src/detect-engine-hhd.c +++ b/src/detect-engine-hhd.c @@ -56,6 +56,123 @@ #include "app-layer-htp.h" #include "app-layer-protos.h" +#define BUFFER_STEP 50 + +static uint8_t *DetectEngineHHDGetBufferForTX(int tx_id, + DetectEngineCtx *de_ctx, + DetectEngineThreadCtx *det_ctx, + Flow *f, HtpState *htp_state, + uint8_t flags, + uint32_t *buffer_len) +{ +#define HHDCreateSpace(det_ctx, size) do { \ + if (size > det_ctx->hhd_buffers_size) { \ + det_ctx->hhd_buffers = SCRealloc(det_ctx->hhd_buffers, (det_ctx->hhd_buffers_size + BUFFER_STEP) * sizeof(uint8_t *)); \ + if (det_ctx->hhd_buffers == NULL) { \ + det_ctx->hhd_buffers_size = 0; \ + det_ctx->hhd_buffers_list_len = 0; \ + goto end; \ + } \ + memset(det_ctx->hhd_buffers + det_ctx->hhd_buffers_size, 0, BUFFER_STEP * sizeof(uint8_t *)); \ + det_ctx->hhd_buffers_len = SCRealloc(det_ctx->hhd_buffers_len, (det_ctx->hhd_buffers_size + BUFFER_STEP) * sizeof(uint32_t)); \ + if (det_ctx->hhd_buffers_len == NULL) { \ + det_ctx->hhd_buffers_size = 0; \ + det_ctx->hhd_buffers_list_len = 0; \ + goto end; \ + } \ + memset(det_ctx->hhd_buffers_len + det_ctx->hhd_buffers_size, 0, BUFFER_STEP * sizeof(uint32_t)); \ + det_ctx->hhd_buffers_size += BUFFER_STEP; \ + } \ + memset(det_ctx->hhd_buffers_len + det_ctx->hhd_buffers_list_len, 0, (size - det_ctx->hhd_buffers_list_len) * sizeof(uint32_t)); \ + } while (0) + + int index = 0; + *buffer_len = 0; + + if (det_ctx->hhd_buffers_list_len == 0) { + HHDCreateSpace(det_ctx, 1); + index = 0; + } else { + if ((tx_id - det_ctx->hhd_start_tx_id) < det_ctx->hhd_buffers_list_len) { + if (det_ctx->hhd_buffers_len[(tx_id - det_ctx->hhd_start_tx_id)] != 0) { + *buffer_len = det_ctx->hhd_buffers_len[(tx_id - det_ctx->hhd_start_tx_id)]; + return det_ctx->hhd_buffers[(tx_id - det_ctx->hhd_start_tx_id)]; + } + } else { + HHDCreateSpace(det_ctx, (tx_id - det_ctx->hhd_start_tx_id) + 1); + } + index = (tx_id - det_ctx->hhd_start_tx_id); + } + + if (det_ctx->hhd_buffers_list_len == 0) { + det_ctx->hhd_start_tx_id = tx_id; + } + det_ctx->hhd_buffers_list_len++; + + htp_tx_t *tx = list_get(htp_state->connp->conn->transactions, tx_id); + if (tx == NULL) { + SCLogDebug("no tx"); + goto end; + } + + table_t *headers; + if (flags & STREAM_TOSERVER) { + headers = tx->request_headers; + } else { + headers = tx->response_headers; + } + + htp_header_t *h = NULL; + uint8_t *headers_buffer = det_ctx->hhd_buffers[index]; + size_t headers_buffer_len = 0; + + table_iterator_reset(headers); + while (table_iterator_next(headers, (void **)&h) != NULL) { + size_t size1 = bstr_size(h->name); + size_t size2 = bstr_size(h->value); + + if (flags & STREAM_TOSERVER) { + if (size1 == 6 && + SCMemcmpLowercase("cookie", bstr_ptr(h->name), 6)) { + continue; + } + } else { + if (size1 == 10 && + SCMemcmpLowercase("set-cookie", bstr_ptr(h->name), 10) == 0) { + continue; + } + } + + /* the extra 4 bytes if for ": " and "\r\n" */ + headers_buffer = SCRealloc(headers_buffer, headers_buffer_len + size1 + size2 + 4); + if (headers_buffer == NULL) { + det_ctx->hhd_buffers[index] = NULL; + det_ctx->hhd_buffers_len[index] = 0; + goto end; + } + + memcpy(headers_buffer + headers_buffer_len, bstr_ptr(h->name), size1); + headers_buffer_len += size1; + headers_buffer[headers_buffer_len] = ':'; + headers_buffer[headers_buffer_len + 1] = ' '; + headers_buffer_len += 2; + memcpy(headers_buffer + headers_buffer_len, bstr_ptr(h->value), size2); + headers_buffer_len += size2 + 2; + /* \r */ + headers_buffer[headers_buffer_len - 2] = '\r'; + /* \n */ + headers_buffer[headers_buffer_len - 1] = '\n'; + } + + /* store the buffers. We will need it for further inspection */ + det_ctx->hhd_buffers[index] = headers_buffer; + det_ctx->hhd_buffers_len[index] = headers_buffer_len; + + *buffer_len = headers_buffer_len; + end: + return headers_buffer; +} + /** * \brief Helps buffer http normalized headers from different transactions and * stores them away in detection context. @@ -183,6 +300,48 @@ end: return; } +int DetectEngineRunHttpHeaderMpmV2(DetectEngineThreadCtx *det_ctx, Flow *f, + HtpState *htp_state, uint8_t flags) +{ + uint32_t cnt = 0; + + if (htp_state == NULL) { + SCLogDebug("no HTTP state"); + goto end; + } + + FLOWLOCK_WRLOCK(f); + + if (htp_state->connp == NULL || htp_state->connp->conn == NULL) { + SCLogDebug("HTP state has no conn(p)"); + goto end; + } + + /* get the transaction id */ + int idx = AppLayerTransactionGetInspectId(f); + /* error! get out of here */ + if (idx == -1) + goto end; + + int size = (int)list_size(htp_state->connp->conn->transactions); + for (; idx < size; idx++) { + uint32_t buffer_len = 0; + uint8_t *buffer = DetectEngineHHDGetBufferForTX(idx, + NULL, det_ctx, + f, htp_state, + flags, + &buffer_len); + if (buffer_len == 0) + continue; + + cnt += HttpHeaderPatternSearch(det_ctx, buffer, buffer_len, flags); + } + + end: + FLOWLOCK_UNLOCK(f); + return cnt; +} + /** * \brief run the mpm against the assembled http header buffer(s) * \retval cnt Number of matches reported by the mpm algo. @@ -216,6 +375,63 @@ int DetectEngineRunHttpHeaderMpm(DetectEngineThreadCtx *det_ctx, Flow *f, return cnt; } +int DetectEngineInspectHttpHeaderV2(DetectEngineCtx *de_ctx, + DetectEngineThreadCtx *det_ctx, + Signature *s, Flow *f, uint8_t flags, + void *alstate) +{ + int r = 0; + + HtpState *htp_state = (HtpState *)alstate; + + if (htp_state == NULL) { + SCLogDebug("no HTTP state"); + goto end; + } + + FLOWLOCK_WRLOCK(f); + + if (htp_state->connp == NULL || htp_state->connp->conn == NULL) { + SCLogDebug("HTP state has no conn(p)"); + goto end; + } + + /* get the transaction id */ + int idx = AppLayerTransactionGetInspectId(f); + /* error! get out of here */ + if (idx == -1) + goto end; + + int size = (int)list_size(htp_state->connp->conn->transactions); + for (; idx < size; idx++) { + det_ctx->buffer_offset = 0; + det_ctx->discontinue_matching = 0; + det_ctx->inspection_recursion_counter = 0; + + uint32_t buffer_len = 0; + uint8_t *buffer = DetectEngineHHDGetBufferForTX(idx, + de_ctx, det_ctx, + f, htp_state, + flags, + &buffer_len); + if (buffer_len == 0) + continue; + + r = DetectEngineContentInspection(de_ctx, det_ctx, s, s->sm_lists[DETECT_SM_LIST_HHDMATCH], + f, + buffer, + buffer_len, + DETECT_ENGINE_CONTENT_INSPECTION_MODE_HHD, NULL); + if (r == 1) { + break; + } + } + + end: + FLOWLOCK_UNLOCK(f); + return r; +} + /** * \brief Do the http_header content inspection for a signature. * @@ -270,6 +486,20 @@ int DetectEngineInspectHttpHeader(DetectEngineCtx *de_ctx, SCReturnInt(r); } +void DetectEngineCleanHHDBuffersV2(DetectEngineThreadCtx *det_ctx) +{ + if (det_ctx->hhd_buffers_list_len != 0) { + int i; + for (i = 0; i < det_ctx->hhd_buffers_list_len; i++) { + det_ctx->hhd_buffers_len[i] = 0; + } + det_ctx->hhd_buffers_list_len = 0; + } + det_ctx->hhd_start_tx_id = 0; + + return; +} + /** * \brief Clean the hhd buffers. * diff --git a/src/detect-engine-hhd.h b/src/detect-engine-hhd.h index 0fb0569593..b86c41bdad 100644 --- a/src/detect-engine-hhd.h +++ b/src/detect-engine-hhd.h @@ -32,5 +32,13 @@ int DetectEngineInspectHttpHeader(DetectEngineCtx *, DetectEngineThreadCtx *, void DetectEngineCleanHHDBuffers(DetectEngineThreadCtx *); void DetectEngineHttpHeaderRegisterTests(void); +int DetectEngineInspectHttpHeaderV2(DetectEngineCtx *de_ctx, + DetectEngineThreadCtx *det_ctx, + Signature *s, Flow *f, uint8_t flags, + void *alstate); +int DetectEngineRunHttpHeaderMpmV2(DetectEngineThreadCtx *det_ctx, Flow *f, + HtpState *htp_state, uint8_t flags); +void DetectEngineCleanHHDBuffersV2(DetectEngineThreadCtx *det_ctx); + #endif /* __DETECT_ENGINE_HHD_H__ */ diff --git a/src/detect-engine-hsbd.c b/src/detect-engine-hsbd.c index c6f0585cf4..4c47e7546d 100644 --- a/src/detect-engine-hsbd.c +++ b/src/detect-engine-hsbd.c @@ -59,6 +59,161 @@ #define BODY_SCAN_WINDOW 4096 #define BODY_MINIMAL_SIZE 32768 +#define BUFFER_STEP 50 + +static uint8_t *DetectEngineHSBDGetBufferForTX(int tx_id, + DetectEngineCtx *de_ctx, + DetectEngineThreadCtx *det_ctx, + Flow *f, HtpState *htp_state, + uint8_t flags, + uint32_t *buffer_len) +{ +#define HSBDCreateSpace(det_ctx, size) do { \ + if (size > det_ctx->hsbd_buffers_size) { \ + det_ctx->hsbd = SCRealloc(det_ctx->hsbd, (det_ctx->hsbd_buffers_size + BUFFER_STEP) * sizeof(HttpReassembledBody)); \ + if (det_ctx->hsbd == NULL) { \ + det_ctx->hsbd_buffers_size = 0; \ + det_ctx->hsbd_buffers_list_len = 0; \ + goto end; \ + } \ + memset(det_ctx->hsbd + det_ctx->hsbd_buffers_size, 0, BUFFER_STEP * sizeof(HttpReassembledBody)); \ + det_ctx->hsbd_buffers_size += BUFFER_STEP; \ + } \ + for (int i = det_ctx->hsbd_buffers_list_len; i < (size); i++) { \ + det_ctx->hsbd[i].buffer_len = 0; \ + det_ctx->hsbd[i].offset = 0; \ + } \ + } while (0) + + int index = 0; + uint8_t *buffer = NULL; + *buffer_len = 0; + + if (det_ctx->hsbd_buffers_list_len == 0) { + HSBDCreateSpace(det_ctx, 1); + index = 0; + } else { + if ((tx_id - det_ctx->hsbd_start_tx_id) < det_ctx->hsbd_buffers_list_len) { + if (det_ctx->hsbd[(tx_id - det_ctx->hsbd_start_tx_id)].buffer_len != 0) { + *buffer_len = det_ctx->hsbd[(tx_id - det_ctx->hsbd_start_tx_id)].buffer_len; + return det_ctx->hsbd[(tx_id - det_ctx->hsbd_start_tx_id)].buffer; + } + } else { + HSBDCreateSpace(det_ctx, (tx_id - det_ctx->hsbd_start_tx_id) + 1); + } + index = (tx_id - det_ctx->hsbd_start_tx_id); + } + + if (det_ctx->hsbd_buffers_list_len == 0) { + det_ctx->hsbd_start_tx_id = tx_id; + } + det_ctx->hsbd_buffers_list_len++; + + htp_tx_t *tx = list_get(htp_state->connp->conn->transactions, tx_id); + if (tx == NULL) { + SCLogDebug("no tx"); + goto end; + } + + HtpTxUserData *htud = (HtpTxUserData *)htp_tx_get_user_data(tx); + if (htud == NULL) { + SCLogDebug("no htud"); + goto end; + } + + /* no new data */ + if (htud->response_body.body_inspected == htud->response_body.content_len_so_far) { + SCLogDebug("no new data"); + goto end; + } + + HtpBodyChunk *cur = htud->response_body.first; + if (cur == NULL) { + SCLogDebug("No http chunks to inspect for this transacation"); + goto end; + } + + /* in case of chunked transfer encoding, we don't have the length + * of the response body until we see a chunk with length 0. This + * doesn't let us use the response body callback function to + * figure out the end of response body. Instead we do it here. If + * the length is 0, and we have already seen content, it indicates + * chunked transfer. We also check if the parser has truly seen + * the last chunk by checking the progress state for the + * transaction. If we are done parsing all the chunks, we would + * have it set to something other than TX_PROGRESS_REQ_BODY. + * Either ways we should be moving away from buffering in the end + * and running content validation on this buffer type of architecture + * to a stateful inspection, where we can inspect body chunks as and + * when they come */ + if (htud->response_body.content_len == 0) { + if ((htud->response_body.content_len_so_far > 0) && + tx->progress != TX_PROGRESS_RES_BODY) { + /* final length of the body */ + htud->tcflags |= HTP_RES_BODY_COMPLETE; + } + } + + if (flags & STREAM_EOF) { + htud->tcflags |= HTP_RES_BODY_COMPLETE; + } + + /* inspect the body if the transfer is complete or we have hit + * our body size limit */ + if (htud->response_body.content_len_so_far < BODY_MINIMAL_SIZE && + !(htud->tcflags & HTP_RES_BODY_COMPLETE)) { + SCLogDebug("we still haven't seen the entire response body. " + "Let's defer body inspection till we see the " + "entire body."); + goto end; + } + + int first = 1; + while (cur != NULL) { + /* see if we can filter out chunks */ + if (htud->response_body.body_inspected > 0) { + if (cur->stream_offset < htud->response_body.body_inspected) { + if ((htud->response_body.body_inspected - cur->stream_offset) > BODY_SCAN_WINDOW) { + cur = cur->next; + continue; + } else { + /* include this one */ + } + } else { + /* include this one */ + } + } + + if (first) { + det_ctx->hsbd[index].offset = cur->stream_offset; + first = 0; + } + + /* see if we need to grow the buffer */ + if (det_ctx->hsbd[index].buffer == NULL || (det_ctx->hsbd[index].buffer_len + cur->len) > det_ctx->hsbd[index].buffer_size) { + det_ctx->hsbd[index].buffer_size += cur->len * 2; + + if ((det_ctx->hsbd[index].buffer = SCRealloc(det_ctx->hsbd[index].buffer, det_ctx->hsbd[index].buffer_size)) == NULL) { + det_ctx->hsbd[index].buffer_size = 0; + det_ctx->hsbd[index].buffer_len = 0; + goto end; + } + } + memcpy(det_ctx->hsbd[index].buffer + det_ctx->hsbd[index].buffer_len, cur->data, cur->len); + det_ctx->hsbd[index].buffer_len += cur->len; + + cur = cur->next; + } + + /* update inspected tracker */ + htud->response_body.body_inspected = htud->response_body.last->stream_offset + htud->response_body.last->len; + + buffer = det_ctx->hsbd[index].buffer; + *buffer_len = det_ctx->hsbd[index].buffer_len; + end: + return buffer; +} + /** * \brief Helps buffer response bodies for different transactions and stores them * away in detection code. @@ -211,6 +366,49 @@ end: return; } +int DetectEngineRunHttpServerBodyMpmV2(DetectEngineCtx *de_ctx, + DetectEngineThreadCtx *det_ctx, Flow *f, + HtpState *htp_state, uint8_t flags) +{ + uint32_t cnt = 0; + + if (htp_state == NULL) { + SCLogDebug("no HTTP state"); + goto end; + } + + FLOWLOCK_WRLOCK(f); + + if (htp_state->connp == NULL || htp_state->connp->conn == NULL) { + SCLogDebug("HTP state has no conn(p)"); + goto end; + } + + /* get the transaction id */ + int idx = AppLayerTransactionGetInspectId(f); + /* error! get out of here */ + if (idx == -1) + goto end; + + int size = (int)list_size(htp_state->connp->conn->transactions); + for (; idx < size; idx++) { + uint32_t buffer_len = 0; + uint8_t *buffer = DetectEngineHSBDGetBufferForTX(idx, + de_ctx, det_ctx, + f, htp_state, + flags, + &buffer_len); + if (buffer_len == 0) + continue; + + cnt += HttpServerBodyPatternSearch(det_ctx,buffer, buffer_len, flags); + } + + end: + FLOWLOCK_UNLOCK(f); + return cnt; +} + int DetectEngineRunHttpServerBodyMpm(DetectEngineCtx *de_ctx, DetectEngineThreadCtx *det_ctx, Flow *f, HtpState *htp_state, uint8_t flags) @@ -237,6 +435,63 @@ int DetectEngineRunHttpServerBodyMpm(DetectEngineCtx *de_ctx, return cnt; } +int DetectEngineInspectHttpServerBodyV2(DetectEngineCtx *de_ctx, + DetectEngineThreadCtx *det_ctx, + Signature *s, Flow *f, uint8_t flags, + void *alstate) +{ + int r = 0; + + HtpState *htp_state = (HtpState *)alstate; + + if (htp_state == NULL) { + SCLogDebug("no HTTP state"); + goto end; + } + + FLOWLOCK_WRLOCK(f); + + if (htp_state->connp == NULL || htp_state->connp->conn == NULL) { + SCLogDebug("HTP state has no conn(p)"); + goto end; + } + + /* get the transaction id */ + int idx = AppLayerTransactionGetInspectId(f); + /* error! get out of here */ + if (idx == -1) + goto end; + + int size = (int)list_size(htp_state->connp->conn->transactions); + for (; idx < size; idx++) { + det_ctx->buffer_offset = 0; + det_ctx->discontinue_matching = 0; + det_ctx->inspection_recursion_counter = 0; + + uint32_t buffer_len = 0; + uint8_t *buffer = DetectEngineHSBDGetBufferForTX(idx, + de_ctx, det_ctx, + f, htp_state, + flags, + &buffer_len); + if (buffer_len == 0) + continue; + + r = DetectEngineContentInspection(de_ctx, det_ctx, s, s->sm_lists[DETECT_SM_LIST_HSBDMATCH], + f, + buffer, + buffer_len, + DETECT_ENGINE_CONTENT_INSPECTION_MODE_HSBD, NULL); + if (r == 1) { + break; + } + } + + end: + FLOWLOCK_UNLOCK(f); + return r; +} + /** * \brief Do the http_server_body content inspection for a signature. @@ -289,6 +544,20 @@ int DetectEngineInspectHttpServerBody(DetectEngineCtx *de_ctx, SCReturnInt(r); } +void DetectEngineCleanHSBDBuffersV2(DetectEngineThreadCtx *det_ctx) +{ + if (det_ctx->hsbd_buffers_list_len > 0) { + for (int i = 0; i < det_ctx->hsbd_buffers_list_len; i++) { + det_ctx->hsbd[i].buffer_len = 0; + det_ctx->hsbd[i].offset = 0; + } + } + det_ctx->hsbd_buffers_list_len = 0; + det_ctx->hsbd_start_tx_id = 0; + + return; +} + /** * \brief Clean the hsbd buffers. * diff --git a/src/detect-engine-hsbd.h b/src/detect-engine-hsbd.h index a4f8000871..b9760a6165 100644 --- a/src/detect-engine-hsbd.h +++ b/src/detect-engine-hsbd.h @@ -36,5 +36,14 @@ int DetectEngineInspectHttpServerBody(DetectEngineCtx *, void DetectEngineCleanHSBDBuffers(DetectEngineThreadCtx *); void DetectEngineHttpServerBodyRegisterTests(void); +int DetectEngineRunHttpServerBodyMpmV2(DetectEngineCtx *de_ctx, + DetectEngineThreadCtx *det_ctx, Flow *f, + HtpState *htp_state, uint8_t flags); +int DetectEngineInspectHttpServerBodyV2(DetectEngineCtx *de_ctx, + DetectEngineThreadCtx *det_ctx, + Signature *s, Flow *f, uint8_t flags, + void *alstate); +void DetectEngineCleanHSBDBuffersV2(DetectEngineThreadCtx *det_ctx); + #endif /* __DETECT_ENGINE_HSBD_H__ */ diff --git a/src/detect-engine-state.c b/src/detect-engine-state.c index 77e3bab7a4..1d7134fb9c 100644 --- a/src/detect-engine-state.c +++ b/src/detect-engine-state.c @@ -426,8 +426,10 @@ int DeStateDetectStartDetection(ThreadVars *tv, DetectEngineCtx *de_ctx, } if (s->sm_lists[DETECT_SM_LIST_HCBDMATCH] != NULL) { inspect_flags |= DE_STATE_FLAG_HCBD_INSPECT; - if (DetectEngineInspectHttpClientBody(de_ctx, det_ctx, s, f, + if (DetectEngineInspectHttpClientBodyV2(de_ctx, det_ctx, s, f, flags, alstate) == 1) { + //if (DetectEngineInspectHttpClientBody(de_ctx, det_ctx, s, f, + //flags, alstate) == 1) { match_flags |= DE_STATE_FLAG_HCBD_MATCH; } SCLogDebug("inspecting http client body"); @@ -438,8 +440,10 @@ int DeStateDetectStartDetection(ThreadVars *tv, DetectEngineCtx *de_ctx, } if (s->sm_lists[DETECT_SM_LIST_HHDMATCH] != NULL) { inspect_flags |= DE_STATE_FLAG_HHD_INSPECT; - if (DetectEngineInspectHttpHeader(de_ctx, det_ctx, s, f, + if (DetectEngineInspectHttpHeaderV2(de_ctx, det_ctx, s, f, flags, alstate) == 1) { + //if (DetectEngineInspectHttpHeader(de_ctx, det_ctx, s, f, + //flags, alstate) == 1) { match_flags |= DE_STATE_FLAG_HHD_MATCH; } SCLogDebug("inspecting http header"); @@ -524,7 +528,7 @@ int DeStateDetectStartDetection(ThreadVars *tv, DetectEngineCtx *de_ctx, } if (s->sm_lists[DETECT_SM_LIST_HSBDMATCH] != NULL) { inspect_flags |= DE_STATE_FLAG_HSBD_INSPECT; - if (DetectEngineInspectHttpServerBody(de_ctx, det_ctx, s, f, + if (DetectEngineInspectHttpServerBodyV2(de_ctx, det_ctx, s, f, flags, alstate) == 1) { match_flags |= DE_STATE_FLAG_HSBD_MATCH; } @@ -532,8 +536,10 @@ int DeStateDetectStartDetection(ThreadVars *tv, DetectEngineCtx *de_ctx, } if (s->sm_lists[DETECT_SM_LIST_HHDMATCH] != NULL) { inspect_flags |= DE_STATE_FLAG_HHD_INSPECT; - if (DetectEngineInspectHttpHeader(de_ctx, det_ctx, s, f, + if (DetectEngineInspectHttpHeaderV2(de_ctx, det_ctx, s, f, flags, alstate) == 1) { + //if (DetectEngineInspectHttpHeader(de_ctx, det_ctx, s, f, + //flags, alstate) == 1) { match_flags |= DE_STATE_FLAG_HHD_MATCH; } SCLogDebug("inspecting http header"); @@ -832,8 +838,10 @@ int DeStateDetectContinueDetection(ThreadVars *tv, DetectEngineCtx *de_ctx, Dete SCLogDebug("inspecting http client body data"); inspect_flags |= DE_STATE_FLAG_HCBD_INSPECT; - if (DetectEngineInspectHttpClientBody(de_ctx, det_ctx, s, f, - flags, alstate) == 1) { + if (DetectEngineInspectHttpClientBodyV2(de_ctx, det_ctx, s, f, + flags, alstate) == 1) { + //if (DetectEngineInspectHttpClientBody(de_ctx, det_ctx, s, f, + // flags, alstate) == 1) { SCLogDebug("http client body matched"); match_flags |= DE_STATE_FLAG_HCBD_MATCH; } @@ -850,8 +858,10 @@ int DeStateDetectContinueDetection(ThreadVars *tv, DetectEngineCtx *de_ctx, Dete SCLogDebug("inspecting http header data"); inspect_flags |= DE_STATE_FLAG_HHD_INSPECT; - if (DetectEngineInspectHttpHeader(de_ctx, det_ctx, s, f, + if (DetectEngineInspectHttpHeaderV2(de_ctx, det_ctx, s, f, flags, alstate) == 1) { + //if (DetectEngineInspectHttpHeader(de_ctx, det_ctx, s, f, + // flags, alstate) == 1) { SCLogDebug("http header matched"); match_flags |= DE_STATE_FLAG_HHD_MATCH; } @@ -971,7 +981,7 @@ int DeStateDetectContinueDetection(ThreadVars *tv, DetectEngineCtx *de_ctx, Dete SCLogDebug("inspecting http server body data"); inspect_flags |= DE_STATE_FLAG_HSBD_INSPECT; - if (DetectEngineInspectHttpServerBody(de_ctx, det_ctx, s, f, + if (DetectEngineInspectHttpServerBodyV2(de_ctx, det_ctx, s, f, flags, alstate) == 1) { SCLogDebug("http server body matched"); match_flags |= DE_STATE_FLAG_HSBD_MATCH; @@ -981,8 +991,10 @@ int DeStateDetectContinueDetection(ThreadVars *tv, DetectEngineCtx *de_ctx, Dete if (s->sm_lists[DETECT_SM_LIST_HHDMATCH] != NULL) { if (!(item->flags & DE_STATE_FLAG_HHD_MATCH)) { inspect_flags |= DE_STATE_FLAG_HHD_INSPECT; - if (DetectEngineInspectHttpHeader(de_ctx, det_ctx, s, f, + if (DetectEngineInspectHttpHeaderV2(de_ctx, det_ctx, s, f, flags, alstate) == 1) { + //if (DetectEngineInspectHttpHeader(de_ctx, det_ctx, s, f, + //flags, alstate) == 1) { match_flags |= DE_STATE_FLAG_HHD_MATCH; } } diff --git a/src/detect.c b/src/detect.c index 692f62fa11..9b8728518e 100644 --- a/src/detect.c +++ b/src/detect.c @@ -1261,7 +1261,8 @@ static inline void DetectMpmPrefilter(DetectEngineCtx *de_ctx, } if (det_ctx->sgh->flags & SIG_GROUP_HEAD_MPM_HCBD) { PACKET_PROFILING_DETECT_START(p, PROF_DETECT_MPM_HCBD); - DetectEngineRunHttpClientBodyMpm(de_ctx, det_ctx, p->flow, alstate, flags); + DetectEngineRunHttpClientBodyMpmV2(de_ctx, det_ctx, p->flow, alstate, flags); + //DetectEngineRunHttpClientBodyMpm(de_ctx, det_ctx, p->flow, alstate, flags); PACKET_PROFILING_DETECT_END(p, PROF_DETECT_MPM_HCBD); } if (det_ctx->sgh->flags & SIG_GROUP_HEAD_MPM_HMD) { @@ -1277,7 +1278,7 @@ static inline void DetectMpmPrefilter(DetectEngineCtx *de_ctx, } else { /* implied FLOW_PKT_TOCLIENT */ if (p->flowflags & FLOW_PKT_TOCLIENT && det_ctx->sgh->flags & SIG_GROUP_HEAD_MPM_HSBD) { PACKET_PROFILING_DETECT_START(p, PROF_DETECT_MPM_HSBD); - DetectEngineRunHttpServerBodyMpm(de_ctx, det_ctx, p->flow, alstate, flags); + DetectEngineRunHttpServerBodyMpmV2(de_ctx, det_ctx, p->flow, alstate, flags); PACKET_PROFILING_DETECT_END(p, PROF_DETECT_MPM_HSBD); } if (det_ctx->sgh->flags & SIG_GROUP_HEAD_MPM_HSMD) { @@ -1293,7 +1294,8 @@ static inline void DetectMpmPrefilter(DetectEngineCtx *de_ctx, } if (det_ctx->sgh->flags & SIG_GROUP_HEAD_MPM_HHD) { PACKET_PROFILING_DETECT_START(p, PROF_DETECT_MPM_HHD); - DetectEngineRunHttpHeaderMpm(det_ctx, p->flow, alstate, flags); + //DetectEngineRunHttpHeaderMpm(det_ctx, p->flow, alstate, flags); + DetectEngineRunHttpHeaderMpmV2(det_ctx, p->flow, alstate, flags); PACKET_PROFILING_DETECT_END(p, PROF_DETECT_MPM_HHD); } if (det_ctx->sgh->flags & SIG_GROUP_HEAD_MPM_HRHD) { @@ -1901,9 +1903,12 @@ end: /* cleanup pkt specific part of the patternmatcher */ PacketPatternCleanup(th_v, det_ctx); - DetectEngineCleanHCBDBuffers(det_ctx); - DetectEngineCleanHSBDBuffers(det_ctx); - DetectEngineCleanHHDBuffers(det_ctx); + //DetectEngineCleanHCBDBuffers(det_ctx); + DetectEngineCleanHCBDBuffersV2(det_ctx); + //DetectEngineCleanHSBDBuffers(det_ctx); + DetectEngineCleanHSBDBuffersV2(det_ctx); + //DetectEngineCleanHHDBuffers(det_ctx); + DetectEngineCleanHHDBuffersV2(det_ctx); /* store the found sgh (or NULL) in the flow to save us from looking it * up again for the next packet. Also return any stream chunk we processed diff --git a/src/detect.h b/src/detect.h index 7b0eb7841f..5c9619f323 100644 --- a/src/detect.h +++ b/src/detect.h @@ -740,16 +740,21 @@ typedef struct DetectionEngineThreadCtx_ { /* counter for the filestore array below -- up here for cache reasons. */ uint16_t filestore_cnt; - uint16_t hhd_buffers_list_len; - - uint16_t hcbd_buffers_list_len; + HttpReassembledBody *hsbd; + int hsbd_start_tx_id; + uint16_t hsbd_buffers_size; uint16_t hsbd_buffers_list_len; - HttpReassembledBody *hsbd; HttpReassembledBody *hcbd; + int hcbd_start_tx_id; + uint16_t hcbd_buffers_size; + uint16_t hcbd_buffers_list_len; uint8_t **hhd_buffers; uint32_t *hhd_buffers_len; + uint16_t hhd_buffers_size; + uint16_t hhd_buffers_list_len; + int hhd_start_tx_id; /** id for alert counter */ uint16_t counter_alerts; diff --git a/src/util-error.c b/src/util-error.c index 221c9e0744..bdada841f2 100644 --- a/src/util-error.c +++ b/src/util-error.c @@ -198,6 +198,7 @@ const char * SCErrorToString(SCError err) CASE_CODE (SC_ERR_NAPATECH_NOSUPPORT); CASE_CODE (SC_WARN_COMPATIBILITY); CASE_CODE (SC_ERR_DCERPC); + CASE_CODE (SC_ERR_DETECT_PREPARE); CASE_CODE (SC_ERR_AHO_CORASICK); CASE_CODE (SC_ERR_REFERENCE_CONFIG); CASE_CODE (SC_ERR_DUPLICATE_SIG);