src/detect-file-data.c

   1 /* Copyright (C) 2007-2021 Open Information Security Foundation
   2  *
   3  * You can copy, redistribute or modify this Program under the terms of
   4  * the GNU General Public License version 2 as published by the Free
   5  * Software Foundation.
   6  *
   7  * This program is distributed in the hope that it will be useful,
   8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
   9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  10  * GNU General Public License for more details.
  11  *
  12  * You should have received a copy of the GNU General Public License
  13  * version 2 along with this program; if not, write to the Free Software
  14  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  15  * 02110-1301, USA.
  16  */
  17
  18 /**
  19  * \file
  20  *
  21  * \author Victor Julien <victor@inliniac.net>
  22  *
  23  */
  24
  25 #include "suricata-common.h"
  26 #include "threads.h"
  27 #include "debug.h"
  28 #include "decode.h"
  29
  30 #include "detect.h"
  31 #include "detect-parse.h"
  32
  33 #include "detect-engine.h"
  34 #include "detect-engine-mpm.h"
  35 #include "detect-engine-state.h"
  36 #include "detect-engine-prefilter.h"
  37 #include "detect-engine-content-inspection.h"
  38 #include "detect-file-data.h"
  39
  40 #include "app-layer-parser.h"
  41 #include "app-layer-htp.h"
  42 #include "app-layer-smtp.h"
  43
  44 #include "flow.h"
  45 #include "flow-var.h"
  46 #include "flow-util.h"
  47
  48 #include "util-debug.h"
  49 #include "util-spm-bm.h"
  50 #include "util-unittest.h"
  51 #include "util-unittest-helper.h"
  52 #include "util-file-decompression.h"
  53
  54 static int DetectFiledataSetup (DetectEngineCtx *, Signature *, const char *);
  55 #ifdef UNITTESTS
  56 static void DetectFiledataRegisterTests(void);
  57 #endif
  58 static void DetectFiledataSetupCallback(const DetectEngineCtx *de_ctx,
  59                                         Signature *s);
  60 static int g_file_data_buffer_id = 0;
  61
  62 static inline HtpBody *GetResponseBody(htp_tx_t *tx);
  63
  64 /* HTTP */
  65 static int PrefilterMpmHTTPFiledataRegister(DetectEngineCtx *de_ctx, SigGroupHead *sgh,
  66         MpmCtx *mpm_ctx, const DetectBufferMpmRegistery *mpm_reg, int list_id);
  67
  68 /* file API */
  69 static int DetectEngineInspectFiledata(
  70         DetectEngineCtx *de_ctx, DetectEngineThreadCtx *det_ctx,
  71         const DetectEngineAppInspectionEngine *engine,
  72         const Signature *s,
  73         Flow *f, uint8_t flags, void *alstate, void *txv, uint64_t tx_id);
  74 int PrefilterMpmFiledataRegister(DetectEngineCtx *de_ctx,
  75         SigGroupHead *sgh, MpmCtx *mpm_ctx,
  76         const DetectBufferMpmRegistery *mpm_reg, int list_id);
  77
  78 static int DetectEngineInspectBufferHttpBody(DetectEngineCtx *de_ctx,
  79         DetectEngineThreadCtx *det_ctx, const DetectEngineAppInspectionEngine *engine,
  80         const Signature *s, Flow *f, uint8_t flags, void *alstate, void *txv, uint64_t tx_id);
  81
  82 /**
  83  * \brief Registration function for keyword: file_data
  84  */
  85 void DetectFiledataRegister(void)
  86 {
  87     sigmatch_table[DETECT_FILE_DATA].name = "file.data";
  88     sigmatch_table[DETECT_FILE_DATA].alias = "file_data";
  89     sigmatch_table[DETECT_FILE_DATA].desc = "make content keywords match on file data";
  90     sigmatch_table[DETECT_FILE_DATA].url = "/rules/http-keywords.html#file-data";
  91     sigmatch_table[DETECT_FILE_DATA].Setup = DetectFiledataSetup;
  92 #ifdef UNITTESTS
  93     sigmatch_table[DETECT_FILE_DATA].RegisterTests = DetectFiledataRegisterTests;
  94 #endif
  95     sigmatch_table[DETECT_FILE_DATA].flags = SIGMATCH_NOOPT;
  96
  97     DetectAppLayerMpmRegister2("file_data", SIG_FLAG_TOSERVER, 2,
  98             PrefilterMpmFiledataRegister, NULL,
  99             ALPROTO_SMTP, 0);
 100     DetectAppLayerMpmRegister2("file_data", SIG_FLAG_TOCLIENT, 2, PrefilterMpmHTTPFiledataRegister,
 101             NULL, ALPROTO_HTTP1, HTP_RESPONSE_BODY);
 102     DetectAppLayerMpmRegister2("file_data", SIG_FLAG_TOSERVER, 2,
 103             PrefilterMpmFiledataRegister, NULL,
 104             ALPROTO_SMB, 0);
 105     DetectAppLayerMpmRegister2("file_data", SIG_FLAG_TOCLIENT, 2,
 106             PrefilterMpmFiledataRegister, NULL,
 107             ALPROTO_SMB, 0);
 108     DetectAppLayerMpmRegister2("file_data", SIG_FLAG_TOSERVER, 2,
 109             PrefilterMpmFiledataRegister, NULL,
 110             ALPROTO_HTTP2, HTTP2StateDataClient);
 111     DetectAppLayerMpmRegister2("file_data", SIG_FLAG_TOCLIENT, 2,
 112             PrefilterMpmFiledataRegister, NULL,
 113             ALPROTO_HTTP2, HTTP2StateDataServer);
 114     DetectAppLayerMpmRegister2("file_data", SIG_FLAG_TOSERVER, 2, PrefilterMpmFiledataRegister,
 115             NULL, ALPROTO_FTPDATA, 0);
 116     DetectAppLayerMpmRegister2("file_data", SIG_FLAG_TOCLIENT, 2, PrefilterMpmFiledataRegister,
 117             NULL, ALPROTO_FTPDATA, 0);
 118     DetectAppLayerMpmRegister2(
 119             "file_data", SIG_FLAG_TOSERVER, 2, PrefilterMpmFiledataRegister, NULL, ALPROTO_FTP, 0);
 120     DetectAppLayerMpmRegister2(
 121             "file_data", SIG_FLAG_TOCLIENT, 2, PrefilterMpmFiledataRegister, NULL, ALPROTO_FTP, 0);
 122
 123     DetectAppLayerInspectEngineRegister2("file_data", ALPROTO_HTTP1, SIG_FLAG_TOCLIENT,
 124             HTP_RESPONSE_BODY, DetectEngineInspectBufferHttpBody, NULL);
 125     DetectAppLayerInspectEngineRegister2("file_data",
 126             ALPROTO_SMTP, SIG_FLAG_TOSERVER, 0,
 127             DetectEngineInspectFiledata, NULL);
 128     DetectBufferTypeRegisterSetupCallback("file_data",
 129             DetectFiledataSetupCallback);
 130     DetectAppLayerInspectEngineRegister2("file_data",
 131             ALPROTO_SMB, SIG_FLAG_TOSERVER, 0,
 132             DetectEngineInspectFiledata, NULL);
 133     DetectAppLayerInspectEngineRegister2("file_data",
 134             ALPROTO_SMB, SIG_FLAG_TOCLIENT, 0,
 135             DetectEngineInspectFiledata, NULL);
 136     DetectAppLayerInspectEngineRegister2("file_data",
 137             ALPROTO_HTTP2, SIG_FLAG_TOSERVER, HTTP2StateDataClient,
 138             DetectEngineInspectFiledata, NULL);
 139     DetectAppLayerInspectEngineRegister2("file_data",
 140             ALPROTO_HTTP2, SIG_FLAG_TOCLIENT, HTTP2StateDataServer,
 141             DetectEngineInspectFiledata, NULL);
 142     DetectAppLayerInspectEngineRegister2(
 143             "file_data", ALPROTO_FTPDATA, SIG_FLAG_TOSERVER, 0, DetectEngineInspectFiledata, NULL);
 144     DetectAppLayerInspectEngineRegister2(
 145             "file_data", ALPROTO_FTPDATA, SIG_FLAG_TOCLIENT, 0, DetectEngineInspectFiledata, NULL);
 146     DetectAppLayerInspectEngineRegister2(
 147             "file_data", ALPROTO_FTP, SIG_FLAG_TOSERVER, 0, DetectEngineInspectFiledata, NULL);
 148     DetectAppLayerInspectEngineRegister2(
 149             "file_data", ALPROTO_FTP, SIG_FLAG_TOCLIENT, 0, DetectEngineInspectFiledata, NULL);
 150
 151     DetectBufferTypeSetDescriptionByName("file_data",
 152             "http response body, smb files or smtp attachments data");
 153
 154     g_file_data_buffer_id = DetectBufferTypeGetByName("file_data");
 155 }
 156
 157 #define FILEDATA_CONTENT_LIMIT 100000
 158 #define FILEDATA_CONTENT_INSPECT_MIN_SIZE 32768
 159 #define FILEDATA_CONTENT_INSPECT_WINDOW 4096
 160
 161 static void SetupDetectEngineConfig(DetectEngineCtx *de_ctx) {
 162     if (de_ctx->filedata_config_initialized)
 163         return;
 164
 165     /* initialize default */
 166     for (int i = 0; i < (int)ALPROTO_MAX; i++) {
 167         de_ctx->filedata_config[i].content_limit = FILEDATA_CONTENT_LIMIT;
 168         de_ctx->filedata_config[i].content_inspect_min_size = FILEDATA_CONTENT_INSPECT_MIN_SIZE;
 169         de_ctx->filedata_config[i].content_inspect_window = FILEDATA_CONTENT_INSPECT_WINDOW;
 170     }
 171
 172     /* add protocol specific settings here */
 173
 174     /* SMTP */
 175     de_ctx->filedata_config[ALPROTO_SMTP].content_limit = smtp_config.content_limit;
 176     de_ctx->filedata_config[ALPROTO_SMTP].content_inspect_min_size = smtp_config.content_inspect_min_size;
 177     de_ctx->filedata_config[ALPROTO_SMTP].content_inspect_window = smtp_config.content_inspect_window;
 178
 179     de_ctx->filedata_config_initialized = true;
 180 }
 181
 182 /**
 183  * \brief this function is used to parse filedata options
 184  * \brief into the current signature
 185  *
 186  * \param de_ctx pointer to the Detection Engine Context
 187  * \param s pointer to the Current Signature
 188  * \param str pointer to the user provided "filestore" option
 189  *
 190  * \retval 0 on Success
 191  * \retval -1 on Failure
 192  */
 193 static int DetectFiledataSetup (DetectEngineCtx *de_ctx, Signature *s, const char *str)
 194 {
 195     SCEnter();
 196
 197     if (!DetectProtoContainsProto(&s->proto, IPPROTO_TCP) ||
 198             (s->alproto != ALPROTO_UNKNOWN && s->alproto != ALPROTO_HTTP1 &&
 199                     s->alproto != ALPROTO_SMTP && s->alproto != ALPROTO_SMB &&
 200                     s->alproto != ALPROTO_HTTP2 && s->alproto != ALPROTO_FTP &&
 201                     s->alproto != ALPROTO_FTPDATA && s->alproto != ALPROTO_HTTP)) {
 202         SCLogError(SC_ERR_CONFLICTING_RULE_KEYWORDS, "rule contains conflicting keywords.");
 203         return -1;
 204     }
 205
 206     if ((s->alproto == ALPROTO_HTTP1 || s->alproto == ALPROTO_HTTP) &&
 207             (s->init_data->init_flags & SIG_FLAG_INIT_FLOW) && (s->flags & SIG_FLAG_TOSERVER) &&
 208             !(s->flags & SIG_FLAG_TOCLIENT)) {
 209         SCLogError(SC_ERR_INVALID_SIGNATURE, "Can't use file_data with "
 210                 "flow:to_server or flow:from_client with http.");
 211         return -1;
 212     }
 213
 214     if (s->alproto == ALPROTO_SMTP && (s->init_data->init_flags & SIG_FLAG_INIT_FLOW) &&
 215         !(s->flags & SIG_FLAG_TOSERVER) && (s->flags & SIG_FLAG_TOCLIENT)) {
 216         SCLogError(SC_ERR_INVALID_SIGNATURE, "Can't use file_data with "
 217                 "flow:to_client or flow:from_server with smtp.");
 218         return -1;
 219     }
 220
 221     if (DetectBufferSetActiveList(s, DetectBufferTypeGetByName("file_data")) < 0)
 222         return -1;
 223
 224     s->init_data->init_flags |= SIG_FLAG_INIT_FILEDATA;
 225     SetupDetectEngineConfig(de_ctx);
 226     return 0;
 227 }
 228
 229 static void DetectFiledataSetupCallback(const DetectEngineCtx *de_ctx,
 230                                         Signature *s)
 231 {
 232     if (s->alproto == ALPROTO_HTTP1 || s->alproto == ALPROTO_UNKNOWN ||
 233             s->alproto == ALPROTO_HTTP) {
 234         AppLayerHtpEnableResponseBodyCallback();
 235     }
 236
 237     /* server body needs to be inspected in sync with stream if possible */
 238     s->init_data->init_flags |= SIG_FLAG_INIT_NEED_FLUSH;
 239
 240     SCLogDebug("callback invoked by %u", s->id);
 241 }
 242
 243 /* common */
 244
 245 typedef struct PrefilterMpmFiledata {
 246     int list_id;
 247     int base_list_id;
 248     const MpmCtx *mpm_ctx;
 249     const DetectEngineTransforms *transforms;
 250 } PrefilterMpmFiledata;
 251
 252 static void PrefilterMpmFiledataFree(void *ptr)
 253 {
 254     SCFree(ptr);
 255 }
 256
 257 /* HTTP based detection */
 258
 259 static inline HtpBody *GetResponseBody(htp_tx_t *tx)
 260 {
 261     HtpTxUserData *htud = (HtpTxUserData *)htp_tx_get_user_data(tx);
 262     if (htud == NULL) {
 263         SCLogDebug("no htud");
 264         return NULL;
 265     }
 266
 267     return &htud->response_body;
 268 }
 269
 270 static inline InspectionBuffer *HttpServerBodyXformsGetDataCallback(DetectEngineThreadCtx *det_ctx,
 271         const DetectEngineTransforms *transforms, const int list_id, InspectionBuffer *base_buffer)
 272 {
 273     InspectionBuffer *buffer = InspectionBufferGet(det_ctx, list_id);
 274     if (buffer->inspect != NULL)
 275         return buffer;
 276
 277     InspectionBufferSetup(det_ctx, list_id, buffer, base_buffer->inspect, base_buffer->inspect_len);
 278     buffer->inspect_offset = base_buffer->inspect_offset;
 279     InspectionBufferApplyTransforms(buffer, transforms);
 280     SCLogDebug("xformed buffer %p size %u", buffer, buffer->inspect_len);
 281     SCReturnPtr(buffer, "InspectionBuffer");
 282 }
 283
 284 static InspectionBuffer *HttpServerBodyGetDataCallback(DetectEngineThreadCtx *det_ctx,
 285         const DetectEngineTransforms *transforms, Flow *f, const uint8_t flow_flags, void *txv,
 286         const int list_id, const int base_id)
 287 {
 288     SCEnter();
 289
 290     InspectionBuffer *buffer = InspectionBufferGet(det_ctx, base_id);
 291     if (base_id != list_id && buffer->inspect != NULL)
 292         return HttpServerBodyXformsGetDataCallback(det_ctx, transforms, list_id, buffer);
 293     else if (buffer->inspect != NULL)
 294         return buffer;
 295
 296     htp_tx_t *tx = txv;
 297     HtpState *htp_state = f->alstate;
 298     const uint8_t flags = flow_flags;
 299
 300     HtpBody *body = GetResponseBody(tx);
 301     if (body == NULL) {
 302         return NULL;
 303     }
 304
 305     /* no new data */
 306     if (body->body_inspected == body->content_len_so_far) {
 307         SCLogDebug("no new data");
 308         return NULL;
 309     }
 310
 311     HtpBodyChunk *cur = body->first;
 312     if (cur == NULL) {
 313         SCLogDebug("No http chunks to inspect for this transaction");
 314         return NULL;
 315     }
 316
 317     SCLogDebug("response.body_limit %u response_body.content_len_so_far %" PRIu64
 318                ", response.inspect_min_size %" PRIu32 ", EOF %s, progress > body? %s",
 319             htp_state->cfg->response.body_limit, body->content_len_so_far,
 320             htp_state->cfg->response.inspect_min_size, flags & STREAM_EOF ? "true" : "false",
 321             (AppLayerParserGetStateProgress(IPPROTO_TCP, ALPROTO_HTTP1, tx, flags) >
 322                     HTP_RESPONSE_BODY)
 323                     ? "true"
 324                     : "false");
 325
 326     if (!htp_state->cfg->http_body_inline) {
 327         /* inspect the body if the transfer is complete or we have hit
 328         * our body size limit */
 329         if ((htp_state->cfg->response.body_limit == 0 ||
 330                     body->content_len_so_far < htp_state->cfg->response.body_limit) &&
 331                 body->content_len_so_far < htp_state->cfg->response.inspect_min_size &&
 332                 !(AppLayerParserGetStateProgress(IPPROTO_TCP, ALPROTO_HTTP1, tx, flags) >
 333                         HTP_RESPONSE_BODY) &&
 334                 !(flags & STREAM_EOF)) {
 335             SCLogDebug("we still haven't seen the entire response body.  "
 336                        "Let's defer body inspection till we see the "
 337                        "entire body.");
 338             return NULL;
 339         }
 340     }
 341
 342     /* get the inspect buffer
 343      *
 344      * make sure that we have at least the configured inspect_win size.
 345      * If we have more, take at least 1/4 of the inspect win size before
 346      * the new data.
 347      */
 348     uint64_t offset = 0;
 349     if (body->body_inspected > htp_state->cfg->response.inspect_min_size) {
 350         BUG_ON(body->content_len_so_far < body->body_inspected);
 351         uint64_t inspect_win = body->content_len_so_far - body->body_inspected;
 352         SCLogDebug("inspect_win %"PRIu64, inspect_win);
 353         if (inspect_win < htp_state->cfg->response.inspect_window) {
 354             uint64_t inspect_short = htp_state->cfg->response.inspect_window - inspect_win;
 355             if (body->body_inspected < inspect_short)
 356                 offset = 0;
 357             else
 358                 offset = body->body_inspected - inspect_short;
 359         } else {
 360             offset = body->body_inspected - (htp_state->cfg->response.inspect_window / 4);
 361         }
 362     }
 363
 364     const uint8_t *data;
 365     uint32_t data_len;
 366
 367     StreamingBufferGetDataAtOffset(body->sb,
 368             &data, &data_len, offset);
 369     InspectionBufferSetup(det_ctx, base_id, buffer, data, data_len);
 370     buffer->inspect_offset = offset;
 371     body->body_inspected = body->content_len_so_far;
 372     SCLogDebug("body->body_inspected now: %" PRIu64, body->body_inspected);
 373
 374     /* built-in 'transformation' */
 375     if (htp_state->cfg->swf_decompression_enabled) {
 376         int swf_file_type = FileIsSwfFile(data, data_len);
 377         if (swf_file_type == FILE_SWF_ZLIB_COMPRESSION ||
 378             swf_file_type == FILE_SWF_LZMA_COMPRESSION)
 379         {
 380             (void)FileSwfDecompression(data, data_len,
 381                                        det_ctx,
 382                                        buffer,
 383                                        htp_state->cfg->swf_compression_type,
 384                                        htp_state->cfg->swf_decompress_depth,
 385                                        htp_state->cfg->swf_compress_depth);
 386         }
 387     }
 388
 389     if (base_id != list_id) {
 390         buffer = HttpServerBodyXformsGetDataCallback(det_ctx, transforms, list_id, buffer);
 391     }
 392     SCReturnPtr(buffer, "InspectionBuffer");
 393 }
 394
 395 static int DetectEngineInspectBufferHttpBody(DetectEngineCtx *de_ctx,
 396         DetectEngineThreadCtx *det_ctx, const DetectEngineAppInspectionEngine *engine,
 397         const Signature *s, Flow *f, uint8_t flags, void *alstate, void *txv, uint64_t tx_id)
 398 {
 399     bool eof =
 400             (AppLayerParserGetStateProgress(f->proto, f->alproto, txv, flags) > engine->progress);
 401     const InspectionBuffer *buffer = HttpServerBodyGetDataCallback(
 402             det_ctx, engine->v2.transforms, f, flags, txv, engine->sm_list, engine->sm_list_base);
 403     if (buffer == NULL || buffer->inspect == NULL) {
 404         return eof ? DETECT_ENGINE_INSPECT_SIG_CANT_MATCH : DETECT_ENGINE_INSPECT_SIG_NO_MATCH;
 405     }
 406
 407     const uint32_t data_len = buffer->inspect_len;
 408     const uint8_t *data = buffer->inspect;
 409     const uint64_t offset = buffer->inspect_offset;
 410
 411     uint8_t ci_flags = eof ? DETECT_CI_FLAGS_END : 0;
 412     ci_flags |= (offset == 0 ? DETECT_CI_FLAGS_START : 0);
 413     ci_flags |= buffer->flags;
 414
 415     det_ctx->discontinue_matching = 0;
 416     det_ctx->buffer_offset = 0;
 417     det_ctx->inspection_recursion_counter = 0;
 418
 419     /* Inspect all the uricontents fetched on each
 420      * transaction at the app layer */
 421     int r = DetectEngineContentInspection(de_ctx, det_ctx, s, engine->smd, NULL, f, (uint8_t *)data,
 422             data_len, offset, ci_flags, DETECT_ENGINE_CONTENT_INSPECTION_MODE_STATE);
 423     if (r == 1) {
 424         return DETECT_ENGINE_INSPECT_SIG_MATCH;
 425     }
 426
 427     if (flags & STREAM_TOSERVER) {
 428         if (AppLayerParserGetStateProgress(IPPROTO_TCP, ALPROTO_HTTP1, txv, flags) >
 429                 HTP_REQUEST_BODY)
 430             return DETECT_ENGINE_INSPECT_SIG_CANT_MATCH;
 431     } else {
 432         if (AppLayerParserGetStateProgress(IPPROTO_TCP, ALPROTO_HTTP1, txv, flags) >
 433                 HTP_RESPONSE_BODY)
 434             return DETECT_ENGINE_INSPECT_SIG_CANT_MATCH;
 435     }
 436     return DETECT_ENGINE_INSPECT_SIG_NO_MATCH;
 437 }
 438
 439 /** \brief Filedata Filedata Mpm prefilter callback
 440  *
 441  *  \param det_ctx detection engine thread ctx
 442  *  \param pectx inspection context
 443  *  \param p packet to inspect
 444  *  \param f flow to inspect
 445  *  \param txv tx to inspect
 446  *  \param idx transaction id
 447  *  \param flags STREAM_* flags including direction
 448  */
 449 static void PrefilterTxHTTPFiledata(DetectEngineThreadCtx *det_ctx, const void *pectx, Packet *p,
 450         Flow *f, void *txv, const uint64_t idx, const uint8_t flags)
 451 {
 452     SCEnter();
 453
 454     const PrefilterMpmFiledata *ctx = (const PrefilterMpmFiledata *)pectx;
 455     const MpmCtx *mpm_ctx = ctx->mpm_ctx;
 456     const int list_id = ctx->list_id;
 457
 458     InspectionBuffer *buffer = HttpServerBodyGetDataCallback(
 459             det_ctx, ctx->transforms, f, flags, txv, list_id, ctx->base_list_id);
 460     if (buffer == NULL)
 461         return;
 462
 463     if (buffer->inspect_len >= mpm_ctx->minlen) {
 464         (void)mpm_table[mpm_ctx->mpm_type].Search(
 465                 mpm_ctx, &det_ctx->mtcu, &det_ctx->pmq, buffer->inspect, buffer->inspect_len);
 466     }
 467 }
 468
 469 static int PrefilterMpmHTTPFiledataRegister(DetectEngineCtx *de_ctx, SigGroupHead *sgh,
 470         MpmCtx *mpm_ctx, const DetectBufferMpmRegistery *mpm_reg, int list_id)
 471 {
 472     PrefilterMpmFiledata *pectx = SCCalloc(1, sizeof(*pectx));
 473     if (pectx == NULL)
 474         return -1;
 475     pectx->list_id = list_id;
 476     pectx->base_list_id = mpm_reg->sm_list_base;
 477     SCLogDebug("list_id %d base_list_id %d", list_id, pectx->base_list_id);
 478     pectx->mpm_ctx = mpm_ctx;
 479     pectx->transforms = &mpm_reg->transforms;
 480
 481     return PrefilterAppendTxEngine(de_ctx, sgh, PrefilterTxHTTPFiledata, mpm_reg->app_v2.alproto,
 482             mpm_reg->app_v2.tx_min_progress, pectx, PrefilterMpmFiledataFree, mpm_reg->pname);
 483 }
 484
 485 /* file API based inspection */
 486
 487 static inline InspectionBuffer *FiledataWithXformsGetDataCallback(DetectEngineThreadCtx *det_ctx,
 488         const DetectEngineTransforms *transforms, const int list_id, int local_file_id,
 489         InspectionBuffer *base_buffer, const bool first)
 490 {
 491     InspectionBuffer *buffer = InspectionBufferMultipleForListGet(det_ctx, list_id, local_file_id);
 492     if (buffer == NULL) {
 493         SCLogDebug("list_id: %d: no buffer", list_id);
 494         return NULL;
 495     }
 496     if (!first && buffer->inspect != NULL) {
 497         SCLogDebug("list_id: %d: returning %p", list_id, buffer);
 498         return buffer;
 499     }
 500
 501     InspectionBufferSetupMulti(buffer, transforms, base_buffer->inspect, base_buffer->inspect_len);
 502     buffer->inspect_offset = base_buffer->inspect_offset;
 503     SCLogDebug("xformed buffer %p size %u", buffer, buffer->inspect_len);
 504     SCReturnPtr(buffer, "InspectionBuffer");
 505 }
 506
 507 static InspectionBuffer *FiledataGetDataCallback(DetectEngineThreadCtx *det_ctx,
 508         const DetectEngineTransforms *transforms, Flow *f, uint8_t flow_flags, File *cur_file,
 509         const int list_id, const int base_id, int local_file_id, bool first)
 510 {
 511     SCEnter();
 512     SCLogDebug(
 513             "starting: list_id %d base_id %d first %s", list_id, base_id, first ? "true" : "false");
 514
 515     InspectionBuffer *buffer = InspectionBufferMultipleForListGet(det_ctx, base_id, local_file_id);
 516     SCLogDebug("base: buffer %p", buffer);
 517     if (buffer == NULL)
 518         return NULL;
 519     if (base_id != list_id && buffer->inspect != NULL) {
 520         SCLogDebug("handle xform %s", (list_id != base_id) ? "true" : "false");
 521         return FiledataWithXformsGetDataCallback(
 522                 det_ctx, transforms, list_id, local_file_id, buffer, first);
 523     }
 524     if (!first && buffer->inspect != NULL) {
 525         SCLogDebug("base_id: %d, not first: use %p", base_id, buffer);
 526         return buffer;
 527     }
 528
 529     const uint64_t file_size = FileDataSize(cur_file);
 530     const DetectEngineCtx *de_ctx = det_ctx->de_ctx;
 531     const uint32_t content_limit = de_ctx->filedata_config[f->alproto].content_limit;
 532     const uint32_t content_inspect_min_size = de_ctx->filedata_config[f->alproto].content_inspect_min_size;
 533     // TODO this is unused, is that right?
 534     //const uint32_t content_inspect_window = de_ctx->filedata_config[f->alproto].content_inspect_window;
 535
 536     SCLogDebug("[list %d] first: %d, content_limit %u, content_inspect_min_size %u", list_id,
 537             first ? 1 : 0, content_limit, content_inspect_min_size);
 538
 539     SCLogDebug("[list %d] file %p size %" PRIu64 ", state %d", list_id, cur_file, file_size,
 540             cur_file->state);
 541
 542     /* no new data */
 543     if (cur_file->content_inspected == file_size) {
 544         SCLogDebug("no new data");
 545         return NULL;
 546     }
 547
 548     if (file_size == 0) {
 549         SCLogDebug("no data to inspect for this transaction");
 550         return NULL;
 551     }
 552
 553     if ((content_limit == 0 || file_size < content_limit) &&
 554         file_size < content_inspect_min_size &&
 555         !(flow_flags & STREAM_EOF) && !(cur_file->state > FILE_STATE_OPENED)) {
 556         SCLogDebug("we still haven't seen the entire content. "
 557                    "Let's defer content inspection till we see the "
 558                    "entire content.");
 559         return NULL;
 560     }
 561
 562     const uint8_t *data;
 563     uint32_t data_len;
 564
 565     StreamingBufferGetDataAtOffset(cur_file->sb,
 566             &data, &data_len,
 567             cur_file->content_inspected);
 568     InspectionBufferSetupMulti(buffer, NULL, data, data_len);
 569     SCLogDebug("[list %d] [before] buffer offset %" PRIu64 "; buffer len %" PRIu32
 570                "; data_len %" PRIu32 "; file_size %" PRIu64,
 571             list_id, buffer->inspect_offset, buffer->inspect_len, data_len, file_size);
 572     buffer->inspect_offset = cur_file->content_inspected;
 573
 574     /* update inspected tracker */
 575     cur_file->content_inspected = buffer->inspect_len + buffer->inspect_offset;
 576     SCLogDebug("content inspected: %" PRIu64, cur_file->content_inspected);
 577
 578     /* get buffer for the list id if it is different from the base id */
 579     if (list_id != base_id) {
 580         SCLogDebug("regular %d has been set up: now handle xforms id %d", base_id, list_id);
 581         InspectionBuffer *tbuffer = FiledataWithXformsGetDataCallback(
 582                 det_ctx, transforms, list_id, local_file_id, buffer, first);
 583         SCReturnPtr(tbuffer, "InspectionBuffer");
 584     } else {
 585         SCLogDebug("regular buffer %p size %u", buffer, buffer->inspect_len);
 586         SCReturnPtr(buffer, "InspectionBuffer");
 587     }
 588 }
 589
 590 static int DetectEngineInspectFiledata(
 591         DetectEngineCtx *de_ctx, DetectEngineThreadCtx *det_ctx,
 592         const DetectEngineAppInspectionEngine *engine,
 593         const Signature *s,
 594         Flow *f, uint8_t flags, void *alstate, void *txv, uint64_t tx_id)
 595 {
 596     int r = 0;
 597     int match = 0;
 598
 599     const DetectEngineTransforms *transforms = NULL;
 600     if (!engine->mpm) {
 601         transforms = engine->v2.transforms;
 602     }
 603
 604     FileContainer *ffc = AppLayerParserGetFiles(f, flags);
 605     if (ffc == NULL) {
 606         return DETECT_ENGINE_INSPECT_SIG_NO_MATCH;
 607     }
 608
 609     int local_file_id = 0;
 610     File *file = ffc->head;
 611     for (; file != NULL; file = file->next) {
 612         if (file->txid != tx_id)
 613             continue;
 614
 615         InspectionBuffer *buffer = FiledataGetDataCallback(det_ctx, transforms, f, flags, file,
 616                 engine->sm_list, engine->sm_list_base, local_file_id, false);
 617         if (buffer == NULL)
 618             continue;
 619
 620         bool eof = (file->state == FILE_STATE_CLOSED);
 621         uint8_t ciflags = eof ? DETECT_CI_FLAGS_END : 0;
 622         if (buffer->inspect_offset == 0)
 623             ciflags |= DETECT_CI_FLAGS_START;
 624
 625         det_ctx->buffer_offset = 0;
 626         det_ctx->discontinue_matching = 0;
 627         det_ctx->inspection_recursion_counter = 0;
 628         match = DetectEngineContentInspection(de_ctx, det_ctx, s, engine->smd,
 629                                               NULL, f,
 630                                               (uint8_t *)buffer->inspect,
 631                                               buffer->inspect_len,
 632                                               buffer->inspect_offset, ciflags,
 633                                               DETECT_ENGINE_CONTENT_INSPECTION_MODE_STATE);
 634         if (match == 1) {
 635             r = 1;
 636             break;
 637         }
 638         local_file_id++;
 639     }
 640
 641     if (r == 1)
 642         return DETECT_ENGINE_INSPECT_SIG_MATCH;
 643     else
 644         return DETECT_ENGINE_INSPECT_SIG_NO_MATCH;
 645 }
 646
 647 /** \brief Filedata Filedata Mpm prefilter callback
 648  *
 649  *  \param det_ctx detection engine thread ctx
 650  *  \param pectx inspection context
 651  *  \param p packet to inspect
 652  *  \param f flow to inspect
 653  *  \param txv tx to inspect
 654  *  \param idx transaction id
 655  *  \param flags STREAM_* flags including direction
 656  */
 657 static void PrefilterTxFiledata(DetectEngineThreadCtx *det_ctx,
 658         const void *pectx,
 659         Packet *p, Flow *f, void *txv,
 660         const uint64_t idx, const uint8_t flags)
 661 {
 662     SCEnter();
 663
 664     const PrefilterMpmFiledata *ctx = (const PrefilterMpmFiledata *)pectx;
 665     const MpmCtx *mpm_ctx = ctx->mpm_ctx;
 666     const int list_id = ctx->list_id;
 667
 668     FileContainer *ffc = AppLayerParserGetFiles(f, flags);
 669     int local_file_id = 0;
 670     if (ffc != NULL) {
 671         File *file = ffc->head;
 672         for (; file != NULL; file = file->next) {
 673             if (file->txid != idx)
 674                 continue;
 675
 676             InspectionBuffer *buffer = FiledataGetDataCallback(det_ctx, ctx->transforms, f, flags,
 677                     file, list_id, ctx->base_list_id, local_file_id, true);
 678             if (buffer == NULL)
 679                 continue;
 680
 681             if (buffer->inspect_len >= mpm_ctx->minlen) {
 682                 (void)mpm_table[mpm_ctx->mpm_type].Search(mpm_ctx,
 683                         &det_ctx->mtcu, &det_ctx->pmq,
 684                         buffer->inspect, buffer->inspect_len);
 685             }
 686             local_file_id++;
 687         }
 688     }
 689 }
 690
 691 int PrefilterMpmFiledataRegister(DetectEngineCtx *de_ctx,
 692         SigGroupHead *sgh, MpmCtx *mpm_ctx,
 693         const DetectBufferMpmRegistery *mpm_reg, int list_id)
 694 {
 695     PrefilterMpmFiledata *pectx = SCCalloc(1, sizeof(*pectx));
 696     if (pectx == NULL)
 697         return -1;
 698     pectx->list_id = list_id;
 699     pectx->base_list_id = mpm_reg->sm_list_base;
 700     pectx->mpm_ctx = mpm_ctx;
 701     pectx->transforms = &mpm_reg->transforms;
 702
 703     return PrefilterAppendTxEngine(de_ctx, sgh, PrefilterTxFiledata,
 704             mpm_reg->app_v2.alproto, mpm_reg->app_v2.tx_min_progress,
 705             pectx, PrefilterMpmFiledataFree, mpm_reg->pname);
 706 }
 707
 708 #ifdef UNITTESTS
 709 #include "tests/detect-file-data.c"
 710 #endif