From: Victor Julien <victor@inliniac.net>
Date: Mon, 30 Nov 2015 20:21:50 +0000 (+0100)
Subject: file: switch to streaming buffer API
X-Git-Tag: suricata-3.1RC1~115
X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=e43ce0a9ecc32fa1e574fc3c9e1bfc246a45bc01;p=thirdparty%2Fsuricata.git

file: switch to streaming buffer API

Make the file storage use the streaming buffer API.

As the individual file chunks were not needed by themselves, this
approach uses a chunkless implementation.
---

diff --git a/src/app-layer-htp-file.c b/src/app-layer-htp-file.c
index 33b747f13e..4dcddf6865 100644
--- a/src/app-layer-htp-file.c
+++ b/src/app-layer-htp-file.c
@@ -83,6 +83,7 @@ int HTPFileOpen(HtpState *s, const uint8_t *filename, uint16_t filename_len,
     uint8_t flags = 0;
     FileContainer *files = NULL;
     FileContainer *files_opposite = NULL;
+    const StreamingBufferConfig *sbcfg = NULL;
 
     SCLogDebug("data %p data_len %"PRIu32, data, data_len);
 
@@ -120,6 +121,9 @@ int HTPFileOpen(HtpState *s, const uint8_t *filename, uint16_t filename_len,
         if (!(flags & FILE_STORE) && (s->f->flags & FLOW_FILE_NO_STORE_TC)) {
             flags |= FILE_NOSTORE;
         }
+
+        sbcfg = &s->cfg->response.sbcfg;
+
     } else {
         if (s->files_ts == NULL) {
             s->files_ts = FileContainerAlloc();
@@ -149,6 +153,8 @@ int HTPFileOpen(HtpState *s, const uint8_t *filename, uint16_t filename_len,
         if (!(flags & FILE_STORE) && (s->f->flags & FLOW_FILE_NO_STORE_TS)) {
             flags |= FILE_NOSTORE;
         }
+
+        sbcfg = &s->cfg->request.sbcfg;
     }
 
     /* if the previous file is in the same txid, we reset the file part of the
@@ -176,7 +182,7 @@ int HTPFileOpen(HtpState *s, const uint8_t *filename, uint16_t filename_len,
         }
     }
 
-    if (FileOpenFile(files, filename, filename_len,
+    if (FileOpenFile(files, sbcfg, filename, filename_len,
                 data, data_len, flags) == NULL)
     {
         retval = -1;
@@ -629,11 +635,12 @@ static int HTPFileParserTest03(void)
         goto end;
     }
 
-    if (http_state->files_ts->head->chunks_head == NULL ||
-        http_state->files_ts->head->chunks_head->len != 11)
+    if (http_state->files_ts->head == NULL ||
+        FileSize(http_state->files_ts->head) != 11)
     {
-        if (http_state->files_ts->head->chunks_head != NULL)
-            printf("filedata len not 11 but %u: ", http_state->files_ts->head->chunks_head->len);
+        if (http_state->files_ts->head != NULL)
+            printf("filedata len not 11 but %"PRIu64": ",
+                    FileSize(http_state->files_ts->head));
         goto end;
     }
 
@@ -882,31 +889,18 @@ static int HTPFileParserTest05(void)
     if (http_state->files_ts->head->next != http_state->files_ts->tail)
         goto end;
 
-    if (http_state->files_ts->head->chunks_head->len != 11) {
-        printf("expected 11 but file is %u bytes instead: ",
-                http_state->files_ts->head->chunks_head->len);
-        PrintRawDataFp(stdout, http_state->files_ts->head->chunks_head->data,
-                http_state->files_ts->head->chunks_head->len);
-        goto end;
-    }
-
-    if (memcmp("filecontent", http_state->files_ts->head->chunks_head->data,
-                http_state->files_ts->head->chunks_head->len) != 0) {
+    if (StreamingBufferCompareRawData(http_state->files_ts->head->sb,
+                (uint8_t *)"filecontent", 11) != 1)
+    {
         goto end;
     }
 
-    if (http_state->files_ts->tail->chunks_head->len != 11) {
-        printf("expected 11 but file is %u bytes instead: ",
-                http_state->files_ts->tail->chunks_head->len);
-        PrintRawDataFp(stdout, http_state->files_ts->tail->chunks_head->data,
-                http_state->files_ts->tail->chunks_head->len);
+    if (StreamingBufferCompareRawData(http_state->files_ts->tail->sb,
+                (uint8_t *)"FILECONTENT", 11) != 1)
+    {
         goto end;
     }
 
-    if (memcmp("FILECONTENT", http_state->files_ts->tail->chunks_head->data,
-                http_state->files_ts->tail->chunks_head->len) != 0) {
-        goto end;
-    }
     result = 1;
 end:
     if (alp_tctx != NULL)
@@ -1007,31 +1001,18 @@ static int HTPFileParserTest06(void)
     if (http_state->files_ts->head->next != http_state->files_ts->tail)
         goto end;
 
-    if (http_state->files_ts->head->chunks_head->len != 11) {
-        printf("expected 11 but file is %u bytes instead: ",
-                http_state->files_ts->head->chunks_head->len);
-        PrintRawDataFp(stdout, http_state->files_ts->head->chunks_head->data,
-                http_state->files_ts->head->chunks_head->len);
-        goto end;
-    }
-
-    if (memcmp("filecontent", http_state->files_ts->head->chunks_head->data,
-                http_state->files_ts->head->chunks_head->len) != 0) {
+    if (StreamingBufferCompareRawData(http_state->files_ts->head->sb,
+                (uint8_t *)"filecontent", 11) != 1)
+    {
         goto end;
     }
 
-    if (http_state->files_ts->tail->chunks_head->len != 11) {
-        printf("expected 11 but file is %u bytes instead: ",
-                http_state->files_ts->tail->chunks_head->len);
-        PrintRawDataFp(stdout, http_state->files_ts->tail->chunks_head->data,
-                http_state->files_ts->tail->chunks_head->len);
+    if (StreamingBufferCompareRawData(http_state->files_ts->tail->sb,
+                (uint8_t *)"FILECONTENT", 11) != 1)
+    {
         goto end;
     }
 
-    if (memcmp("FILECONTENT", http_state->files_ts->tail->chunks_head->data,
-                http_state->files_ts->tail->chunks_head->len) != 0) {
-        goto end;
-    }
     result = 1;
 end:
     if (alp_tctx != NULL)
@@ -1116,16 +1097,9 @@ static int HTPFileParserTest07(void)
         goto end;
     }
 
-    if (http_state->files_ts->head->chunks_head->len != 11) {
-        printf("expected 11 but file is %u bytes instead: ",
-                http_state->files_ts->head->chunks_head->len);
-        PrintRawDataFp(stdout, http_state->files_ts->head->chunks_head->data,
-                http_state->files_ts->head->chunks_head->len);
-        goto end;
-    }
-
-    if (memcmp("FILECONTENT", http_state->files_ts->head->chunks_head->data,
-                http_state->files_ts->head->chunks_head->len) != 0) {
+    if (StreamingBufferCompareRawData(http_state->files_ts->tail->sb,
+                (uint8_t *)"FILECONTENT", 11) != 1)
+    {
         goto end;
     }
 
@@ -1594,16 +1568,9 @@ static int HTPFileParserTest11(void)
         goto end;
     }
 
-    if (http_state->files_ts->head->chunks_head->len != 11) {
-        printf("expected 11 but file is %u bytes instead: ",
-                http_state->files_ts->head->chunks_head->len);
-        PrintRawDataFp(stdout, http_state->files_ts->head->chunks_head->data,
-                http_state->files_ts->head->chunks_head->len);
-        goto end;
-    }
-
-    if (memcmp("FILECONTENT", http_state->files_ts->head->chunks_head->data,
-                http_state->files_ts->head->chunks_head->len) != 0) {
+    if (StreamingBufferCompareRawData(http_state->files_ts->head->sb,
+                (uint8_t *)"FILECONTENT", 11) != 1)
+    {
         goto end;
     }
 
diff --git a/src/app-layer-smtp.c b/src/app-layer-smtp.c
index 5f89aea034..3f80d3e37a 100644
--- a/src/app-layer-smtp.c
+++ b/src/app-layer-smtp.c
@@ -225,7 +225,7 @@ SCEnumCharMap smtp_reply_map[ ] = {
 };
 
 /* Create SMTP config structure */
-SMTPConfig smtp_config = { 0, { 0, 0, 0, 0, 0 }, 0, 0, 0};
+SMTPConfig smtp_config = { 0, { 0, 0, 0, 0, 0 }, 0, 0, 0, STREAMING_BUFFER_CONFIG_INITIALIZER};
 
 static SMTPString *SMTPStringAlloc(void);
 
@@ -319,6 +319,8 @@ static void SMTPConfigure(void) {
         }
     }
 
+    smtp_config.sbcfg.buf_size = content_limit ? content_limit : 256;
+
     SCReturn;
 }
 
@@ -360,23 +362,21 @@ static void SMTPPruneFiles(FileContainer *files)
 
     File *file = files->head;
     while (file) {
-        SCLogDebug("file %p, file->chunks_head %p", file, file->chunks_head);
-        if (file->chunks_head) {
-            uint32_t window = smtp_config.content_inspect_window;
-            if (file->chunks_head->stream_offset == 0)
-                window = MAX(window, smtp_config.content_inspect_min_size);
-
-            uint64_t file_size = file->content_len_so_far;
-            uint64_t data_size = file_size - file->chunks_head->stream_offset;
-
-            SCLogDebug("window %"PRIu32", file_size %"PRIu64", data_size %"PRIu64,
-                    window, file_size, data_size);
-
-            if (data_size > (window * 3)) {
-                uint64_t left_edge = file_size - window;
-                SCLogDebug("file->content_inspected now %"PRIu64, left_edge);
-                file->content_inspected = left_edge;
-            }
+        SCLogDebug("file %p", file);
+        uint32_t window = smtp_config.content_inspect_window;
+        if (file->sb->stream_offset == 0)
+            window = MAX(window, smtp_config.content_inspect_min_size);
+
+        uint64_t file_size = FileSize(file);
+        uint64_t data_size = file_size - file->sb->stream_offset;
+
+        SCLogDebug("window %"PRIu32", file_size %"PRIu64", data_size %"PRIu64,
+                window, file_size, data_size);
+
+        if (data_size > (window * 3)) {
+            uint64_t left_edge = left_edge = file_size - window;
+            SCLogDebug("file->content_inspected now %"PRIu64, left_edge);
+            file->content_inspected = left_edge;
         }
 
         file = file->next;
@@ -451,7 +451,7 @@ int SMTPProcessDataChunk(const uint8_t *chunk, uint32_t len,
                 flags |= FILE_STORE;
             }
 
-            if (FileOpenFile(files, (uint8_t *) entity->filename, entity->filename_len,
+            if (FileOpenFile(files, &smtp_config.sbcfg, (uint8_t *) entity->filename, entity->filename_len,
                     (uint8_t *) chunk, len, flags|FILE_USE_DETECT) == NULL) {
                 ret = MIME_DEC_ERR_DATA;
                 SCLogDebug("FileOpenFile() failed");
@@ -1681,10 +1681,21 @@ void RegisterSMTPParsers(void)
 
 #ifdef UNITTESTS
 
+static void SMTPTestInitConfig(void)
+{
+    MimeDecSetConfig(&smtp_config.mime_config);
+
+    smtp_config.content_limit = FILEDATA_CONTENT_LIMIT;
+    smtp_config.content_inspect_window = FILEDATA_CONTENT_INSPECT_WINDOW;
+    smtp_config.content_inspect_min_size = FILEDATA_CONTENT_INSPECT_MIN_SIZE;
+
+    smtp_config.sbcfg.buf_size = FILEDATA_CONTENT_INSPECT_WINDOW;
+}
+
 /*
  * \test Test STARTTLS.
  */
-int SMTPParserTest01(void)
+static int SMTPParserTest01(void)
 {
     int result = 0;
     Flow f;
@@ -1760,6 +1771,7 @@ int SMTPParserTest01(void)
     f.proto = IPPROTO_TCP;
 
     StreamTcpInitConfig(TRUE);
+    SMTPTestInitConfig();
 
     SCMutexLock(&f.m);
     r = AppLayerParserParse(alp_tctx, &f, ALPROTO_SMTP, STREAM_TOCLIENT,
@@ -1873,7 +1885,7 @@ end:
 /**
  * \test Test multiple DATA commands(full mail transactions).
  */
-int SMTPParserTest02(void)
+static int SMTPParserTest02(void)
 {
     int result = 0;
     Flow f;
@@ -2118,6 +2130,7 @@ int SMTPParserTest02(void)
     f.proto = IPPROTO_TCP;
 
     StreamTcpInitConfig(TRUE);
+    SMTPTestInitConfig();
 
     SCMutexLock(&f.m);
     r = AppLayerParserParse(alp_tctx, &f, ALPROTO_SMTP, STREAM_TOCLIENT,
@@ -2657,7 +2670,7 @@ end:
 /**
  * \test Testing parsing pipelined commands.
  */
-int SMTPParserTest03(void)
+static int SMTPParserTest03(void)
 {
     int result = 0;
     Flow f;
@@ -2752,6 +2765,7 @@ int SMTPParserTest03(void)
     f.proto = IPPROTO_TCP;
 
     StreamTcpInitConfig(TRUE);
+    SMTPTestInitConfig();
 
     SCMutexLock(&f.m);
     r = AppLayerParserParse(alp_tctx, &f, ALPROTO_SMTP, STREAM_TOCLIENT,
@@ -2860,7 +2874,7 @@ end:
 /*
  * \test Test smtp with just <LF> delimter instead of <CR><LF>.
  */
-int SMTPParserTest04(void)
+static int SMTPParserTest04(void)
 {
     int result = 0;
     Flow f;
@@ -2899,6 +2913,7 @@ int SMTPParserTest04(void)
     f.proto = IPPROTO_TCP;
 
     StreamTcpInitConfig(TRUE);
+    SMTPTestInitConfig();
 
     SCMutexLock(&f.m);
     r = AppLayerParserParse(alp_tctx, &f, ALPROTO_SMTP, STREAM_TOCLIENT,
@@ -2952,7 +2967,7 @@ end:
 /*
  * \test Test STARTTLS fail.
  */
-int SMTPParserTest05(void)
+static int SMTPParserTest05(void)
 {
     int result = 0;
     Flow f;
@@ -3046,6 +3061,7 @@ int SMTPParserTest05(void)
     f.proto = IPPROTO_TCP;
 
     StreamTcpInitConfig(TRUE);
+    SMTPTestInitConfig();
 
     SCMutexLock(&f.m);
     r = AppLayerParserParse(alp_tctx, &f, ALPROTO_SMTP, STREAM_TOCLIENT,
@@ -3193,7 +3209,7 @@ end:
 /**
  * \test Test multiple DATA commands(full mail transactions).
  */
-int SMTPParserTest06(void)
+static int SMTPParserTest06(void)
 {
     int result = 0;
     Flow f;
@@ -3342,6 +3358,7 @@ int SMTPParserTest06(void)
     f.proto = IPPROTO_TCP;
 
     StreamTcpInitConfig(TRUE);
+    SMTPTestInitConfig();
 
     SCMutexLock(&f.m);
     r = AppLayerParserParse(alp_tctx, &f, ALPROTO_SMTP, STREAM_TOCLIENT,
@@ -3542,7 +3559,7 @@ end:
 /*
  * \test Test retrieving lines when frag'ed.
  */
-int SMTPParserTest07(void)
+static int SMTPParserTest07(void)
 {
     int result = 0;
     Flow f;
@@ -3580,6 +3597,7 @@ int SMTPParserTest07(void)
     f.proto = IPPROTO_TCP;
 
     StreamTcpInitConfig(TRUE);
+    SMTPTestInitConfig();
 
     SCMutexLock(&f.m);
     r = AppLayerParserParse(alp_tctx, &f, ALPROTO_SMTP, STREAM_TOSERVER,
@@ -3655,7 +3673,7 @@ end:
 /*
  * \test Test retrieving lines when frag'ed.
  */
-int SMTPParserTest08(void)
+static int SMTPParserTest08(void)
 {
     int result = 0;
     Flow f;
@@ -3693,6 +3711,7 @@ int SMTPParserTest08(void)
     f.proto = IPPROTO_TCP;
 
     StreamTcpInitConfig(TRUE);
+    SMTPTestInitConfig();
 
     SCMutexLock(&f.m);
     r = AppLayerParserParse(alp_tctx, &f, ALPROTO_SMTP, STREAM_TOSERVER,
@@ -3768,7 +3787,7 @@ end:
 /*
  * \test Test retrieving lines when frag'ed.
  */
-int SMTPParserTest09(void)
+static int SMTPParserTest09(void)
 {
     int result = 0;
     Flow f;
@@ -3806,6 +3825,7 @@ int SMTPParserTest09(void)
     f.proto = IPPROTO_TCP;
 
     StreamTcpInitConfig(TRUE);
+    SMTPTestInitConfig();
 
     SCMutexLock(&f.m);
     r = AppLayerParserParse(alp_tctx, &f, ALPROTO_SMTP, STREAM_TOSERVER,
@@ -3881,7 +3901,7 @@ end:
 /*
  * \test Test retrieving lines when frag'ed.
  */
-int SMTPParserTest10(void)
+static int SMTPParserTest10(void)
 {
     int result = 0;
     Flow f;
@@ -3919,6 +3939,7 @@ int SMTPParserTest10(void)
     f.proto = IPPROTO_TCP;
 
     StreamTcpInitConfig(TRUE);
+    SMTPTestInitConfig();
 
     SCMutexLock(&f.m);
     r = AppLayerParserParse(alp_tctx, &f, ALPROTO_SMTP, STREAM_TOSERVER,
@@ -3994,7 +4015,7 @@ end:
 /*
  * \test Test retrieving lines when frag'ed.
  */
-int SMTPParserTest11(void)
+static int SMTPParserTest11(void)
 {
     int result = 0;
     Flow f;
@@ -4026,6 +4047,7 @@ int SMTPParserTest11(void)
     f.proto = IPPROTO_TCP;
 
     StreamTcpInitConfig(TRUE);
+    SMTPTestInitConfig();
 
     SCMutexLock(&f.m);
     r = AppLayerParserParse(alp_tctx, &f, ALPROTO_SMTP, STREAM_TOSERVER,
@@ -4079,7 +4101,7 @@ end:
     return result;
 }
 
-int SMTPParserTest12(void)
+static int SMTPParserTest12(void)
 {
     int result = 0;
     Signature *s = NULL;
@@ -4124,6 +4146,7 @@ int SMTPParserTest12(void)
     f.alproto = ALPROTO_SMTP;
 
     StreamTcpInitConfig(TRUE);
+    SMTPTestInitConfig();
 
     de_ctx = DetectEngineCtxInit();
     if (de_ctx == NULL)
@@ -4200,7 +4223,7 @@ end:
     return result;
 }
 
-int SMTPParserTest13(void)
+static int SMTPParserTest13(void)
 {
     int result = 0;
     Signature *s = NULL;
@@ -4263,6 +4286,7 @@ int SMTPParserTest13(void)
     f.alproto = ALPROTO_SMTP;
 
     StreamTcpInitConfig(TRUE);
+    SMTPTestInitConfig();
 
     de_ctx = DetectEngineCtxInit();
     if (de_ctx == NULL)
@@ -4361,7 +4385,7 @@ end:
 /**
  * \test Test DATA command w/MIME message.
  */
-int SMTPParserTest14(void)
+static int SMTPParserTest14(void)
 {
     int result = 0;
     Flow f;
@@ -4541,6 +4565,7 @@ int SMTPParserTest14(void)
     f.proto = IPPROTO_TCP;
 
     StreamTcpInitConfig(TRUE);
+    SMTPTestInitConfig();
 
     SCMutexLock(&f.m);
     /* Welcome reply */
@@ -4788,8 +4813,8 @@ int SMTPParserTest14(void)
             printf("smtp-mime file name is incorrect");
             goto end;
         }
-        if(file->size != filesize){
-            printf("smtp-mime file size %"PRIu64" is incorrect", file->size);
+        if (FileSize(file) != filesize){
+            printf("smtp-mime file size %"PRIu64" is incorrect", FileSize(file));
             goto end;
         }
         static uint8_t org_binary[] = {
@@ -4810,12 +4835,12 @@ int SMTPParserTest14(void)
                 0x00, 0x00, 0x00, 0x00, 0x5C, 0x5C, 0x36, 0x36,
                 0x2E, 0x39, 0x33, 0x2E, 0x36, 0x38, 0x2E, 0x36,
                 0x5C, 0x7A, 0x00, 0x00, 0x38,};
-        uint64_t z;
-        for (z=0; z < filesize; z++){
-            if(org_binary[z] != file->chunks_head->data[z]){
-                printf("smtp-mime file data incorrect\n");
-                goto end;
-            }
+
+        if (StreamingBufferCompareRawData(file->sb,
+                    org_binary, sizeof(org_binary)) != 1)
+        {
+            printf("smtp-mime file data incorrect\n");
+            goto end;
         }
     }
 
@@ -4883,7 +4908,7 @@ end:
     return result;
 }
 
-int SMTPProcessDataChunkTest01(void){
+static int SMTPProcessDataChunkTest01(void){
     Flow f;
     FLOW_INITIALIZE(&f);
     f.flags = FLOW_FILE_NO_STORE_TS;
@@ -4895,7 +4920,7 @@ int SMTPProcessDataChunkTest01(void){
 }
 
 
-int SMTPProcessDataChunkTest02(void){
+static int SMTPProcessDataChunkTest02(void){
     char mimemsg[] = {0x4D, 0x49, 0x4D, 0x45, 0x2D, 0x56, 0x65, 0x72,
             0x73, 0x69, 0x6F, 0x6E, 0x3A, 0x20, 0x31, 0x2E,
             0x30, 0x0D, 0x0A, 0x43, 0x6F, 0x6E, 0x74, 0x65,
@@ -4953,7 +4978,7 @@ int SMTPProcessDataChunkTest02(void){
 
 
 
-int SMTPProcessDataChunkTest03(void){
+static int SMTPProcessDataChunkTest03(void){
     char mimemsg[] = {0x4D, 0x49, 0x4D, 0x45, 0x2D, 0x56, 0x65, 0x72, };
     char mimemsg2[] = {0x73, 0x69, 0x6F, 0x6E, 0x3A, 0x20, 0x31, 0x2E, };
     char mimemsg3[] = {0x30, 0x0D, 0x0A, 0x43, 0x6F, 0x6E, 0x74, 0x65, };
@@ -5007,7 +5032,7 @@ int SMTPProcessDataChunkTest03(void){
 }
 
 
-int SMTPProcessDataChunkTest04(void){
+static int SMTPProcessDataChunkTest04(void){
     char mimemsg[] = {0x4D, 0x49, 0x4D, 0x45, 0x2D, 0x56, 0x65, 0x72, };
     char mimemsg2[] = {0x73, 0x69, 0x6F, 0x6E, 0x3A, 0x20, 0x31, 0x2E, };
     char mimemsg3[] = {0x30, 0x0D, 0x0A, 0x43, 0x6F, 0x6E, 0x74, 0x65, };
@@ -5047,7 +5072,7 @@ int SMTPProcessDataChunkTest04(void){
     return ret == 0;
 }
 
-int SMTPProcessDataChunkTest05(void){
+static int SMTPProcessDataChunkTest05(void){
     char mimemsg[] = {0x4D, 0x49, 0x4D, 0x45, 0x2D, 0x56, 0x65, 0x72,
             0x73, 0x69, 0x6F, 0x6E, 0x3A, 0x20, 0x31, 0x2E,
             0x30, 0x0D, 0x0A, 0x43, 0x6F, 0x6E, 0x74, 0x65,
@@ -5075,7 +5100,7 @@ int SMTPProcessDataChunkTest05(void){
     FAIL_IF(file == NULL);
     ret = SMTPProcessDataChunk((uint8_t *)mimemsg, sizeof(mimemsg), state);
     FAIL_IF(ret != 0);
-    FAIL_IF(file->size != 106);
+    FAIL_IF((uint32_t)FileSize(file) != 106);
     SMTPStateFree(smtp_state);
     FLOW_DESTROY(&f);
     PASS;
diff --git a/src/app-layer-smtp.h b/src/app-layer-smtp.h
index f36029efe1..1f925c625e 100644
--- a/src/app-layer-smtp.h
+++ b/src/app-layer-smtp.h
@@ -27,6 +27,7 @@
 #include "decode-events.h"
 #include "util-decode-mime.h"
 #include "queue.h"
+#include "util-streaming-buffer.h"
 
 enum {
     SMTP_DECODER_EVENT_INVALID_REPLY,
@@ -94,6 +95,8 @@ typedef struct SMTPConfig {
     uint32_t content_limit;
     uint32_t content_inspect_min_size;
     uint32_t content_inspect_window;
+
+    StreamingBufferConfig sbcfg;
 } SMTPConfig;
 
 typedef struct SMTPState_ {
diff --git a/src/detect-engine-file.c b/src/detect-engine-file.c
index e50b4ce964..1afe967d6a 100644
--- a/src/detect-engine-file.c
+++ b/src/detect-engine-file.c
@@ -88,6 +88,7 @@ static int DetectFileInspect(ThreadVars *tv, DetectEngineThreadCtx *det_ctx,
         File *file = ffc->head;
         for (; file != NULL; file = file->next) {
             SCLogDebug("file");
+
             if (file->state == FILE_STATE_NONE) {
                 SCLogDebug("file state FILE_STATE_NONE");
                 continue;
@@ -109,13 +110,14 @@ static int DetectFileInspect(ThreadVars *tv, DetectEngineThreadCtx *det_ctx,
                 break;
             }
 
-            if ((s->file_flags & FILE_SIG_NEED_MAGIC) && file->chunks_head == NULL) {
+            uint64_t file_size = FileSize(file);
+            if ((s->file_flags & FILE_SIG_NEED_MAGIC) && file_size == 0) {
                 SCLogDebug("sig needs file content, but we don't have any");
                 r = DETECT_ENGINE_INSPECT_SIG_NO_MATCH;
                 break;
             }
 
-            if ((s->file_flags & FILE_SIG_NEED_FILECONTENT) && file->chunks_head == NULL) {
+            if ((s->file_flags & FILE_SIG_NEED_FILECONTENT) && file_size == 0) {
                 SCLogDebug("sig needs file content, but we don't have any");
                 r = DETECT_ENGINE_INSPECT_SIG_NO_MATCH;
                 break;
diff --git a/src/detect-engine-filedata-smtp.c b/src/detect-engine-filedata-smtp.c
index 64c3ba661b..e8bcf7769d 100644
--- a/src/detect-engine-filedata-smtp.c
+++ b/src/detect-engine-filedata-smtp.c
@@ -81,7 +81,7 @@ static inline int SMTPCreateSpace(DetectEngineThreadCtx *det_ctx, uint16_t size)
     return 0;
 }
 
-static uint8_t *DetectEngineSMTPGetBufferForTX(uint64_t tx_id,
+static const uint8_t *DetectEngineSMTPGetBufferForTX(uint64_t tx_id,
                                                DetectEngineCtx *de_ctx,
                                                DetectEngineThreadCtx *det_ctx,
                                                Flow *f, File *curr_file,
@@ -91,10 +91,10 @@ static uint8_t *DetectEngineSMTPGetBufferForTX(uint64_t tx_id,
 {
     SCEnter();
     int index = 0;
-    uint8_t *buffer = NULL;
+    const uint8_t *buffer = NULL;
     *buffer_len = 0;
     *stream_start_offset = 0;
-    FileData *curr_chunk = NULL;
+    uint64_t file_size = FileSize(curr_file);
 
     if (det_ctx->smtp_buffers_list_len == 0) {
         if (SMTPCreateSpace(det_ctx, 1) < 0)
@@ -129,23 +129,21 @@ static uint8_t *DetectEngineSMTPGetBufferForTX(uint64_t tx_id,
     SCLogDebug("smtp_config.content_limit %u, smtp_config.content_inspect_min_size %u",
                 smtp_config.content_limit, smtp_config.content_inspect_min_size);
 
-    SCLogDebug("file %p size %"PRIu64", state %d", curr_file, curr_file->content_len_so_far, curr_file->state);
+    SCLogDebug("file %p size %"PRIu64", state %d", curr_file, file_size, curr_file->state);
 
     /* no new data */
-    if (curr_file->content_inspected == curr_file->content_len_so_far) {
+    if (curr_file->content_inspected == file_size) {
         SCLogDebug("no new data");
         goto end;
     }
 
-    curr_chunk = curr_file->chunks_head;
-    if (curr_chunk == NULL) {
-        SCLogDebug("no data chunks to inspect for this transaction");
+    if (file_size == 0) {
+        SCLogDebug("no data to inspect for this transaction");
         goto end;
     }
 
-    if ((smtp_config.content_limit == 0 ||
-         curr_file->content_len_so_far < smtp_config.content_limit) &&
-        curr_file->content_len_so_far < smtp_config.content_inspect_min_size &&
+    if ((smtp_config.content_limit == 0 || file_size < smtp_config.content_limit) &&
+        file_size < smtp_config.content_inspect_min_size &&
         !(flags & STREAM_EOF) && !(curr_file->state > FILE_STATE_OPENED)) {
         SCLogDebug("we still haven't seen the entire content. "
                    "Let's defer content inspection till we see the "
@@ -153,51 +151,16 @@ static uint8_t *DetectEngineSMTPGetBufferForTX(uint64_t tx_id,
         goto end;
     }
 
-    int first = 1;
-    curr_chunk = curr_file->chunks_head;
-    while (curr_chunk != NULL) {
-        /* see if we can filter out chunks */
-        if (curr_file->content_inspected > 0) {
-            if (curr_chunk->stream_offset < curr_file->content_inspected) {
-                if ((curr_file->content_inspected - curr_chunk->stream_offset) > smtp_config.content_inspect_window) {
-                    curr_chunk = curr_chunk->next;
-                    continue;
-                } else {
-                    /* include this one */
-                }
-            } else {
-                /* include this one */
-            }
-        }
-
-        if (first) {
-            det_ctx->smtp[index].offset = curr_chunk->stream_offset;
-            first = 0;
-        }
+    StreamingBufferGetDataAtOffset(curr_file->sb,
+            &det_ctx->smtp[index].buffer, &det_ctx->smtp[index].buffer_len,
+            curr_file->content_inspected);
 
-        /* see if we need to grow the buffer */
-        if (det_ctx->smtp[index].buffer == NULL || (det_ctx->smtp[index].buffer_len + curr_chunk->len) > det_ctx->smtp[index].buffer_size) {
-            void *ptmp;
-            det_ctx->smtp[index].buffer_size += curr_chunk->len * 2;
-
-            if ((ptmp = SCRealloc(det_ctx->smtp[index].buffer, det_ctx->smtp[index].buffer_size)) == NULL) {
-                SCFree(det_ctx->smtp[index].buffer);
-                det_ctx->smtp[index].buffer = NULL;
-                det_ctx->smtp[index].buffer_size = 0;
-                det_ctx->smtp[index].buffer_len = 0;
-                goto end;
-            }
-            det_ctx->smtp[index].buffer = ptmp;
-        }
-        memcpy(det_ctx->smtp[index].buffer + det_ctx->smtp[index].buffer_len, curr_chunk->data, curr_chunk->len);
-        det_ctx->smtp[index].buffer_len += curr_chunk->len;
-
-        curr_chunk = curr_chunk->next;
-    }
+    det_ctx->smtp[index].offset = curr_file->content_inspected;
 
     /* updat inspected tracker */
-    curr_file->content_inspected = curr_file->chunks_tail->stream_offset + curr_file->chunks_tail->len;
-    SCLogDebug("curr_file->content_inspected now %"PRIu64, curr_file->content_inspected);
+    curr_file->content_inspected = FileSize(curr_file);
+
+    SCLogDebug("content_inspected %u, offset %u", (uint)curr_file->content_inspected, (uint)det_ctx->smtp[index].offset);
 
     buffer = det_ctx->smtp[index].buffer;
     *buffer_len = det_ctx->smtp[index].buffer_len;
@@ -221,7 +184,7 @@ int DetectEngineInspectSMTPFiledata(ThreadVars *tv,
     int match = 0;
     uint32_t buffer_len = 0;
     uint32_t stream_start_offset = 0;
-    uint8_t *buffer = 0;
+    const uint8_t *buffer = 0;
 
     if (ffc != NULL) {
         File *file = ffc->head;
@@ -240,7 +203,7 @@ int DetectEngineInspectSMTPFiledata(ThreadVars *tv,
         det_ctx->inspection_recursion_counter = 0;
         match = DetectEngineContentInspection(de_ctx, det_ctx, s, s->sm_lists[DETECT_SM_LIST_FILEDATA],
                                               f,
-                                              buffer,
+                                              (uint8_t *)buffer,
                                               buffer_len,
                                               stream_start_offset,
                                               DETECT_ENGINE_CONTENT_INSPECTION_MODE_FD_SMTP, NULL);
@@ -309,7 +272,7 @@ int DetectEngineRunSMTPMpm(DetectEngineCtx *de_ctx,
     uint32_t cnt = 0;
     uint32_t buffer_len = 0;
     uint32_t stream_start_offset = 0;
-    uint8_t *buffer = NULL;
+    const uint8_t *buffer = NULL;
 
     if (ffc != NULL) {
         File *file = ffc->head;
@@ -323,7 +286,7 @@ int DetectEngineRunSMTPMpm(DetectEngineCtx *de_ctx,
             if (buffer_len == 0)
                 goto end;
 
-            cnt += SMTPFiledataPatternSearch(det_ctx, buffer, buffer_len, flags);
+            cnt += SMTPFiledataPatternSearch(det_ctx, (uint8_t *)buffer, buffer_len, flags);
         }
     }
 end:
@@ -410,7 +373,7 @@ static int DetectEngineSMTPFiledataTest01(void)
     DetectEngineThreadCtxInit(&th_v, (void *)de_ctx, (void *)&det_ctx);
 
     SCMutexLock(&f.m);
-    int r = AppLayerParserParse(alp_tctx, &f, ALPROTO_SMTP, STREAM_TOSERVER, mimemsg, mimemsg_len);
+    int r = AppLayerParserParse(alp_tctx, &f, ALPROTO_SMTP, STREAM_TOSERVER|STREAM_START|STREAM_EOF, mimemsg, mimemsg_len);
     if (r != 0) {
         printf("AppLayerParse for smtp failed. Returned %d", r);
         SCMutexUnlock(&f.m);
diff --git a/src/detect-engine.c b/src/detect-engine.c
index f591056848..a319a0ea32 100644
--- a/src/detect-engine.c
+++ b/src/detect-engine.c
@@ -1699,11 +1699,6 @@ void DetectEngineThreadCtxFree(DetectEngineThreadCtx *det_ctx)
     /* SMTP */
     if (det_ctx->smtp != NULL) {
         SCLogDebug("det_ctx smtp %u", det_ctx->smtp_buffers_size);
-        for (i = 0; i < det_ctx->smtp_buffers_size; i++) {
-            if (det_ctx->smtp[i].buffer != NULL)
-                SCFree(det_ctx->smtp[i].buffer);
-            SCLogDebug("det_ctx->smtp[i].buffer_size %u", det_ctx->smtp[i].buffer_size);
-        }
         SCFree(det_ctx->smtp);
     }
 
diff --git a/src/detect-filemagic.c b/src/detect-filemagic.c
index 0ffabe741d..6ca7b6cd43 100644
--- a/src/detect-filemagic.c
+++ b/src/detect-filemagic.c
@@ -90,40 +90,21 @@ void DetectFilemagicRegister(void)
  */
 int FilemagicGlobalLookup(File *file)
 {
-    if (file == NULL || file->chunks_head == NULL) {
+    if (file == NULL || FileSize(file) == 0) {
         SCReturnInt(-1);
     }
 
-    /* initial chunk already matching our requirement */
-    if (file->chunks_head->len >= FILEMAGIC_MIN_SIZE) {
-        file->magic = MagicGlobalLookup(file->chunks_head->data, FILEMAGIC_MIN_SIZE);
-    } else {
-        uint8_t *buf = SCMalloc(FILEMAGIC_MIN_SIZE);
-        uint32_t size = 0;
-
-        if (likely(buf != NULL)) {
-            FileData *ffd = file->chunks_head;
-
-            for ( ; ffd != NULL; ffd = ffd->next) {
-                uint32_t copy_len = ffd->len;
-                if (size + ffd->len > FILEMAGIC_MIN_SIZE)
-                    copy_len = FILEMAGIC_MIN_SIZE - size;
-
-                memcpy(buf + size, ffd->data, copy_len);
-                size += copy_len;
-
-                if (size >= FILEMAGIC_MIN_SIZE) {
-                    file->magic = MagicGlobalLookup(buf, size);
-                    break;
-                }
-                /* file is done but smaller than FILEMAGIC_MIN_SIZE */
-                if (ffd->next == NULL && file->state >= FILE_STATE_CLOSED) {
-                    file->magic = MagicGlobalLookup(buf, size);
-                    break;
-                }
-            }
-
-            SCFree(buf);
+    const uint8_t *data = NULL;
+    uint32_t data_len = 0;
+    uint64_t offset = 0;
+
+    StreamingBufferGetData(file->sb,
+                           &data, &data_len, &offset);
+    if (offset == 0) {
+        if (FileSize(file) >= FILEMAGIC_MIN_SIZE) {
+            file->magic = MagicGlobalLookup(data, data_len);
+        } else if (file->state >= FILE_STATE_CLOSED) {
+            file->magic = MagicGlobalLookup(data, data_len);
         }
     }
 
@@ -140,43 +121,23 @@ int FilemagicGlobalLookup(File *file)
  */
 int FilemagicThreadLookup(magic_t *ctx, File *file)
 {
-    if (ctx == NULL || file == NULL || file->chunks_head == NULL) {
+    if (ctx == NULL || file == NULL || FileSize(file) == 0) {
         SCReturnInt(-1);
     }
 
-    /* initial chunk already matching our requirement */
-    if (file->chunks_head->len >= FILEMAGIC_MIN_SIZE) {
-        file->magic = MagicThreadLookup(ctx, file->chunks_head->data, FILEMAGIC_MIN_SIZE);
-    } else {
-        uint8_t *buf = SCMalloc(FILEMAGIC_MIN_SIZE);
-        uint32_t size = 0;
-
-        if (likely(buf != NULL)) {
-            FileData *ffd = file->chunks_head;
-
-            for ( ; ffd != NULL; ffd = ffd->next) {
-                uint32_t copy_len = ffd->len;
-                if (size + ffd->len > FILEMAGIC_MIN_SIZE)
-                    copy_len = FILEMAGIC_MIN_SIZE - size;
-
-                memcpy(buf + size, ffd->data, copy_len);
-                size += copy_len;
-
-                if (size >= FILEMAGIC_MIN_SIZE) {
-                    file->magic = MagicThreadLookup(ctx, buf, size);
-                    break;
-                }
-                /* file is done but smaller than FILEMAGIC_MIN_SIZE */
-                if (ffd->next == NULL && file->state >= FILE_STATE_CLOSED) {
-                    file->magic = MagicThreadLookup(ctx, buf, size);
-                    break;
-                }
-            }
-
-            SCFree(buf);
+    const uint8_t *data = NULL;
+    uint32_t data_len = 0;
+    uint64_t offset = 0;
+
+    StreamingBufferGetData(file->sb,
+                           &data, &data_len, &offset);
+    if (offset == 0) {
+        if (FileSize(file) >= FILEMAGIC_MIN_SIZE) {
+            file->magic = MagicThreadLookup(ctx, data, data_len);
+        } else if (file->state >= FILE_STATE_CLOSED) {
+            file->magic = MagicThreadLookup(ctx, data, data_len);
         }
     }
-
     SCReturnInt(0);
 }
 
diff --git a/src/detect-filesize.c b/src/detect-filesize.c
index b9a1177af4..92dcfebb9b 100644
--- a/src/detect-filesize.c
+++ b/src/detect-filesize.c
@@ -94,30 +94,32 @@ static int DetectFilesizeMatch (ThreadVars *t, DetectEngineThreadCtx *det_ctx, F
 
     DetectFilesizeData *fsd = (DetectFilesizeData *)m->ctx;
     int ret = 0;
-    SCLogDebug("file size %"PRIu64", check %"PRIu64, file->size, fsd->size1);
+    uint64_t file_size = FileSize(file);
+
+    SCLogDebug("file size %"PRIu64", check %"PRIu64, file_size, fsd->size1);
 
     if (file->state == FILE_STATE_CLOSED) {
         switch (fsd->mode) {
             case DETECT_FILESIZE_EQ:
-                if (file->size == fsd->size1)
+                if (file_size == fsd->size1)
                     ret = 1;
                 break;
             case DETECT_FILESIZE_LT:
-                if (file->size < fsd->size1)
+                if (file_size < fsd->size1)
                     ret = 1;
                 break;
             case DETECT_FILESIZE_GT:
-                if (file->size > fsd->size1)
+                if (file_size > fsd->size1)
                     ret = 1;
                 break;
             case DETECT_FILESIZE_RA:
-                if (file->size > fsd->size1 && file->size < fsd->size2)
+                if (file_size > fsd->size1 && file_size < fsd->size2)
                     ret = 1;
                 break;
         }
     /* truncated, error: only see if what we have meets the GT condition */
     } else if (file->state > FILE_STATE_CLOSED) {
-        if (fsd->mode == DETECT_FILESIZE_GT && file->size > fsd->size1)
+        if (fsd->mode == DETECT_FILESIZE_GT && file_size > fsd->size1)
             ret = 1;
     }
     SCReturnInt(ret);
diff --git a/src/detect.h b/src/detect.h
index 8a57b08161..4123922fac 100644
--- a/src/detect.h
+++ b/src/detect.h
@@ -704,7 +704,7 @@ typedef struct HttpReassembledBody_ {
 } HttpReassembledBody;
 
 typedef struct FiledataReassembledBody_ {
-    uint8_t *buffer;
+    const uint8_t *buffer;
     uint32_t buffer_size;   /**< size of the buffer itself */
     uint32_t buffer_len;    /**< data len in the buffer */
     uint64_t offset;        /**< data offset */
diff --git a/src/log-file.c b/src/log-file.c
index d95c4310ac..402bf936a2 100644
--- a/src/log-file.c
+++ b/src/log-file.c
@@ -303,7 +303,7 @@ static void LogFileWriteJsonRecord(LogFileLogThread *aft, const Packet *p, const
             break;
     }
     fprintf(fp, "\"stored\": %s, ", ff->flags & FILE_STORED ? "true" : "false");
-    fprintf(fp, "\"size\": %"PRIu64" ", ff->size);
+    fprintf(fp, "\"size\": %"PRIu64" ", FileSize(ff));
     fprintf(fp, "}\n");
     fflush(fp);
     SCMutexUnlock(&aft->file_ctx->fp_mutex);
diff --git a/src/log-filestore.c b/src/log-filestore.c
index 61a9800f74..15c7c9a586 100644
--- a/src/log-filestore.c
+++ b/src/log-filestore.c
@@ -273,7 +273,7 @@ static void LogFilestoreLogCloseMetaFile(const File *ff)
                 fprintf(fp, "STATE:             UNKNOWN\n");
                 break;
         }
-        fprintf(fp, "SIZE:              %"PRIu64"\n", ff->size);
+        fprintf(fp, "SIZE:              %"PRIu64"\n", FileSize(ff));
 
         fclose(fp);
     } else {
@@ -281,7 +281,8 @@ static void LogFilestoreLogCloseMetaFile(const File *ff)
     }
 }
 
-static int LogFilestoreLogger(ThreadVars *tv, void *thread_data, const Packet *p, const File *ff, const FileData *ffd, uint8_t flags)
+static int LogFilestoreLogger(ThreadVars *tv, void *thread_data, const Packet *p,
+        const File *ff, const uint8_t *data, uint32_t data_len, uint8_t flags)
 {
     SCEnter();
     LogFilestoreLogThread *aft = (LogFilestoreLogThread *)thread_data;
@@ -302,7 +303,7 @@ static int LogFilestoreLogger(ThreadVars *tv, void *thread_data, const Packet *p
         return 0;
     }
 
-    SCLogDebug("ff %p, ffd %p", ff, ffd);
+    SCLogDebug("ff %p, data %p, data_len %u", ff, data, data_len);
 
     snprintf(filename, sizeof(filename), "%s/file.%u",
             g_logfile_base_dir, ff->file_id);
@@ -319,7 +320,7 @@ static int LogFilestoreLogger(ThreadVars *tv, void *thread_data, const Packet *p
             return -1;
         }
     /* we can get called with a NULL ffd when we need to close */
-    } else if (ffd != NULL) {
+    } else if (data != NULL) {
         file_fd = open(filename, O_APPEND | O_NOFOLLOW | O_WRONLY);
         if (file_fd == -1) {
             SCLogDebug("failed to open file %s: %s", filename, strerror(errno));
@@ -328,7 +329,7 @@ static int LogFilestoreLogger(ThreadVars *tv, void *thread_data, const Packet *p
     }
 
     if (file_fd != -1) {
-        ssize_t r = write(file_fd, (const void *)ffd->data, (size_t)ffd->len);
+        ssize_t r = write(file_fd, (const void *)data, (size_t)data_len);
         if (r == -1) {
             SCLogDebug("write failed: %s", strerror(errno));
         }
diff --git a/src/output-filedata.c b/src/output-filedata.c
index 1e19cc9f7e..d77e837fd1 100644
--- a/src/output-filedata.c
+++ b/src/output-filedata.c
@@ -92,7 +92,8 @@ int OutputRegisterFiledataLogger(const char *name, FiledataLogger LogFunc, Outpu
 SC_ATOMIC_DECLARE(unsigned int, file_id);
 
 static int CallLoggers(ThreadVars *tv, OutputLoggerThreadStore *store_list,
-        Packet *p, const File *ff, const FileData *ffd, uint8_t flags)
+        Packet *p, const File *ff,
+        const uint8_t *data, uint32_t data_len, uint8_t flags)
 {
     OutputFiledataLogger *logger = list;
     OutputLoggerThreadStore *store = store_list;
@@ -103,7 +104,7 @@ static int CallLoggers(ThreadVars *tv, OutputLoggerThreadStore *store_list,
 
         SCLogDebug("logger %p", logger);
         PACKET_PROFILING_TMM_START(p, logger->module_id);
-        logger->LogFunc(tv, store->thread_data, (const Packet *)p, ff, ffd, flags);
+        logger->LogFunc(tv, store->thread_data, (const Packet *)p, ff, data, data_len, flags);
         PACKET_PROFILING_TMM_END(p, logger->module_id);
 
         file_logged = 1;
@@ -172,73 +173,50 @@ static TmEcode OutputFiledataLog(ThreadVars *tv, Packet *p, void *thread_data, P
 
             /* if we have no data chunks left to log, we should still
              * close the logger(s) */
-            if (ff->chunks_head == NULL && (file_trunc || file_close)) {
-                CallLoggers(tv, store, p, ff, NULL, OUTPUT_FILEDATA_FLAG_CLOSE);
+            if (FileSize(ff) == ff->content_stored &&
+                (file_trunc || file_close)) {
+                CallLoggers(tv, store, p, ff, NULL, 0, OUTPUT_FILEDATA_FLAG_CLOSE);
                 ff->flags |= FILE_STORED;
                 continue;
             }
 
-            FileData *ffd;
-            for (ffd = ff->chunks_head; ffd != NULL; ffd = ffd->next) {
-                uint8_t flags = 0;
-                int file_logged = 0;
-                FileData *write_ffd = ffd;
+            /* store */
 
-                SCLogDebug("ffd %p", ffd);
-
-                /* special case: on stream end we may inform the
-                 * loggers that the file is truncated. In this
-                 * case we already logged the current ffd, which
-                 * is the last in our list. */
-                if (ffd->stored == 1) {
-                    if (!(file_close == 1 && ffd->next == NULL)) {
-                        continue;
-                    }
-
-                    // call writer with NULL ffd, so it can 'close'
-                    // so really a 'close' msg
-                    write_ffd = NULL;
-                    flags |= OUTPUT_FILEDATA_FLAG_CLOSE;
-                    SCLogDebug("OUTPUT_FILEDATA_FLAG_CLOSE set");
-                }
-
-                /* store */
+            /* if file_id == 0, this is the first store of this file */
+            if (ff->file_id == 0) {
+                /* new file */
+                ff->file_id = SC_ATOMIC_ADD(file_id, 1);
+                flags |= OUTPUT_FILEDATA_FLAG_OPEN;
+            } else {
+                /* existing file */
+            }
 
-                /* if file_id == 0, this is the first store of this file */
-                if (ff->file_id == 0) {
-                    /* new file */
-                    ff->file_id = SC_ATOMIC_ADD(file_id, 1);
-                    flags |= OUTPUT_FILEDATA_FLAG_OPEN;
-                } else {
-                    /* existing file */
-                }
+            /* if file needs to be closed or truncated, inform
+             * loggers */
+            if ((file_close || file_trunc) && ff->state < FILE_STATE_CLOSED) {
+                ff->state = FILE_STATE_TRUNCATED;
+            }
 
-                /* if file needs to be closed or truncated, inform
-                 * loggers */
-                if ((file_close || file_trunc) && ff->state < FILE_STATE_CLOSED) {
-                    ff->state = FILE_STATE_TRUNCATED;
-                    SCLogDebug("ff->state = FILE_STATE_TRUNCATED set");
-                }
+            /* tell the logger we're closing up */
+            if (ff->state >= FILE_STATE_CLOSED)
+                flags |= OUTPUT_FILEDATA_FLAG_CLOSE;
 
-                /* for the last data chunk we have, also tell the logger
-                 * we're closing up */
-                if (ffd->next == NULL && ff->state >= FILE_STATE_CLOSED) {
-                    flags |= OUTPUT_FILEDATA_FLAG_CLOSE;
-                    SCLogDebug("OUTPUT_FILEDATA_FLAG_CLOSE set");
-                }
+            /* do the actual logging */
+            const uint8_t *data = NULL;
+            uint32_t data_len = 0;
 
-                SCLogDebug("ff %p ffd %p flags %02x", ff, write_ffd, flags);
-                /* do the actual logging */
-                file_logged = CallLoggers(tv, store, p, ff, write_ffd, flags);
+            StreamingBufferGetDataAtOffset(ff->sb,
+                    &data, &data_len,
+                    ff->content_stored);
 
-                if (file_logged) {
-                    ffd->stored = 1;
+            int file_logged = CallLoggers(tv, store, p, ff, data, data_len, flags);
+            if (file_logged) {
+                ff->content_stored += data_len;
 
-                    /* all done */
-                    if (flags & OUTPUT_FILEDATA_FLAG_CLOSE) {
-                        ff->flags |= FILE_STORED;
-                        break;
-                    }
+                /* all done */
+                if (flags & OUTPUT_FILEDATA_FLAG_CLOSE) {
+                    ff->flags |= FILE_STORED;
+                    break;
                 }
             }
         }
diff --git a/src/output-filedata.h b/src/output-filedata.h
index 8900cd489d..3a0a5d4341 100644
--- a/src/output-filedata.h
+++ b/src/output-filedata.h
@@ -34,7 +34,7 @@
 
 /** filedata logger function pointer type */
 typedef int (*FiledataLogger)(ThreadVars *, void *thread_data, const Packet *,
-        const File *, const FileData *, uint8_t);
+        const File *, const uint8_t *, uint32_t, uint8_t);
 
 /** packet logger condition function pointer type,
  *  must return true for packets that should be logged
diff --git a/src/output-json-file.c b/src/output-json-file.c
index 964638ea61..c4a7f5276b 100644
--- a/src/output-json-file.c
+++ b/src/output-json-file.c
@@ -151,7 +151,7 @@ static void FileWriteJsonRecord(JsonFileLogThread *aft, const Packet *p, const F
     if (ff->flags & FILE_STORED) {
         json_object_set_new(fjs, "file_id", json_integer(ff->file_id));
     }
-    json_object_set_new(fjs, "size", json_integer(ff->size));
+    json_object_set_new(fjs, "size", json_integer(FileSize(ff)));
     json_object_set_new(fjs, "tx_id", json_integer(ff->txid));
 
     /* originally just 'file', but due to bug 1127 naming it fileinfo */
diff --git a/src/util-file.c b/src/util-file.c
index c9aab3e126..3fc7d5ff23 100644
--- a/src/util-file.c
+++ b/src/util-file.c
@@ -58,7 +58,6 @@ static int g_file_force_tracking = 0;
 
 /* prototypes */
 static void FileFree(File *);
-static void FileDataFree(FileData *);
 
 void FileForceFilestoreEnable(void)
 {
@@ -101,61 +100,27 @@ int FileMagicSize(void)
     return 512;
 }
 
-static int FileAppendFileDataFilePtr(File *ff, FileData *ffd)
-{
-    SCEnter();
-
-    if (ff == NULL) {
-        SCReturnInt(-1);
-    }
-
-    if (ff->chunks_tail == NULL) {
-        ff->chunks_head = ffd;
-        ff->chunks_tail = ffd;
-        ff->content_len_so_far = ffd->len;
-    } else {
-        ff->chunks_tail->next = ffd;
-        ff->chunks_tail = ffd;
-        ff->content_len_so_far += ffd->len;
-    }
-
-#ifdef DEBUG
-    ff->chunks_cnt++;
-    if (ff->chunks_cnt > ff->chunks_cnt_max)
-        ff->chunks_cnt_max = ff->chunks_cnt;
-#endif
-
-#ifdef HAVE_NSS
-    if (ff->md5_ctx)
-        HASH_Update(ff->md5_ctx, ffd->data, ffd->len);
-#endif
-    SCReturnInt(0);
-}
-
-static int FileAppendFileData(FileContainer *ffc, FileData *ffd)
+/**
+ *  \brief get the size of the file
+ *
+ *  This doesn't reflect how much of the file we have in memory, just the
+ *  total size tracked so far.
+ */
+uint64_t FileSize(const File *file)
 {
-    SCEnter();
-
-    if (ffc == NULL) {
-        SCReturnInt(-1);
-    }
-
-    if (FileAppendFileDataFilePtr(ffc->tail, ffd) == -1)
-    {
-        SCReturnInt(-1);
+    if (file != NULL && file->sb != NULL) {
+        SCLogDebug("returning %"PRIu64,
+                file->sb->stream_offset + file->sb->buf_offset);
+        return file->sb->stream_offset + file->sb->buf_offset;
     }
-
-    SCReturnInt(0);
+    SCLogDebug("returning 0 (default)");
+    return 0;
 }
 
-
-
 static int FilePruneFile(File *file)
 {
     SCEnter();
 
-    SCLogDebug("file %p, file->chunks_cnt %"PRIu64, file, file->chunks_cnt);
-
     if (!(file->flags & FILE_NOMAGIC)) {
         /* need magic but haven't set it yet, bail out */
         if (file->magic == NULL)
@@ -166,37 +131,20 @@ static int FilePruneFile(File *file)
         SCLogDebug("file->flags & FILE_NOMAGIC == true");
     }
 
-    /* okay, we now know we can prune */
-    FileData *fd = file->chunks_head;
-
-    while (fd != NULL) {
-        SCLogDebug("fd %p", fd);
-
-        if (file->flags & FILE_NOSTORE || fd->stored == 1) {
-            /* keep chunks in memory as long as we still need to
-             * inspect them or parts of them */
-            if (file->flags & FILE_USE_DETECT) {
-                uint64_t right_edge = fd->stream_offset + fd->len;
-                if (file->content_inspected < right_edge)
-                    break;
-            }
-
-            file->chunks_head = fd->next;
-            if (file->chunks_tail == fd)
-                file->chunks_tail = fd->next;
+    uint64_t left_edge = file->content_stored;
+    if (file->flags & FILE_NOSTORE) {
+        left_edge = FileSize(file);
+    }
+    if (file->flags & FILE_USE_DETECT) {
+        left_edge = MIN(left_edge, file->content_inspected);
+    }
 
-            FileDataFree(fd);
+    if (left_edge) {
+        StreamingBufferSlideToOffset(file->sb, left_edge);
+    }
 
-            fd = file->chunks_head;
-#ifdef DEBUG
-            file->chunks_cnt--;
-            SCLogDebug("file->chunks_cnt %"PRIu64, file->chunks_cnt);
-#endif
-        } else if (fd->stored == 0) {
-            fd = NULL;
-            SCReturnInt(0);
-            break;
-        }
+    if (left_edge != FileSize(file)) {
+        SCReturnInt(0);
     }
 
     SCLogDebug("file->state %d. Is >= FILE_STATE_CLOSED: %s", file->state, (file->state >= FILE_STATE_CLOSED) ? "yes" : "no");
@@ -293,56 +241,6 @@ void FileContainerFree(FileContainer *ffc)
     SCFree(ffc);
 }
 
-/**
- *  \internal
- *
- *  \brief allocate a FileData chunk and set it up
- *
- *  \param data data chunk to store in the FileData
- *  \param data_len lenght of the data
- *
- *  \retval new FileData object
- */
-static FileData *FileDataAlloc(const uint8_t *data, uint32_t data_len)
-{
-    FileData *new = SCMalloc(sizeof(FileData));
-    if (unlikely(new == NULL)) {
-        return NULL;
-    }
-    memset(new, 0, sizeof(FileData));
-
-    new->data = SCMalloc(data_len);
-    if (new->data == NULL) {
-        SCFree(new);
-        return NULL;
-    }
-
-    new->len = data_len;
-    memcpy(new->data, data, data_len);
-
-    new->next = NULL;
-    return new;
-}
-
-/**
- *  \internal
- *
- *  \brief free a FileData object
- *
- *  \param ffd the flow file data object to free
- */
-static void FileDataFree(FileData *ffd)
-{
-    if (ffd == NULL)
-        return;
-
-    if (ffd->data != NULL) {
-        SCFree(ffd->data);
-    }
-
-    SCFree(ffd);
-}
-
 /**
  *  \brief Alloc a new File
  *
@@ -384,22 +282,14 @@ static void FileFree(File *ff)
     if (ff->magic != NULL)
         SCFree(ff->magic);
 
-    if (ff->chunks_head != NULL) {
-        FileData *ffd = ff->chunks_head;
-
-        while (ffd != NULL) {
-            FileData *next_ffd = ffd->next;
-            FileDataFree(ffd);
-            ffd = next_ffd;
-        }
+    if (ff->sb != NULL) {
+        StreamingBufferFree(ff->sb);
     }
 
 #ifdef HAVE_NSS
     if (ff->md5_ctx)
         HASH_Destroy(ff->md5_ctx);
 #endif
-    SCLogDebug("ff chunks_cnt %"PRIu64", chunks_cnt_max %"PRIu64,
-            ff->chunks_cnt, ff->chunks_cnt_max);
     SCFree(ff);
 }
 
@@ -456,7 +346,7 @@ static int FileStoreNoStoreCheck(File *ff)
 
     if (ff->flags & FILE_NOSTORE) {
         if (ff->state == FILE_STATE_OPENED &&
-                ff->size >= (uint64_t)FileMagicSize())
+            FileSize(ff) >= (uint64_t)FileMagicSize())
         {
             SCReturnInt(1);
         }
@@ -465,6 +355,18 @@ static int FileStoreNoStoreCheck(File *ff)
     SCReturnInt(0);
 }
 
+static int AppendData(File *file, const uint8_t *data, uint32_t data_len)
+{
+    StreamingBufferAppendNoTrack(file->sb, data, data_len);
+
+#ifdef HAVE_NSS
+    if (file->md5_ctx) {
+        HASH_Update(file->md5_ctx, data, data_len);
+    }
+#endif
+    SCReturnInt(0);
+}
+
 /**
  *  \brief Store a chunk of file data in the flow. The open "flowfile"
  *         will be used.
@@ -492,9 +394,6 @@ int FileAppendData(FileContainer *ffc, const uint8_t *data, uint32_t data_len)
         SCReturnInt(-1);
     }
 
-    ffc->tail->size += data_len;
-    SCLogDebug("file size is now %"PRIu64, ffc->tail->size);
-
     if (FileStoreNoStoreCheck(ffc->tail) == 1) {
 #ifdef HAVE_NSS
         /* no storage but forced md5 */
@@ -515,23 +414,11 @@ int FileAppendData(FileContainer *ffc, const uint8_t *data, uint32_t data_len)
 
     SCLogDebug("appending %"PRIu32" bytes", data_len);
 
-    FileData *ffd = FileDataAlloc(data, data_len);
-    if (ffd == NULL) {
+    if (AppendData(ffc->tail, data, data_len) != 0) {
         ffc->tail->state = FILE_STATE_ERROR;
         SCReturnInt(-1);
     }
 
-    if (ffc->tail->chunks_head == NULL)
-        ffd->stream_offset = 0;
-    else
-        ffd->stream_offset = ffc->tail->size;
-
-    /* append the data */
-    if (FileAppendFileData(ffc, ffd) < 0) {
-        ffc->tail->state = FILE_STATE_ERROR;
-        FileDataFree(ffd);
-        SCReturnInt(-1);
-    }
     SCReturnInt(0);
 }
 
@@ -539,6 +426,7 @@ int FileAppendData(FileContainer *ffc, const uint8_t *data, uint32_t data_len)
  *  \brief Open a new File
  *
  *  \param ffc flow container
+ *  \param sbcfg buffer config
  *  \param name filename character array
  *  \param name_len filename len
  *  \param data initial data
@@ -549,7 +437,8 @@ int FileAppendData(FileContainer *ffc, const uint8_t *data, uint32_t data_len)
  *
  *  \note filename is not a string, so it's not nul terminated.
  */
-File *FileOpenFile(FileContainer *ffc, const uint8_t *name, uint16_t name_len,
+File *FileOpenFile(FileContainer *ffc, const StreamingBufferConfig *sbcfg,
+        const uint8_t *name, uint16_t name_len,
         const uint8_t *data, uint32_t data_len, uint16_t flags)
 {
     SCEnter();
@@ -561,6 +450,13 @@ File *FileOpenFile(FileContainer *ffc, const uint8_t *name, uint16_t name_len,
         SCReturnPtr(NULL, "File");
     }
 
+    ff->sb = StreamingBufferInit(sbcfg);
+    if (ff->sb == NULL) {
+        FileFree(ff);
+        SCReturnPtr(NULL, "File");
+    }
+    SCLogDebug("ff->sb %p", ff->sb);
+
     if (flags & FILE_STORE || g_file_force_filestore) {
         FileStore(ff);
     } else if (flags & FILE_NOSTORE) {
@@ -595,22 +491,11 @@ File *FileOpenFile(FileContainer *ffc, const uint8_t *name, uint16_t name_len,
     FileContainerAdd(ffc, ff);
 
     if (data != NULL) {
-        //PrintRawDataFp(stdout, data, data_len);
-        ff->size += data_len;
-        SCLogDebug("file size is now %"PRIu64, ff->size);
-
-        FileData *ffd = FileDataAlloc(data, data_len);
-        if (ffd == NULL) {
-            ff->state = FILE_STATE_ERROR;
-            SCReturnPtr(NULL, "File");
-        }
-
-        /* append the data */
-        if (FileAppendFileData(ffc, ffd) < 0) {
+        if (AppendData(ff, data, data_len) != 0) {
             ff->state = FILE_STATE_ERROR;
-            FileDataFree(ffd);
             SCReturnPtr(NULL, "File");
         }
+        SCLogDebug("file size is now %"PRIu64, FileSize(ff));
     }
 
     SCReturnPtr(ff, "File");
@@ -629,12 +514,7 @@ static int FileCloseFilePtr(File *ff, const uint8_t *data,
         SCReturnInt(-1);
     }
 
-    ff->size += data_len;
-    SCLogDebug("file size is now %"PRIu64, ff->size);
-
     if (data != NULL) {
-        //PrintRawDataFp(stdout, data, data_len);
-
         if (ff->flags & FILE_NOSTORE) {
 #ifdef HAVE_NSS
             /* no storage but md5 */
@@ -642,16 +522,8 @@ static int FileCloseFilePtr(File *ff, const uint8_t *data,
                 HASH_Update(ff->md5_ctx, data, data_len);
 #endif
         } else {
-            FileData *ffd = FileDataAlloc(data, data_len);
-            if (ffd == NULL) {
-                ff->state = FILE_STATE_ERROR;
-                SCReturnInt(-1);
-            }
-
-            /* append the data */
-            if (FileAppendFileDataFilePtr(ff, ffd) < 0) {
+            if (AppendData(ff, data, data_len) != 0) {
                 ff->state = FILE_STATE_ERROR;
-                FileDataFree(ffd);
                 SCReturnInt(-1);
             }
         }
@@ -858,7 +730,7 @@ void FileDisableStoringForFile(File *ff)
     SCLogDebug("not storing this file");
     ff->flags |= FILE_NOSTORE;
 
-    if (ff->state == FILE_STATE_OPENED && ff->size >= (uint64_t)FileMagicSize()) {
+    if (ff->state == FILE_STATE_OPENED && FileSize(ff) >= (uint64_t)FileMagicSize()) {
         if (g_file_force_md5 == 0 && g_file_force_tracking == 0) {
             (void)FileCloseFilePtr(ff, NULL, 0,
                     (FILE_TRUNCATED|FILE_NOSTORE));
diff --git a/src/util-file.h b/src/util-file.h
index 9d87b593df..bdea4702f6 100644
--- a/src/util-file.h
+++ b/src/util-file.h
@@ -29,6 +29,8 @@
 #include <sechash.h>
 #endif
 
+#include "util-streaming-buffer.h"
+
 #define FILE_TRUNCATED  0x0001
 #define FILE_NOMAGIC    0x0002
 #define FILE_NOMD5      0x0004
@@ -51,38 +53,23 @@ typedef enum FileState_ {
     FILE_STATE_MAX
 } FileState;
 
-typedef struct FileData_ {
-    uint8_t *data;
-    uint32_t len;
-    uint64_t stream_offset;
-    int stored;     /* true if this chunk has been stored already
-                     * false otherwise */
-    struct FileData_ *next;
-} FileData;
-
 typedef struct File_ {
     uint16_t flags;
     uint16_t name_len;
     int16_t state;
+    StreamingBuffer *sb;
     uint64_t txid;                  /**< tx this file is part of */
     uint32_t file_id;
     uint8_t *name;
-    uint64_t size;                  /**< size tracked so far */
     char *magic;
-    FileData *chunks_head;
-    FileData *chunks_tail;
     struct File_ *next;
 #ifdef HAVE_NSS
     HASHContext *md5_ctx;
     uint8_t md5[MD5_LENGTH];
 #endif
-#ifdef DEBUG
-    uint64_t chunks_cnt;
-    uint64_t chunks_cnt_max;
-#endif
-    uint64_t content_len_so_far;
     uint64_t content_inspected;     /**< used in pruning if FILE_USE_DETECT
                                      *   flag is set */
+    uint64_t content_stored;
 } File;
 
 typedef struct FileContainer_ {
@@ -101,6 +88,7 @@ void FileContainerAdd(FileContainer *, File *);
  *  \brief Open a new File
  *
  *  \param ffc flow container
+ *  \param sbcfg buffer config
  *  \param name filename character array
  *  \param name_len filename len
  *  \param data initial data
@@ -110,9 +98,16 @@ void FileContainerAdd(FileContainer *, File *);
  *  \retval ff flowfile object
  *
  *  \note filename is not a string, so it's not nul terminated.
+ *
+ *  If flags contains the FILE_USE_DETECT bit, the pruning code will
+ *  consider not just the content_stored tracker, but also content_inspected.
+ *  It's the responsibility of the API user to make sure this tracker is
+ *  properly updated.
  */
-File *FileOpenFile(FileContainer *, const uint8_t *name, uint16_t name_len,
+File *FileOpenFile(FileContainer *, const StreamingBufferConfig *,
+        const uint8_t *name, uint16_t name_len,
         const uint8_t *data, uint32_t data_len, uint16_t flags);
+
 /**
  *  \brief Close a File
  *
@@ -194,4 +189,6 @@ void FileStoreFileById(FileContainer *fc, uint32_t);
 
 void FileTruncateAllOpenFiles(FileContainer *);
 
+uint64_t FileSize(const File *file);
+
 #endif /* __UTIL_FILE_H__ */
diff --git a/src/util-lua-common.c b/src/util-lua-common.c
index 0db2309102..d44d5a7c82 100644
--- a/src/util-lua-common.c
+++ b/src/util-lua-common.c
@@ -624,7 +624,7 @@ static int LuaCallbackFileInfoPushToStackFromFile(lua_State *luastate, const Fil
     lua_pushnumber(luastate, file->file_id);
     lua_pushnumber(luastate, file->txid);
     lua_pushlstring(luastate, (char *)file->name, file->name_len);
-    lua_pushnumber(luastate, file->size);
+    lua_pushnumber(luastate, FileSize(file));
     lua_pushstring (luastate, file->magic);
     lua_pushstring(luastate, md5ptr);
     return 6;