From: Russ Combs (rucombs) Date: Thu, 11 Aug 2016 19:44:13 +0000 (-0400) Subject: Merge pull request #554 in SNORT/snort3 from file_capture_mul to master X-Git-Tag: 3.0.0-233~297 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=ede36da313daeb078f7db102bb87b508c81d2ef6;p=thirdparty%2Fsnort3.git Merge pull request #554 in SNORT/snort3 from file_capture_mul to master Squashed commit of the following: commit efc9408cd944f1f9c570f4335950fdd52bdaaf21 Author: huica Date: Thu Aug 11 11:52:59 2016 -0400 remove additional header file commit c00c3ff16c1fbd3ad9d1025655998e552646f047 Author: huica Date: Thu Aug 11 10:20:26 2016 -0400 uncrusify commit 109afa308a619beb3c9273db8a59ce59c1971fba Merge: 33dd61d 6481ee3 Author: huica Date: Thu Aug 11 08:58:48 2016 -0400 Merge branch 'master' of https://bitbucket-eng-rtp1.cisco.com/bitbucket/scm/snort/snort3.git into file_capture_mul commit 33dd61d83dc2249cd223d2ccd2c08ce0139ffe6f Author: huica Date: Wed Aug 10 17:28:22 2016 -0400 mempool class commit acbf8857f6f4d88afe9e7eed5e9ebdba751a5bae Author: huica Date: Wed Aug 10 16:21:35 2016 -0400 update devnotes commit a5c98d6b7b894cc62ec01035ba858a735ef666c4 Merge: 5503866 a6b74b3 Author: huica Date: Wed Aug 10 15:17:29 2016 -0400 Merge branch 'master' of https://bitbucket-eng-rtp1.cisco.com/bitbucket/scm/snort/snort3.git into file_capture_mul commit 5503866788ad2ee6ddc7bd0b8a46f5c9aa00f0b9 Author: huica Date: Wed Aug 10 15:17:08 2016 -0400 address comments commit b375a13a0614f949f559cbb223ce788aae115022 Merge: 151ed4f 867b9c6 Author: huica Date: Wed Aug 10 10:29:34 2016 -0400 Merge branch 'master' of https://bitbucket-eng-rtp1.cisco.com/bitbucket/scm/snort/snort3.git into file_capture_mul commit 151ed4f94c19867770f238c2c4176e8fbe5f3e52 Merge: 7630daa f67d217 Author: huica Date: Tue Aug 9 10:21:15 2016 -0400 Merge branch 'master' of https://bitbucket-eng-rtp1.cisco.com/bitbucket/scm/snort/snort3.git into file_capture_mul commit 7630daa6615e2a7c2b563b29939c26d31d373860 Merge: 534326b 9b4b81e Author: huica Date: Tue Aug 2 14:20:19 2016 -0400 Merge branch 'master' of https://bitbucket-eng-rtp1.cisco.com/bitbucket/scm/snort/snort3.git into file_capture_mul commit 534326b51f8b6803edd3fcf38d4b69e5c6be82eb Author: huica Date: Tue Aug 2 14:20:11 2016 -0400 supports multi-thread safety for file mempool commit 9a9097a00e340c734c089cb624e70d2e4eff0614 Author: huica Date: Tue Aug 2 13:36:02 2016 -0400 refactor the file capture interfaces commit c98a98cf0756d015afeb64b1f2ad7acdd8cdc2eb Merge: e15ba25 41fcd73 Author: huica Date: Fri Jul 29 09:56:12 2016 -0400 Merge branch 'master' of https://bitbucket-eng-rtp1.cisco.com/bitbucket/scm/snort/snort3.git into file_capture_mul commit e15ba25ba46fb1a0d8df32a0801e3bf3e76f2899 Merge: 9ff6683 6f0c2cf Author: huica Date: Wed Jul 27 12:53:02 2016 -0400 Merge branch 'master' of https://bitbucket-eng-rtp1.cisco.com/bitbucket/scm/snort/snort3.git into file_capture_mul commit 9ff668388fc3d55fea41ac2c4d3c7bdfd22d7602 Merge: 0e65cdb bc4ea3f Author: huica Date: Fri Jul 22 15:42:15 2016 -0400 Merge branch 'master' of https://bitbucket-eng-rtp1.cisco.com/bitbucket/scm/snort/snort3.git into file_capture_mul commit 0e65cdb292300afe65b4411274083c257eb199e9 Author: huica Date: Mon Jul 11 14:16:23 2016 -0400 remove debug commit 771307d62b66ab16c9682ed38e912abf0d33ce97 Author: huica Date: Mon Jul 11 14:05:03 2016 -0400 Store and release file in another thread commit 41692c3fb38972de52fcbabe148103b4fe54bbde Merge: 4c4103d 65b2801 Author: huica Date: Mon Jul 11 11:52:58 2016 -0400 Merge branch 'master' of https://bitbucket-eng-rtp1.cisco.com/bitbucket/scm/snort/snort3.git into file_capture_mul commit 4c4103dadff3543e29dcebda403e6e868b9d570e Author: huica Date: Fri Jul 8 09:52:56 2016 -0400 store file to disk in different thread --- diff --git a/src/file_api/dev_notes.txt b/src/file_api/dev_notes.txt index 4e7a68095..089b8008e 100644 --- a/src/file_api/dev_notes.txt +++ b/src/file_api/dev_notes.txt @@ -5,7 +5,12 @@ inpsectors such as HTTP, SMTP, POP, IMAP, SMB, and FTP etc. * File capture: provides the ability to capture file data and save them in the mempool, then they can be stored to disk. Currently, files can be saved to the -logging folder. In this release, writing to disk is done inside packet thread. +logging folder. Writing to disk is done by a separate thread that will not block +packet thread. When a file is available to store, it will be put into a queue. +The writer thread will read from this queue to write to disk. In the multiple +packet thread case, many threads will write into this queue and one writer thread +serves all of them. Thread synchronization is done by mutex and conditional +variables for the queue. * File libraries: provides file type identification and file signature calculation diff --git a/src/file_api/file_capture.cc b/src/file_api/file_capture.cc index 47c9e3a85..7b2f0e9c9 100644 --- a/src/file_api/file_capture.cc +++ b/src/file_api/file_capture.cc @@ -37,6 +37,7 @@ #include #include +#include "file_config.h" #include "file_stats.h" #include "main/snort_config.h" @@ -44,22 +45,108 @@ #include "utils/util.h" #include "utils/stats.h" -FileMemPool* file_mempool = nullptr; +FileMemPool* FileCapture::file_mempool = nullptr; File_Capture_Stats file_capture_stats; +std::mutex FileCapture::capture_mutex; +std::condition_variable FileCapture::capture_cv; +std::thread* FileCapture::file_storer = nullptr; +std::queue FileCapture::files_waiting; +bool FileCapture::running = true; + +// Only one writer thread supported +void FileCapture::writer_thread() +{ + while (1) + { + // Wait until there are files + std::unique_lock lk(capture_mutex); + capture_cv.wait(lk, [] { return !running or files_waiting.size(); }); + + if (!running) + break; + + FileCapture* file = files_waiting.front(); + files_waiting.pop(); + lk.unlock(); + + file->store_file(); + delete file; + } +} + FileCapture::FileCapture() { reserved = 0; - file_size = 0; + capture_size = 0; last = head = nullptr; current_data = nullptr; current_data_len = 0; capture_state = FILE_CAPTURE_SUCCESS; } -FileCapture:: ~FileCapture() +FileCapture::~FileCapture() { - stop(); + FileCaptureBlock* file_block = head; + + if (reserved) + file_capture_stats.files_released_total++; + else + file_capture_stats.files_freed_total++; + + while (file_block) + { + FileCaptureBlock* next_block = file_block->next; + if (reserved) + { + if (file_mempool->m_release(file_block) != FILE_MEM_SUCCESS) + file_capture_stats.file_buffers_release_errors++; + file_capture_stats.file_buffers_released_total++; + } + else + { + if (file_mempool->m_free(file_block) != FILE_MEM_SUCCESS) + file_capture_stats.file_buffers_free_errors++; + file_capture_stats.file_buffers_freed_total++; + } + + file_block = next_block; + } + + head = last = nullptr; + + if (file_info) + delete file_info; +} + +void FileCapture::init() +{ + FileConfig& file_config = snort_conf->file_config; + init_mempool(file_config.capture_memcap, file_config.capture_block_size); + file_storer = new std::thread(writer_thread); +} + +/* + * Release all file capture memory etc, + * this must be called when snort exits + */ +void FileCapture::exit() +{ + running = false; + capture_cv.notify_one(); + + if (file_storer) + { + file_storer->join(); + delete file_storer; + file_storer = nullptr; + } + + if (file_mempool) + { + delete file_mempool; + file_mempool = nullptr; + } } /* @@ -67,73 +154,32 @@ FileCapture:: ~FileCapture() * * Arguments: * int64_t max_file_mem: memcap in megabytes - * int64_t block_size: file block size (metadata size excluded) - * - * Returns: NONE + * int64_t block_len: file block size (metadata size excluded) */ void FileCapture::init_mempool(int64_t max_file_mem, int64_t block_len) { int64_t block_size = block_len + sizeof (FileCapture); - /*Convert megabytes to bytes*/ - int64_t max_file_mem_in_bytes = max_file_mem * 1024 * 1024; - if (block_size <= 0) return; + /*Convert megabytes to bytes*/ + int64_t max_file_mem_in_bytes = max_file_mem * 1024 * 1024; + if (block_size & 7) block_size += (8 - (block_size & 7)); int max_files = max_file_mem_in_bytes / block_size; - file_mempool = (FileMemPool*)snort_calloc(sizeof(FileMemPool)); - - if ( file_mempool_init(file_mempool, max_files, block_size) != 0 ) - { - FatalError("File capture: Could not initialize file buffer mempool.\n"); - } -} - -/* - * Stop file capture, memory resource will be released if not reserved - * - * Returns: NONE - */ -void FileCapture::stop() -{ - /*free mempool*/ - if (reserved) - return; - - file_capture_stats.files_freed_total++; - FileCaptureBlock* fileblock = head; - while (fileblock) - { - if (file_mempool_free(file_mempool, fileblock) != FILE_MEM_SUCCESS) - file_capture_stats.file_buffers_free_errors++; - fileblock = fileblock->next; - file_capture_stats.file_buffers_freed_total++; - } - - head = last = nullptr; + file_mempool = new FileMemPool(max_files, block_size); } -/* - * Create file buffer in file mempool - * - * Args: - * FileMemPool *file_mempool: file mempool - * FileContext* context: file context - * - * Returns: - * FileCapture *: memory block that starts with file capture information - */ -inline FileCaptureBlock* FileCapture::create_file_buffer(FileMemPool* file_mempool) +inline FileCaptureBlock* FileCapture::create_file_buffer() { FileCaptureBlock* fileBlock; uint64_t num_files_queued; - fileBlock = (FileCaptureBlock*)file_mempool_alloc(file_mempool); + fileBlock = (FileCaptureBlock*)file_mempool->m_alloc(); if (fileBlock == nullptr) { @@ -147,7 +193,7 @@ inline FileCaptureBlock* FileCapture::create_file_buffer(FileMemPool* file_mempo fileBlock->length = 0; fileBlock->next = nullptr; /*Only one block initially*/ - num_files_queued = file_mempool_allocated(file_mempool); + num_files_queued = file_mempool->allocated(); if (file_capture_stats.file_buffers_used_max < num_files_queued) file_capture_stats.file_buffers_used_max = num_files_queued; @@ -158,19 +204,15 @@ inline FileCaptureBlock* FileCapture::create_file_buffer(FileMemPool* file_mempo * Save file to the buffer * If file needs to be extracted, buffer will be reserved * If file buffer isn't sufficient, need to add another buffer. - * - * Returns: - * 0: successful or file capture is disabled - * 1: fail to capture the file */ -inline FileCaptureState FileCapture::save_to_file_buffer(FileMemPool* file_mempool, - const uint8_t* file_data, int data_size, int64_t max_size) +inline FileCaptureState FileCapture::save_to_file_buffer(const uint8_t* file_data, + int data_size, int64_t max_size) { FileCaptureBlock* lastBlock = last; int64_t available_bytes; FileConfig& file_config = snort_conf->file_config; - if ( data_size + (int64_t)file_size > max_size) + if ( data_size + (int64_t)capture_size > max_size) { FILE_DEBUG_MSGS("Exceeding max file capture size!\n"); file_capture_stats.file_size_max++; @@ -198,7 +240,7 @@ inline FileCaptureState FileCapture::save_to_file_buffer(FileMemPool* file_mempo while (1) { /*get another block*/ - new_block = (FileCaptureBlock*)create_file_buffer(file_mempool); + new_block = (FileCaptureBlock*)create_file_buffer(); if (new_block == nullptr) { @@ -235,7 +277,7 @@ inline FileCaptureState FileCapture::save_to_file_buffer(FileMemPool* file_mempo lastBlock->length += data_size; } - file_size += data_size; + capture_size += data_size; return FILE_CAPTURE_SUCCESS; } @@ -278,7 +320,7 @@ FileCaptureState FileCapture::process_buffer(const uint8_t* file_data, */ if (!head) { - head = last = create_file_buffer(file_mempool); + head = last = create_file_buffer(); if (!head) { @@ -288,91 +330,79 @@ FileCaptureState FileCapture::process_buffer(const uint8_t* file_data, file_capture_stats.files_buffered_total++; } - return (save_to_file_buffer(file_mempool, file_data, data_size, - file_config.capture_max_size)); + return (save_to_file_buffer(file_data, data_size, file_config.capture_max_size)); } return FILE_CAPTURE_SUCCESS; } -/*Helper function for error*/ -static inline FileCaptureState ERROR_capture(FileCaptureState state) -{ - file_capture_stats.file_reserve_failures++; - return state; -} - // Preserve the file in memory until it is released -FileCaptureState FileCapture::reserve_file(FileContext* context) +FileCaptureState FileCapture::reserve_file(const FileInfo* file) { uint64_t fileSize; FileConfig& file_config = snort_conf->file_config; - if (!context || !context->is_file_capture_enabled()) - { - return ERROR_capture(FILE_CAPTURE_FAIL); - } - if (capture_state != FILE_CAPTURE_SUCCESS) { - return ERROR_capture(capture_state); + return error_capture(capture_state); } - FileCaptureBlock* fileInfo = head; + FileCaptureBlock* fileBlock = head; /* * Note: file size is updated at this point */ - fileSize = context->get_file_size(); + fileSize = file->get_file_size(); if ( fileSize < (unsigned)file_config.capture_min_size) { file_capture_stats.file_size_min++; - return ERROR_capture(FILE_CAPTURE_MIN); + return error_capture(FILE_CAPTURE_MIN); } if ( fileSize > (unsigned)file_config.capture_max_size) { file_capture_stats.file_size_max++; - return ERROR_capture(FILE_CAPTURE_MAX); + return error_capture(FILE_CAPTURE_MAX); } /* Create a file buffer if it is not done yet, * This is the case for small file */ - if (!fileInfo && context->is_file_capture_enabled()) + if (!fileBlock) { - fileInfo = create_file_buffer(file_mempool); + fileBlock = create_file_buffer(); - if (!fileInfo) + if (!fileBlock) { file_capture_stats.file_memcap_failures_reserve++; - return ERROR_capture(FILE_CAPTURE_MEMCAP); + return error_capture(FILE_CAPTURE_MEMCAP); } file_capture_stats.files_buffered_total++; - head = last = fileInfo; + head = last = fileBlock; } - if (!fileInfo) + if (!fileBlock) { - return ERROR_capture(FILE_CAPTURE_MEMCAP); + return error_capture(FILE_CAPTURE_MEMCAP); } /*Copy the last piece of file to file buffer*/ - if (save_to_file_buffer(file_mempool, current_data, + if (save_to_file_buffer(current_data, current_data_len, file_config.capture_max_size) ) { - return ERROR_capture(capture_state); + return error_capture(capture_state); } file_capture_stats.files_captured_total++; - reserved = true; current_block = head; - context->config_file_capture(false); + reserved = true; + + file_info = new FileInfo(*file); return FILE_CAPTURE_SUCCESS; } @@ -405,43 +435,18 @@ FileCaptureBlock* FileCapture::get_file_data(uint8_t** buff, int* size) } // Get the file size captured in the file buffer -uint64_t FileCapture::capture_size() const +uint64_t FileCapture::get_capture_size() const { - return file_size; + return capture_size; } /* - * Release the file that is reserved in memory, this function might be - * called in a different thread. - * - * Arguments: - * void *data: the memory block that stores file and its metadata - */ -void FileCapture::release_file() -{ - reserved = false; - - file_capture_stats.files_released_total++; - FileCaptureBlock* fileblock = head; - - while (fileblock) - { - if (file_mempool_release(file_mempool, fileblock) != FILE_MEM_SUCCESS) - file_capture_stats.file_buffers_release_errors++; - fileblock = fileblock->next; - file_capture_stats.file_buffers_released_total++; - } - - head = last = nullptr; -} - -/* - * writing file to the disk. + * writing file data to the disk. * * In the case of interrupt errors, the write is retried, but only for a * finite number of times. */ -void FileCapture::write_file(uint8_t* buf, size_t buf_len, FILE* fh) +void FileCapture::write_file_data(uint8_t* buf, size_t buf_len, FILE* fh) { int max_retries = 3; size_t bytes_written = 0; @@ -475,16 +480,12 @@ void FileCapture::write_file(uint8_t* buf, size_t buf_len, FILE* fh) } // Store files on local disk -void FileCapture::store_file(FileContext* file) +void FileCapture::store_file() { - uint8_t* sha = file->get_file_sig_sha256(); - if (!sha) + if (!file_info) return; - std::string file_name = file->sha_to_string(sha); - - std::string file_full_name; - get_instance_file(file_full_name, file_name.c_str()); + std::string& file_full_name = file_info->get_file_name(); /*Check whether the file exists*/ struct stat buffer; @@ -510,39 +511,47 @@ void FileCapture::store_file(FileContext* file) // Get file from file buffer if (!buff || !size ) { - //file_inspect_stats.file_read_failures++; return; } - write_file(buff, size, fh); + write_file_data(buff, size, fh); } while (file_mem); fclose(fh); } -/*Log file capture mempool usage*/ -void FileCapture::print_mem_usage() +// Queue files to be stored to disk +void FileCapture::store_file_async() { - if (file_mempool) - { - LogCount("Max buffers can allocate", file_mempool->total); - LogCount("Buffers in use", file_mempool_allocated(file_mempool)); - LogCount("Buffers in free list", file_mempool_freed(file_mempool)); - LogCount("Buffers in release list", file_mempool_released(file_mempool)); - } + // send data to the writer thread + if (!file_info) + return; + + uint8_t* sha = file_info->get_file_sig_sha256(); + if (!sha) + return; + + std::string file_name = file_info->sha_to_string(sha); + + std::string file_full_name; + get_instance_file(file_full_name, file_name.c_str()); + file_info->set_file_name(file_full_name.c_str(), file_full_name.size()); + + std::lock_guard lk(capture_mutex); + files_waiting.push(this); + capture_cv.notify_one(); } -/* - * Release all file capture memory etc, - * this must be called when snort exits - */ -void FileCapture::exit() +/*Log file capture mempool usage*/ +void FileCapture::print_mem_usage() { - if (file_mempool_destroy(file_mempool) == 0) + if (file_mempool) { - snort_free(file_mempool); - file_mempool = nullptr; + LogCount("Max buffers can allocate", file_mempool->total_objects()); + LogCount("Buffers in use", file_mempool->allocated()); + LogCount("Buffers in free list", file_mempool->freed()); + LogCount("Buffers in release list", file_mempool->released()); } } diff --git a/src/file_api/file_capture.h b/src/file_api/file_capture.h index cf21f7d7d..ce824284a 100644 --- a/src/file_api/file_capture.h +++ b/src/file_api/file_capture.h @@ -31,6 +31,11 @@ // 3) Then file data can be read through file_capture_read() // 4) Finally, fila data must be released from mempool file_capture_release() +#include +#include +#include +#include + #include "file_api.h" #include "file_lib.h" #include "file_mempool.h" @@ -48,18 +53,18 @@ public: ~FileCapture(); // this must be called during snort init - static void init_mempool(int64_t max_file_mem, int64_t block_size); + static void init(); // Capture file data to local buffer // This is the main function call to enable file capture FileCaptureState process_buffer(const uint8_t* file_data, int data_size, FilePosition pos); - // Stop file capture, memory resource will be released if not reserved - void stop(); - // Preserve the file in memory until it is released - FileCaptureState reserve_file(FileContext* context); + FileCaptureState reserve_file(const FileInfo*); + + // Whether file is reserved for store or analysis + bool is_reserved() { return reserved; } // Get the file that is reserved in memory, this should be called repeatedly // until NULL is returned to get the full file @@ -69,18 +74,16 @@ public: FileCaptureBlock* get_file_data(uint8_t** buff, int* size); // Get the file size captured in the file buffer - // Returns: - // the size of file - uint64_t capture_size() const; + // Returns: the size of file in bytes + uint64_t get_capture_size() const; // Store files on local disk - void store_file(FileContext *file); + void store_file(); - // Release the file that is reserved in memory, this function might be - // called in a different thread. - void release_file(); + // Store file to disk asynchronously + void store_file_async(); - // Log file capture mempool usage + // Log file capture mempoofile_contentl usage static void print_mem_usage(); // Exit file capture, release all file capture memory etc, @@ -89,22 +92,32 @@ public: private: - inline FileCaptureBlock* create_file_buffer(FileMemPool* file_mempool); - inline FileCaptureState save_to_file_buffer(FileMemPool* file_mempool, - const uint8_t* file_data, int data_size, int64_t max_size); - void write_file(uint8_t *buf, size_t buf_len, FILE *fh); + static void init_mempool(int64_t max_file_mem, int64_t block_size); + static void writer_thread(); + inline FileCaptureBlock* create_file_buffer(); + inline FileCaptureState save_to_file_buffer(const uint8_t* file_data, int data_size, + int64_t max_size); + void write_file_data(uint8_t* buf, size_t buf_len, FILE* fh); + + static FileMemPool* file_mempool; + static std::mutex capture_mutex; + static std::condition_variable capture_cv; + static std::thread* file_storer; + static std::queue files_waiting; + static bool running; bool reserved; - uint64_t file_size; /*file_size*/ + uint64_t capture_size; FileCaptureBlock* last; /* last block of file data */ FileCaptureBlock* head; /* first block of file data */ FileCaptureBlock* current_block = nullptr; /* current block of file data */ - const uint8_t *current_data; /*current file data*/ + const uint8_t* current_data; /*current file data*/ uint32_t current_data_len; FileCaptureState capture_state; + FileInfo* file_info = nullptr; }; -typedef struct _File_Capture_Stats +struct File_Capture_Stats { uint64_t files_buffered_total; uint64_t files_released_total; @@ -122,9 +135,16 @@ typedef struct _File_Capture_Stats uint64_t file_buffers_released_total; uint64_t file_buffers_free_errors; uint64_t file_buffers_release_errors; -} File_Capture_Stats; +}; extern File_Capture_Stats file_capture_stats; +/*Helper function for error*/ +static inline FileCaptureState error_capture(FileCaptureState state) +{ + file_capture_stats.file_reserve_failures++; + return state; +} + #endif diff --git a/src/file_api/file_lib.cc b/src/file_api/file_lib.cc index 47157e1c0..2426b699e 100644 --- a/src/file_api/file_lib.cc +++ b/src/file_api/file_lib.cc @@ -49,17 +49,12 @@ FileInfo::~FileInfo () delete[] sha256; } -FileInfo& FileInfo::operator=(const FileInfo& other) +void FileInfo::copy(const FileInfo& other) { - // check for self-assignment - if(&other == this) - return *this; - if (other.sha256) { sha256 = new uint8_t[SHA256_HASH_SIZE]; - if (sha256) - strncpy( (char *)sha256, (const char *)other.sha256, SHA256_HASH_SIZE); + memcpy( (char *)sha256, (const char *)other.sha256, SHA256_HASH_SIZE); } file_size = other.file_size; @@ -68,7 +63,20 @@ FileInfo& FileInfo::operator=(const FileInfo& other) file_id = other.file_id; file_name = other.file_name; verdict = other.verdict; +} +FileInfo::FileInfo(const FileInfo& other) +{ + copy(other); +} + +FileInfo& FileInfo::operator=(const FileInfo& other) +{ + // check for self-assignment + if(&other == this) + return *this; + + copy(other); return *this; } @@ -92,12 +100,12 @@ void FileInfo::set_file_size(uint64_t size) file_size = size; } -uint64_t FileInfo::get_file_size() +uint64_t FileInfo::get_file_size() const { return file_size; } -uint32_t FileInfo::get_file_type() +uint32_t FileInfo::get_file_type() const { return file_type_id; } @@ -107,7 +115,7 @@ void FileInfo::set_file_id(size_t id) file_id = id; } -size_t FileInfo::get_file_id() +size_t FileInfo::get_file_id() const { return file_id; } @@ -118,7 +126,7 @@ void FileInfo::set_file_direction(FileDirection dir) direction = dir; } -FileDirection FileInfo::get_file_direction() +FileDirection FileInfo::get_file_direction() const { return (direction); } @@ -128,7 +136,7 @@ void FileInfo::set_file_sig_sha256(uint8_t* signature) sha256 = signature; } -uint8_t* FileInfo::get_file_sig_sha256() +uint8_t* FileInfo::get_file_sig_sha256() const { return (sha256); } @@ -291,11 +299,6 @@ void FileContext::process_file_signature_sha256(const uint8_t* file_data, int si } } -FileCapture *FileContext::get_file_capture() -{ - return file_capture; -} - FileCaptureState FileContext::process_file_capture(const uint8_t* file_data, int data_size, FilePosition position) { @@ -313,12 +316,24 @@ FileCaptureState FileContext::process_file_capture(const uint8_t* file_data, return file_state.capture_state; } +FileCaptureState FileContext::reserve_file(FileCapture*& dest) +{ + if (!file_capture || !is_file_capture_enabled()) + return error_capture(FILE_CAPTURE_FAIL); + + FileCaptureState state = file_capture->reserve_file(this); + config_file_capture(false); + dest = file_capture; + file_capture = nullptr; + return state; +} + void FileContext::stop_file_capture() { if (file_capture) { delete file_capture; - file_capture = NULL; + file_capture = nullptr; } file_capture_enabled = false; diff --git a/src/file_api/file_lib.h b/src/file_api/file_lib.h index 4c7b3fefd..d9b2b9be8 100644 --- a/src/file_api/file_lib.h +++ b/src/file_api/file_lib.h @@ -41,19 +41,21 @@ class SO_PUBLIC FileInfo { public: virtual ~FileInfo(); + FileInfo(){}; + FileInfo(const FileInfo& other); FileInfo& operator=(const FileInfo& other); - uint32_t get_file_type(); + uint32_t get_file_type() const; void set_file_name(const char* file_name, uint32_t name_size); std::string& get_file_name(); void set_file_size(uint64_t size); - uint64_t get_file_size(); + uint64_t get_file_size() const; void set_file_direction(FileDirection dir); - FileDirection get_file_direction(); + FileDirection get_file_direction() const; void set_file_sig_sha256(uint8_t* signature); - uint8_t* get_file_sig_sha256(); + uint8_t* get_file_sig_sha256() const; std::string sha_to_string(const uint8_t *sha256); void set_file_id(size_t index); - size_t get_file_id(); + size_t get_file_id() const; FileVerdict verdict = FILE_VERDICT_UNKNOWN; protected: @@ -63,6 +65,9 @@ protected: uint32_t file_type_id = SNORT_FILE_TYPE_CONTINUE; uint8_t* sha256 = nullptr; size_t file_id = 0; + +private: + void copy(const FileInfo& other); }; class SO_PUBLIC FileContext: public FileInfo @@ -78,6 +83,10 @@ public: void stop_file_capture(); FileCaptureState process_file_capture(const uint8_t* file_data, int data_size, FilePosition pos); + // Preserve the file in memory until it is released + // The file reserved will be returned and it will be detached from file context/session + FileCaptureState reserve_file(FileCapture* &dest); + // Configuration functions void config_file_type(bool enabled); bool is_file_type_enabled(); @@ -89,8 +98,6 @@ public: //File properties uint64_t get_processed_bytes(); - FileCapture *get_file_capture(); - void set_file_config(FileConfig* file_config); FileConfig* get_file_config(); diff --git a/src/file_api/file_mempool.cc b/src/file_api/file_mempool.cc index 7e9cb3c2f..e855d9941 100644 --- a/src/file_api/file_mempool.cc +++ b/src/file_api/file_mempool.cc @@ -30,7 +30,6 @@ #include #include "file_mempool.h" -#include "main/snort_debug.h" #include "utils/util.h" /*This magic is used for double free detection*/ @@ -39,21 +38,21 @@ typedef uint64_t MagicType; #ifdef DEBUG_MSGS -static inline void file_mempool_verify(FileMemPool* mempool) +void FileMemPool::verify() { uint64_t free_size; uint64_t release_size; - free_size = cbuffer_used(mempool->free_list); - release_size = cbuffer_used(mempool->released_list); + free_size = cbuffer_used(free_list); + release_size = cbuffer_used(released_list); - if (free_size > cbuffer_size(mempool->free_list)) + if (free_size > cbuffer_size(free_list)) { ErrorMessage("%s(%d) file_mempool: failed to verify free list!\n", __FILE__, __LINE__); } - if (release_size > cbuffer_size(mempool->released_list)) + if (release_size > cbuffer_size(released_list)) { ErrorMessage("%s(%d) file_mempool: failed to verify release list!\n", __FILE__, __LINE__); @@ -62,111 +61,89 @@ static inline void file_mempool_verify(FileMemPool* mempool) /* The free mempool and size of release mempool should be smaller than * or equal to the size of mempool */ - if (free_size + release_size > mempool->total) + if (free_size + release_size > total) { ErrorMessage("%s(%d) file_mempool: failed to verify mempool size!\n", __FILE__, __LINE__); } } + #endif -static inline void file_mempool_free_pools(FileMemPool* mempool) +void FileMemPool::free_pools() { - if (mempool == NULL) - return; - - if (mempool->datapool != NULL) + if (datapool != nullptr) { - snort_free(mempool->datapool); - mempool->datapool = NULL; + snort_free(datapool); + datapool = nullptr; } - cbuffer_free(mempool->free_list); - cbuffer_free(mempool->released_list); + cbuffer_free(free_list); + cbuffer_free(released_list); } -/* Function: int file_mempool_init(FileMemPool *FileMemPool, - * PoolCount num_objects, size_t obj_size) - * +/* * Purpose: initialize a FileMemPool object and allocate memory for it * Args: - * FileMemPool - pointer to a FileMemPool struct * num_objects - number of items in this pool * obj_size - size of the items - * - * Returns: - * FILE_MEM_SUCCESS - * FILE_MEM_FAIL */ -int file_mempool_init(FileMemPool* mempool, uint64_t num_objects, size_t obj_size) +FileMemPool::FileMemPool(uint64_t num_objects, size_t o_size) { unsigned int i; - if ((mempool == NULL) || (num_objects < 1) || (obj_size < 1)) - return FILE_MEM_FAIL; + if ((num_objects < 1) || (o_size < 1)) + return; - mempool->obj_size = obj_size; + obj_size = o_size; // this is the basis pool that represents all the *data pointers in the list - mempool->datapool = (void**)snort_calloc(num_objects, obj_size); + datapool = (void**)snort_calloc(num_objects, obj_size); /* sets up the memory list */ - mempool->free_list = cbuffer_init(num_objects); - if (!mempool->free_list) + free_list = cbuffer_init(num_objects); + if (!free_list) { - ErrorMessage("%s(%d) file_mempool_init(): Failed to init free list\n", + ErrorMessage("%s(%d) file_mempool: Failed to init free list\n", __FILE__, __LINE__); - file_mempool_free_pools(mempool); - return FILE_MEM_FAIL; + free_pools(); + return; } - mempool->released_list = cbuffer_init(num_objects); - if (!mempool->released_list) + released_list = cbuffer_init(num_objects); + if (!released_list) { - ErrorMessage("%s(%d) file_mempool_init(): " + ErrorMessage("%s(%d) file_mempool: " "Failed to init release list\n", __FILE__, __LINE__); - file_mempool_free_pools(mempool); - return FILE_MEM_FAIL; + free_pools(); + return; } for (i=0; idatapool) + (i * mempool->obj_size); + void* data = ((char*)datapool) + (i * obj_size); - if (cbuffer_write(mempool->free_list, data)) + if (cbuffer_write(free_list, data)) { - ErrorMessage("%s(%d) file_mempool_init(): " + ErrorMessage("%s(%d) file_mempool: " "Failed to add to free list\n", __FILE__, __LINE__); - file_mempool_free_pools(mempool); - return FILE_MEM_FAIL; + free_pools(); + return; } *(MagicType*)data = FREE_MAGIC; - mempool->total++; + total++; } - - return FILE_MEM_SUCCESS; } /* * Destroy a set of FileMemPool objects * - * Args: - * FileMemPool: pointer to a FileMemPool struct - * - * Return: - * FILE_MEM_SUCCESS - * FILE_MEM_FAIL */ -int file_mempool_destroy(FileMemPool* mempool) +FileMemPool::~FileMemPool() { - if (mempool == NULL) - return FILE_MEM_FAIL; - - file_mempool_free_pools(mempool); - - return FILE_MEM_SUCCESS; + free_pools(); } /* @@ -178,18 +155,15 @@ int file_mempool_destroy(FileMemPool* mempool) * Returns: a pointer to the FileMemPool object on success, NULL on failure */ -void* file_mempool_alloc(FileMemPool* mempool) +void* FileMemPool::m_alloc() { void* b = NULL; - if (mempool == NULL) - { - return NULL; - } + std::lock_guard lock(pool_mutex); - if (cbuffer_read(mempool->free_list, &b)) + if (cbuffer_read(free_list, &b)) { - if (cbuffer_read(mempool->released_list, &b)) + if (cbuffer_read(released_list, &b)) { return NULL; } @@ -201,7 +175,7 @@ void* file_mempool_alloc(FileMemPool* mempool) __FILE__, __LINE__); } - DEBUG_WRAP(file_mempool_verify(mempool); ); + DEBUG_WRAP(verify(); ); return b; } @@ -209,18 +183,10 @@ void* file_mempool_alloc(FileMemPool* mempool) /* * Free a new object from the buffer * We use circular buffer to synchronize one reader and one writer - * - * Args: - * FileMemPool: pointer to a circular buffer struct - * void *obj : memory object - * - * Return: - * FILE_MEM_SUCCESS - * FILE_MEM_FAIL */ -static inline int _file__mempool_remove(CircularBuffer* cb, void* obj) +int FileMemPool::remove(CircularBuffer* cb, void* obj) { - if (obj == NULL) + if (obj == nullptr) return FILE_MEM_FAIL; if (cbuffer_write(cb, obj)) @@ -240,27 +206,13 @@ static inline int _file__mempool_remove(CircularBuffer* cb, void* obj) return FILE_MEM_SUCCESS; } -/* - * Free a new object from the FileMemPool - * - * Args: - * FileMemPool: pointer to a FileMemPool struct - * void *obj : memory object - * - * Return: - * FILE_MEM_SUCCESS - * FILE_MEM_FAIL - */ - -int file_mempool_free(FileMemPool* mempool, void* obj) +int FileMemPool::m_free(void* obj) { - int ret; + std::lock_guard lock(pool_mutex); - assert(mempool); + int ret = remove(free_list, obj); - ret = _file__mempool_remove(mempool->free_list, obj); - - DEBUG_WRAP(file_mempool_verify(mempool); ); + DEBUG_WRAP(verify(); ); return ret; } @@ -270,47 +222,36 @@ int file_mempool_free(FileMemPool* mempool, void* obj) * This can be called by a different thread calling * file_mempool_alloc() * * - * Args: - * FileMemPool: pointer to a FileMemPool struct - * void *obj : memory object - * - * Return: - * FILE_MEM_SUCCESS - * FILE_MEM_FAIL */ -int file_mempool_release(FileMemPool* mempool, void* obj) +int FileMemPool::m_release(void* obj) { - int ret; - - if (mempool == NULL) - return FILE_MEM_FAIL; + std::lock_guard lock(pool_mutex); /*A writer that might from different thread*/ - ret = _file__mempool_remove(mempool->released_list, obj); + int ret = remove(released_list, obj); - DEBUG_WRAP(file_mempool_verify(mempool); ); + DEBUG_WRAP(verify(); ); return ret; } /* Returns number of elements allocated in current buffer*/ -uint64_t file_mempool_allocated(FileMemPool* mempool) +uint64_t FileMemPool::allocated() { - uint64_t total_freed = - file_mempool_released(mempool) + file_mempool_freed(mempool); - return (mempool->total - total_freed); + uint64_t total_freed = released() + freed(); + return (total - total_freed); } /* Returns number of elements freed in current buffer*/ -uint64_t file_mempool_freed(FileMemPool* mempool) +uint64_t FileMemPool::freed() { - return (cbuffer_used(mempool->free_list)); + return (cbuffer_used(free_list)); } /* Returns number of elements released in current buffer*/ -uint64_t file_mempool_released(FileMemPool* mempool) +uint64_t FileMemPool::released() { - return (cbuffer_used(mempool->released_list)); + return (cbuffer_used(released_list)); } diff --git a/src/file_api/file_mempool.h b/src/file_api/file_mempool.h index 5d015e332..785a62bdb 100644 --- a/src/file_api/file_mempool.h +++ b/src/file_api/file_mempool.h @@ -22,58 +22,68 @@ #ifndef FILE_MEMPOOL_H #define FILE_MEMPOOL_H - // This mempool implementation has very efficient alloc/free operations. - // In addition, it provides thread-safe alloc/free for one allocation/free - // thread and one release thread. - // One more bonus: Double free detection is also added into this library - // This is a thread safe version of memory pool for one writer and one reader thread +// This mempool implementation has very efficient alloc/free operations. +// In addition, it provides thread-safe alloc/free for one allocation/free +// thread and one release thread. +// One more bonus: Double free detection is also added into this library +// This is a thread safe version of memory pool for one writer and one reader thread + +#include #include "main/snort_types.h" +#include "main/snort_debug.h" #include "circular_buffer.h" #define FILE_MEM_SUCCESS 0 // FIXIT-L use bool #define FILE_MEM_FAIL -1 -typedef struct _FileMemPool +class FileMemPool { - void** datapool; /* memory buffer */ +public: - uint64_t total; + FileMemPool(uint64_t num_objects, size_t obj_size); + ~FileMemPool(); - CircularBuffer* free_list; - CircularBuffer* released_list; - size_t obj_size; -} FileMemPool; + // Allocate a new object from the FileMemPool + // Note: Memory block will not be zeroed for performance + // Returns: a pointer to the FileMemPool object on success, nullptr on failure + void* m_alloc(); -// This must be called before file mempool is used -// Return: FILE_MEM_SUCCESS or FILE_MEM_FAIL -int file_mempool_init(FileMemPool* mempool, uint64_t num_objects, size_t obj_size); + // This must be called by the same thread calling file_mempool_alloc() + // Return: FILE_MEM_SUCCESS or FILE_MEM_FAIL + int m_free(void* obj); -// This must be called during snort exits -// Return: FILE_MEM_SUCCESS or FILE_MEM_FAIL -int file_mempool_destroy(FileMemPool* mempool); + // This can be called by a different thread calling file_mempool_alloc() + // Return: FILE_MEM_SUCCESS or FILE_MEM_FAIL + int m_release(void* obj); -// Allocate a new object from the FileMemPool -// Note: Memory block will not be zeroed for performance -// Returns: a pointer to the FileMemPool object on success, NULL on failure -void* file_mempool_alloc(FileMemPool* mempool); + //Returns number of elements allocated + uint64_t allocated(); -// This must be called by the same thread calling file_mempool_alloc() -// Return: FILE_MEM_SUCCESS or FILE_MEM_FAIL -int file_mempool_free(FileMemPool* mempool, void* obj); + // Returns number of elements freed in current buffer + uint64_t freed(); -// This can be called by a different thread calling file_mempool_alloc() -// Return: FILE_MEM_SUCCESS or FILE_MEM_FAIL -int file_mempool_release(FileMemPool* mempool, void* obj); + // Returns number of elements released in current buffer + uint64_t released(); -//Returns number of elements allocated in current buffer -uint64_t file_mempool_allocated(FileMemPool* mempool); + // Returns total number of elements in current buffer + uint64_t total_objects() { return total; } -// Returns number of elements freed in current buffer -uint64_t file_mempool_freed(FileMemPool* mempool); +private: -// Returns number of elements released in current buffer -uint64_t file_mempool_released(FileMemPool* mempool); + void free_pools(); + int remove(CircularBuffer* cb, void* obj); +#ifdef DEBUG_MSGS + void verify(); +#endif + + void** datapool; /* memory buffer */ + uint64_t total; + CircularBuffer* free_list; + CircularBuffer* released_list; + size_t obj_size; + std::mutex pool_mutex; +}; #endif diff --git a/src/file_api/file_policy.cc b/src/file_api/file_policy.cc index 7f866b504..0e3ea16e5 100644 --- a/src/file_api/file_policy.cc +++ b/src/file_api/file_policy.cc @@ -185,14 +185,11 @@ FileVerdict FilePolicy::signature_lookup(Flow* flow, FileContext* file ) if (rule.use.capture_enabled) { - FileCapture *capture = file->get_file_capture(); - if (capture) + FileCapture *captured = nullptr; + + if (file->reserve_file(captured) == FILE_CAPTURE_SUCCESS) { - if (capture->reserve_file(file) == FILE_CAPTURE_SUCCESS) - { - capture->store_file(file); - capture->release_file(); - } + captured->store_file_async(); } } diff --git a/src/file_api/file_service.cc b/src/file_api/file_service.cc index 466c529b5..346b66472 100644 --- a/src/file_api/file_service.cc +++ b/src/file_api/file_service.cc @@ -65,15 +65,12 @@ void FileService::init() void FileService::post_init() { - FileConfig& file_config = snort_conf->file_config; - FilePolicy& fp = get_inspect(); fp.load(); - if ( file_capture_enabled) - FileCapture::init_mempool(file_config.capture_memcap, - file_config.capture_block_size); + if (file_capture_enabled) + FileCapture::init(); } void FileService::close()