* File capture: provides the ability to capture file data and save them in the
mempool, then they can be stored to disk. Currently, files can be saved to the
-logging folder. In this release, writing to disk is done inside packet thread.
+logging folder. Writing to disk is done by a separate thread that will not block
+packet thread. When a file is available to store, it will be put into a queue.
+The writer thread will read from this queue to write to disk. In the multiple
+packet thread case, many threads will write into this queue and one writer thread
+serves all of them. Thread synchronization is done by mutex and conditional
+variables for the queue.
* File libraries: provides file type identification and file signature
calculation
#include <sys/types.h>
#include <sys/stat.h>
+#include "file_config.h"
#include "file_stats.h"
#include "main/snort_config.h"
#include "utils/util.h"
#include "utils/stats.h"
-FileMemPool* file_mempool = nullptr;
+FileMemPool* FileCapture::file_mempool = nullptr;
File_Capture_Stats file_capture_stats;
+std::mutex FileCapture::capture_mutex;
+std::condition_variable FileCapture::capture_cv;
+std::thread* FileCapture::file_storer = nullptr;
+std::queue<FileCapture*> FileCapture::files_waiting;
+bool FileCapture::running = true;
+
+// Only one writer thread supported
+void FileCapture::writer_thread()
+{
+ while (1)
+ {
+ // Wait until there are files
+ std::unique_lock<std::mutex> lk(capture_mutex);
+ capture_cv.wait(lk, [] { return !running or files_waiting.size(); });
+
+ if (!running)
+ break;
+
+ FileCapture* file = files_waiting.front();
+ files_waiting.pop();
+ lk.unlock();
+
+ file->store_file();
+ delete file;
+ }
+}
+
FileCapture::FileCapture()
{
reserved = 0;
- file_size = 0;
+ capture_size = 0;
last = head = nullptr;
current_data = nullptr;
current_data_len = 0;
capture_state = FILE_CAPTURE_SUCCESS;
}
-FileCapture:: ~FileCapture()
+FileCapture::~FileCapture()
{
- stop();
+ FileCaptureBlock* file_block = head;
+
+ if (reserved)
+ file_capture_stats.files_released_total++;
+ else
+ file_capture_stats.files_freed_total++;
+
+ while (file_block)
+ {
+ FileCaptureBlock* next_block = file_block->next;
+ if (reserved)
+ {
+ if (file_mempool->m_release(file_block) != FILE_MEM_SUCCESS)
+ file_capture_stats.file_buffers_release_errors++;
+ file_capture_stats.file_buffers_released_total++;
+ }
+ else
+ {
+ if (file_mempool->m_free(file_block) != FILE_MEM_SUCCESS)
+ file_capture_stats.file_buffers_free_errors++;
+ file_capture_stats.file_buffers_freed_total++;
+ }
+
+ file_block = next_block;
+ }
+
+ head = last = nullptr;
+
+ if (file_info)
+ delete file_info;
+}
+
+void FileCapture::init()
+{
+ FileConfig& file_config = snort_conf->file_config;
+ init_mempool(file_config.capture_memcap, file_config.capture_block_size);
+ file_storer = new std::thread(writer_thread);
+}
+
+/*
+ * Release all file capture memory etc,
+ * this must be called when snort exits
+ */
+void FileCapture::exit()
+{
+ running = false;
+ capture_cv.notify_one();
+
+ if (file_storer)
+ {
+ file_storer->join();
+ delete file_storer;
+ file_storer = nullptr;
+ }
+
+ if (file_mempool)
+ {
+ delete file_mempool;
+ file_mempool = nullptr;
+ }
}
/*
*
* Arguments:
* int64_t max_file_mem: memcap in megabytes
- * int64_t block_size: file block size (metadata size excluded)
- *
- * Returns: NONE
+ * int64_t block_len: file block size (metadata size excluded)
*/
void FileCapture::init_mempool(int64_t max_file_mem, int64_t block_len)
{
int64_t block_size = block_len + sizeof (FileCapture);
- /*Convert megabytes to bytes*/
- int64_t max_file_mem_in_bytes = max_file_mem * 1024 * 1024;
-
if (block_size <= 0)
return;
+ /*Convert megabytes to bytes*/
+ int64_t max_file_mem_in_bytes = max_file_mem * 1024 * 1024;
+
if (block_size & 7)
block_size += (8 - (block_size & 7));
int max_files = max_file_mem_in_bytes / block_size;
- file_mempool = (FileMemPool*)snort_calloc(sizeof(FileMemPool));
-
- if ( file_mempool_init(file_mempool, max_files, block_size) != 0 )
- {
- FatalError("File capture: Could not initialize file buffer mempool.\n");
- }
-}
-
-/*
- * Stop file capture, memory resource will be released if not reserved
- *
- * Returns: NONE
- */
-void FileCapture::stop()
-{
- /*free mempool*/
- if (reserved)
- return;
-
- file_capture_stats.files_freed_total++;
- FileCaptureBlock* fileblock = head;
- while (fileblock)
- {
- if (file_mempool_free(file_mempool, fileblock) != FILE_MEM_SUCCESS)
- file_capture_stats.file_buffers_free_errors++;
- fileblock = fileblock->next;
- file_capture_stats.file_buffers_freed_total++;
- }
-
- head = last = nullptr;
+ file_mempool = new FileMemPool(max_files, block_size);
}
-/*
- * Create file buffer in file mempool
- *
- * Args:
- * FileMemPool *file_mempool: file mempool
- * FileContext* context: file context
- *
- * Returns:
- * FileCapture *: memory block that starts with file capture information
- */
-inline FileCaptureBlock* FileCapture::create_file_buffer(FileMemPool* file_mempool)
+inline FileCaptureBlock* FileCapture::create_file_buffer()
{
FileCaptureBlock* fileBlock;
uint64_t num_files_queued;
- fileBlock = (FileCaptureBlock*)file_mempool_alloc(file_mempool);
+ fileBlock = (FileCaptureBlock*)file_mempool->m_alloc();
if (fileBlock == nullptr)
{
fileBlock->length = 0;
fileBlock->next = nullptr; /*Only one block initially*/
- num_files_queued = file_mempool_allocated(file_mempool);
+ num_files_queued = file_mempool->allocated();
if (file_capture_stats.file_buffers_used_max < num_files_queued)
file_capture_stats.file_buffers_used_max = num_files_queued;
* Save file to the buffer
* If file needs to be extracted, buffer will be reserved
* If file buffer isn't sufficient, need to add another buffer.
- *
- * Returns:
- * 0: successful or file capture is disabled
- * 1: fail to capture the file
*/
-inline FileCaptureState FileCapture::save_to_file_buffer(FileMemPool* file_mempool,
- const uint8_t* file_data, int data_size, int64_t max_size)
+inline FileCaptureState FileCapture::save_to_file_buffer(const uint8_t* file_data,
+ int data_size, int64_t max_size)
{
FileCaptureBlock* lastBlock = last;
int64_t available_bytes;
FileConfig& file_config = snort_conf->file_config;
- if ( data_size + (int64_t)file_size > max_size)
+ if ( data_size + (int64_t)capture_size > max_size)
{
FILE_DEBUG_MSGS("Exceeding max file capture size!\n");
file_capture_stats.file_size_max++;
while (1)
{
/*get another block*/
- new_block = (FileCaptureBlock*)create_file_buffer(file_mempool);
+ new_block = (FileCaptureBlock*)create_file_buffer();
if (new_block == nullptr)
{
lastBlock->length += data_size;
}
- file_size += data_size;
+ capture_size += data_size;
return FILE_CAPTURE_SUCCESS;
}
*/
if (!head)
{
- head = last = create_file_buffer(file_mempool);
+ head = last = create_file_buffer();
if (!head)
{
file_capture_stats.files_buffered_total++;
}
- return (save_to_file_buffer(file_mempool, file_data, data_size,
- file_config.capture_max_size));
+ return (save_to_file_buffer(file_data, data_size, file_config.capture_max_size));
}
return FILE_CAPTURE_SUCCESS;
}
-/*Helper function for error*/
-static inline FileCaptureState ERROR_capture(FileCaptureState state)
-{
- file_capture_stats.file_reserve_failures++;
- return state;
-}
-
// Preserve the file in memory until it is released
-FileCaptureState FileCapture::reserve_file(FileContext* context)
+FileCaptureState FileCapture::reserve_file(const FileInfo* file)
{
uint64_t fileSize;
FileConfig& file_config = snort_conf->file_config;
- if (!context || !context->is_file_capture_enabled())
- {
- return ERROR_capture(FILE_CAPTURE_FAIL);
- }
-
if (capture_state != FILE_CAPTURE_SUCCESS)
{
- return ERROR_capture(capture_state);
+ return error_capture(capture_state);
}
- FileCaptureBlock* fileInfo = head;
+ FileCaptureBlock* fileBlock = head;
/*
* Note: file size is updated at this point
*/
- fileSize = context->get_file_size();
+ fileSize = file->get_file_size();
if ( fileSize < (unsigned)file_config.capture_min_size)
{
file_capture_stats.file_size_min++;
- return ERROR_capture(FILE_CAPTURE_MIN);
+ return error_capture(FILE_CAPTURE_MIN);
}
if ( fileSize > (unsigned)file_config.capture_max_size)
{
file_capture_stats.file_size_max++;
- return ERROR_capture(FILE_CAPTURE_MAX);
+ return error_capture(FILE_CAPTURE_MAX);
}
/* Create a file buffer if it is not done yet,
* This is the case for small file
*/
- if (!fileInfo && context->is_file_capture_enabled())
+ if (!fileBlock)
{
- fileInfo = create_file_buffer(file_mempool);
+ fileBlock = create_file_buffer();
- if (!fileInfo)
+ if (!fileBlock)
{
file_capture_stats.file_memcap_failures_reserve++;
- return ERROR_capture(FILE_CAPTURE_MEMCAP);
+ return error_capture(FILE_CAPTURE_MEMCAP);
}
file_capture_stats.files_buffered_total++;
- head = last = fileInfo;
+ head = last = fileBlock;
}
- if (!fileInfo)
+ if (!fileBlock)
{
- return ERROR_capture(FILE_CAPTURE_MEMCAP);
+ return error_capture(FILE_CAPTURE_MEMCAP);
}
/*Copy the last piece of file to file buffer*/
- if (save_to_file_buffer(file_mempool, current_data,
+ if (save_to_file_buffer(current_data,
current_data_len, file_config.capture_max_size) )
{
- return ERROR_capture(capture_state);
+ return error_capture(capture_state);
}
file_capture_stats.files_captured_total++;
- reserved = true;
current_block = head;
- context->config_file_capture(false);
+ reserved = true;
+
+ file_info = new FileInfo(*file);
return FILE_CAPTURE_SUCCESS;
}
}
// Get the file size captured in the file buffer
-uint64_t FileCapture::capture_size() const
+uint64_t FileCapture::get_capture_size() const
{
- return file_size;
+ return capture_size;
}
/*
- * Release the file that is reserved in memory, this function might be
- * called in a different thread.
- *
- * Arguments:
- * void *data: the memory block that stores file and its metadata
- */
-void FileCapture::release_file()
-{
- reserved = false;
-
- file_capture_stats.files_released_total++;
- FileCaptureBlock* fileblock = head;
-
- while (fileblock)
- {
- if (file_mempool_release(file_mempool, fileblock) != FILE_MEM_SUCCESS)
- file_capture_stats.file_buffers_release_errors++;
- fileblock = fileblock->next;
- file_capture_stats.file_buffers_released_total++;
- }
-
- head = last = nullptr;
-}
-
-/*
- * writing file to the disk.
+ * writing file data to the disk.
*
* In the case of interrupt errors, the write is retried, but only for a
* finite number of times.
*/
-void FileCapture::write_file(uint8_t* buf, size_t buf_len, FILE* fh)
+void FileCapture::write_file_data(uint8_t* buf, size_t buf_len, FILE* fh)
{
int max_retries = 3;
size_t bytes_written = 0;
}
// Store files on local disk
-void FileCapture::store_file(FileContext* file)
+void FileCapture::store_file()
{
- uint8_t* sha = file->get_file_sig_sha256();
- if (!sha)
+ if (!file_info)
return;
- std::string file_name = file->sha_to_string(sha);
-
- std::string file_full_name;
- get_instance_file(file_full_name, file_name.c_str());
+ std::string& file_full_name = file_info->get_file_name();
/*Check whether the file exists*/
struct stat buffer;
// Get file from file buffer
if (!buff || !size )
{
- //file_inspect_stats.file_read_failures++;
return;
}
- write_file(buff, size, fh);
+ write_file_data(buff, size, fh);
}
while (file_mem);
fclose(fh);
}
-/*Log file capture mempool usage*/
-void FileCapture::print_mem_usage()
+// Queue files to be stored to disk
+void FileCapture::store_file_async()
{
- if (file_mempool)
- {
- LogCount("Max buffers can allocate", file_mempool->total);
- LogCount("Buffers in use", file_mempool_allocated(file_mempool));
- LogCount("Buffers in free list", file_mempool_freed(file_mempool));
- LogCount("Buffers in release list", file_mempool_released(file_mempool));
- }
+ // send data to the writer thread
+ if (!file_info)
+ return;
+
+ uint8_t* sha = file_info->get_file_sig_sha256();
+ if (!sha)
+ return;
+
+ std::string file_name = file_info->sha_to_string(sha);
+
+ std::string file_full_name;
+ get_instance_file(file_full_name, file_name.c_str());
+ file_info->set_file_name(file_full_name.c_str(), file_full_name.size());
+
+ std::lock_guard<std::mutex> lk(capture_mutex);
+ files_waiting.push(this);
+ capture_cv.notify_one();
}
-/*
- * Release all file capture memory etc,
- * this must be called when snort exits
- */
-void FileCapture::exit()
+/*Log file capture mempool usage*/
+void FileCapture::print_mem_usage()
{
- if (file_mempool_destroy(file_mempool) == 0)
+ if (file_mempool)
{
- snort_free(file_mempool);
- file_mempool = nullptr;
+ LogCount("Max buffers can allocate", file_mempool->total_objects());
+ LogCount("Buffers in use", file_mempool->allocated());
+ LogCount("Buffers in free list", file_mempool->freed());
+ LogCount("Buffers in release list", file_mempool->released());
}
}
// 3) Then file data can be read through file_capture_read()
// 4) Finally, fila data must be released from mempool file_capture_release()
+#include <condition_variable>
+#include <mutex>
+#include <queue>
+#include <thread>
+
#include "file_api.h"
#include "file_lib.h"
#include "file_mempool.h"
~FileCapture();
// this must be called during snort init
- static void init_mempool(int64_t max_file_mem, int64_t block_size);
+ static void init();
// Capture file data to local buffer
// This is the main function call to enable file capture
FileCaptureState process_buffer(const uint8_t* file_data, int data_size,
FilePosition pos);
- // Stop file capture, memory resource will be released if not reserved
- void stop();
-
// Preserve the file in memory until it is released
- FileCaptureState reserve_file(FileContext* context);
+ FileCaptureState reserve_file(const FileInfo*);
+
+ // Whether file is reserved for store or analysis
+ bool is_reserved() { return reserved; }
// Get the file that is reserved in memory, this should be called repeatedly
// until NULL is returned to get the full file
FileCaptureBlock* get_file_data(uint8_t** buff, int* size);
// Get the file size captured in the file buffer
- // Returns:
- // the size of file
- uint64_t capture_size() const;
+ // Returns: the size of file in bytes
+ uint64_t get_capture_size() const;
// Store files on local disk
- void store_file(FileContext *file);
+ void store_file();
- // Release the file that is reserved in memory, this function might be
- // called in a different thread.
- void release_file();
+ // Store file to disk asynchronously
+ void store_file_async();
- // Log file capture mempool usage
+ // Log file capture mempoofile_contentl usage
static void print_mem_usage();
// Exit file capture, release all file capture memory etc,
private:
- inline FileCaptureBlock* create_file_buffer(FileMemPool* file_mempool);
- inline FileCaptureState save_to_file_buffer(FileMemPool* file_mempool,
- const uint8_t* file_data, int data_size, int64_t max_size);
- void write_file(uint8_t *buf, size_t buf_len, FILE *fh);
+ static void init_mempool(int64_t max_file_mem, int64_t block_size);
+ static void writer_thread();
+ inline FileCaptureBlock* create_file_buffer();
+ inline FileCaptureState save_to_file_buffer(const uint8_t* file_data, int data_size,
+ int64_t max_size);
+ void write_file_data(uint8_t* buf, size_t buf_len, FILE* fh);
+
+ static FileMemPool* file_mempool;
+ static std::mutex capture_mutex;
+ static std::condition_variable capture_cv;
+ static std::thread* file_storer;
+ static std::queue<FileCapture*> files_waiting;
+ static bool running;
bool reserved;
- uint64_t file_size; /*file_size*/
+ uint64_t capture_size;
FileCaptureBlock* last; /* last block of file data */
FileCaptureBlock* head; /* first block of file data */
FileCaptureBlock* current_block = nullptr; /* current block of file data */
- const uint8_t *current_data; /*current file data*/
+ const uint8_t* current_data; /*current file data*/
uint32_t current_data_len;
FileCaptureState capture_state;
+ FileInfo* file_info = nullptr;
};
-typedef struct _File_Capture_Stats
+struct File_Capture_Stats
{
uint64_t files_buffered_total;
uint64_t files_released_total;
uint64_t file_buffers_released_total;
uint64_t file_buffers_free_errors;
uint64_t file_buffers_release_errors;
-} File_Capture_Stats;
+};
extern File_Capture_Stats file_capture_stats;
+/*Helper function for error*/
+static inline FileCaptureState error_capture(FileCaptureState state)
+{
+ file_capture_stats.file_reserve_failures++;
+ return state;
+}
+
#endif
delete[] sha256;
}
-FileInfo& FileInfo::operator=(const FileInfo& other)
+void FileInfo::copy(const FileInfo& other)
{
- // check for self-assignment
- if(&other == this)
- return *this;
-
if (other.sha256)
{
sha256 = new uint8_t[SHA256_HASH_SIZE];
- if (sha256)
- strncpy( (char *)sha256, (const char *)other.sha256, SHA256_HASH_SIZE);
+ memcpy( (char *)sha256, (const char *)other.sha256, SHA256_HASH_SIZE);
}
file_size = other.file_size;
file_id = other.file_id;
file_name = other.file_name;
verdict = other.verdict;
+}
+FileInfo::FileInfo(const FileInfo& other)
+{
+ copy(other);
+}
+
+FileInfo& FileInfo::operator=(const FileInfo& other)
+{
+ // check for self-assignment
+ if(&other == this)
+ return *this;
+
+ copy(other);
return *this;
}
file_size = size;
}
-uint64_t FileInfo::get_file_size()
+uint64_t FileInfo::get_file_size() const
{
return file_size;
}
-uint32_t FileInfo::get_file_type()
+uint32_t FileInfo::get_file_type() const
{
return file_type_id;
}
file_id = id;
}
-size_t FileInfo::get_file_id()
+size_t FileInfo::get_file_id() const
{
return file_id;
}
direction = dir;
}
-FileDirection FileInfo::get_file_direction()
+FileDirection FileInfo::get_file_direction() const
{
return (direction);
}
sha256 = signature;
}
-uint8_t* FileInfo::get_file_sig_sha256()
+uint8_t* FileInfo::get_file_sig_sha256() const
{
return (sha256);
}
}
}
-FileCapture *FileContext::get_file_capture()
-{
- return file_capture;
-}
-
FileCaptureState FileContext::process_file_capture(const uint8_t* file_data,
int data_size, FilePosition position)
{
return file_state.capture_state;
}
+FileCaptureState FileContext::reserve_file(FileCapture*& dest)
+{
+ if (!file_capture || !is_file_capture_enabled())
+ return error_capture(FILE_CAPTURE_FAIL);
+
+ FileCaptureState state = file_capture->reserve_file(this);
+ config_file_capture(false);
+ dest = file_capture;
+ file_capture = nullptr;
+ return state;
+}
+
void FileContext::stop_file_capture()
{
if (file_capture)
{
delete file_capture;
- file_capture = NULL;
+ file_capture = nullptr;
}
file_capture_enabled = false;
{
public:
virtual ~FileInfo();
+ FileInfo(){};
+ FileInfo(const FileInfo& other);
FileInfo& operator=(const FileInfo& other);
- uint32_t get_file_type();
+ uint32_t get_file_type() const;
void set_file_name(const char* file_name, uint32_t name_size);
std::string& get_file_name();
void set_file_size(uint64_t size);
- uint64_t get_file_size();
+ uint64_t get_file_size() const;
void set_file_direction(FileDirection dir);
- FileDirection get_file_direction();
+ FileDirection get_file_direction() const;
void set_file_sig_sha256(uint8_t* signature);
- uint8_t* get_file_sig_sha256();
+ uint8_t* get_file_sig_sha256() const;
std::string sha_to_string(const uint8_t *sha256);
void set_file_id(size_t index);
- size_t get_file_id();
+ size_t get_file_id() const;
FileVerdict verdict = FILE_VERDICT_UNKNOWN;
protected:
uint32_t file_type_id = SNORT_FILE_TYPE_CONTINUE;
uint8_t* sha256 = nullptr;
size_t file_id = 0;
+
+private:
+ void copy(const FileInfo& other);
};
class SO_PUBLIC FileContext: public FileInfo
void stop_file_capture();
FileCaptureState process_file_capture(const uint8_t* file_data, int data_size, FilePosition pos);
+ // Preserve the file in memory until it is released
+ // The file reserved will be returned and it will be detached from file context/session
+ FileCaptureState reserve_file(FileCapture* &dest);
+
// Configuration functions
void config_file_type(bool enabled);
bool is_file_type_enabled();
//File properties
uint64_t get_processed_bytes();
- FileCapture *get_file_capture();
-
void set_file_config(FileConfig* file_config);
FileConfig* get_file_config();
#include <assert.h>
#include "file_mempool.h"
-#include "main/snort_debug.h"
#include "utils/util.h"
/*This magic is used for double free detection*/
typedef uint64_t MagicType;
#ifdef DEBUG_MSGS
-static inline void file_mempool_verify(FileMemPool* mempool)
+void FileMemPool::verify()
{
uint64_t free_size;
uint64_t release_size;
- free_size = cbuffer_used(mempool->free_list);
- release_size = cbuffer_used(mempool->released_list);
+ free_size = cbuffer_used(free_list);
+ release_size = cbuffer_used(released_list);
- if (free_size > cbuffer_size(mempool->free_list))
+ if (free_size > cbuffer_size(free_list))
{
ErrorMessage("%s(%d) file_mempool: failed to verify free list!\n",
__FILE__, __LINE__);
}
- if (release_size > cbuffer_size(mempool->released_list))
+ if (release_size > cbuffer_size(released_list))
{
ErrorMessage("%s(%d) file_mempool: failed to verify release list!\n",
__FILE__, __LINE__);
/* The free mempool and size of release mempool should be smaller than
* or equal to the size of mempool
*/
- if (free_size + release_size > mempool->total)
+ if (free_size + release_size > total)
{
ErrorMessage("%s(%d) file_mempool: failed to verify mempool size!\n",
__FILE__, __LINE__);
}
}
+
#endif
-static inline void file_mempool_free_pools(FileMemPool* mempool)
+void FileMemPool::free_pools()
{
- if (mempool == NULL)
- return;
-
- if (mempool->datapool != NULL)
+ if (datapool != nullptr)
{
- snort_free(mempool->datapool);
- mempool->datapool = NULL;
+ snort_free(datapool);
+ datapool = nullptr;
}
- cbuffer_free(mempool->free_list);
- cbuffer_free(mempool->released_list);
+ cbuffer_free(free_list);
+ cbuffer_free(released_list);
}
-/* Function: int file_mempool_init(FileMemPool *FileMemPool,
- * PoolCount num_objects, size_t obj_size)
- *
+/*
* Purpose: initialize a FileMemPool object and allocate memory for it
* Args:
- * FileMemPool - pointer to a FileMemPool struct
* num_objects - number of items in this pool
* obj_size - size of the items
- *
- * Returns:
- * FILE_MEM_SUCCESS
- * FILE_MEM_FAIL
*/
-int file_mempool_init(FileMemPool* mempool, uint64_t num_objects, size_t obj_size)
+FileMemPool::FileMemPool(uint64_t num_objects, size_t o_size)
{
unsigned int i;
- if ((mempool == NULL) || (num_objects < 1) || (obj_size < 1))
- return FILE_MEM_FAIL;
+ if ((num_objects < 1) || (o_size < 1))
+ return;
- mempool->obj_size = obj_size;
+ obj_size = o_size;
// this is the basis pool that represents all the *data pointers in the list
- mempool->datapool = (void**)snort_calloc(num_objects, obj_size);
+ datapool = (void**)snort_calloc(num_objects, obj_size);
/* sets up the memory list */
- mempool->free_list = cbuffer_init(num_objects);
- if (!mempool->free_list)
+ free_list = cbuffer_init(num_objects);
+ if (!free_list)
{
- ErrorMessage("%s(%d) file_mempool_init(): Failed to init free list\n",
+ ErrorMessage("%s(%d) file_mempool: Failed to init free list\n",
__FILE__, __LINE__);
- file_mempool_free_pools(mempool);
- return FILE_MEM_FAIL;
+ free_pools();
+ return;
}
- mempool->released_list = cbuffer_init(num_objects);
- if (!mempool->released_list)
+ released_list = cbuffer_init(num_objects);
+ if (!released_list)
{
- ErrorMessage("%s(%d) file_mempool_init(): "
+ ErrorMessage("%s(%d) file_mempool: "
"Failed to init release list\n", __FILE__, __LINE__);
- file_mempool_free_pools(mempool);
- return FILE_MEM_FAIL;
+ free_pools();
+ return;
}
for (i=0; i<num_objects; i++)
{
- void* data = ((char*)mempool->datapool) + (i * mempool->obj_size);
+ void* data = ((char*)datapool) + (i * obj_size);
- if (cbuffer_write(mempool->free_list, data))
+ if (cbuffer_write(free_list, data))
{
- ErrorMessage("%s(%d) file_mempool_init(): "
+ ErrorMessage("%s(%d) file_mempool: "
"Failed to add to free list\n",
__FILE__, __LINE__);
- file_mempool_free_pools(mempool);
- return FILE_MEM_FAIL;
+ free_pools();
+ return;
}
*(MagicType*)data = FREE_MAGIC;
- mempool->total++;
+ total++;
}
-
- return FILE_MEM_SUCCESS;
}
/*
* Destroy a set of FileMemPool objects
*
- * Args:
- * FileMemPool: pointer to a FileMemPool struct
- *
- * Return:
- * FILE_MEM_SUCCESS
- * FILE_MEM_FAIL
*/
-int file_mempool_destroy(FileMemPool* mempool)
+FileMemPool::~FileMemPool()
{
- if (mempool == NULL)
- return FILE_MEM_FAIL;
-
- file_mempool_free_pools(mempool);
-
- return FILE_MEM_SUCCESS;
+ free_pools();
}
/*
* Returns: a pointer to the FileMemPool object on success, NULL on failure
*/
-void* file_mempool_alloc(FileMemPool* mempool)
+void* FileMemPool::m_alloc()
{
void* b = NULL;
- if (mempool == NULL)
- {
- return NULL;
- }
+ std::lock_guard<std::mutex> lock(pool_mutex);
- if (cbuffer_read(mempool->free_list, &b))
+ if (cbuffer_read(free_list, &b))
{
- if (cbuffer_read(mempool->released_list, &b))
+ if (cbuffer_read(released_list, &b))
{
return NULL;
}
__FILE__, __LINE__);
}
- DEBUG_WRAP(file_mempool_verify(mempool); );
+ DEBUG_WRAP(verify(); );
return b;
}
/*
* Free a new object from the buffer
* We use circular buffer to synchronize one reader and one writer
- *
- * Args:
- * FileMemPool: pointer to a circular buffer struct
- * void *obj : memory object
- *
- * Return:
- * FILE_MEM_SUCCESS
- * FILE_MEM_FAIL
*/
-static inline int _file__mempool_remove(CircularBuffer* cb, void* obj)
+int FileMemPool::remove(CircularBuffer* cb, void* obj)
{
- if (obj == NULL)
+ if (obj == nullptr)
return FILE_MEM_FAIL;
if (cbuffer_write(cb, obj))
return FILE_MEM_SUCCESS;
}
-/*
- * Free a new object from the FileMemPool
- *
- * Args:
- * FileMemPool: pointer to a FileMemPool struct
- * void *obj : memory object
- *
- * Return:
- * FILE_MEM_SUCCESS
- * FILE_MEM_FAIL
- */
-
-int file_mempool_free(FileMemPool* mempool, void* obj)
+int FileMemPool::m_free(void* obj)
{
- int ret;
+ std::lock_guard<std::mutex> lock(pool_mutex);
- assert(mempool);
+ int ret = remove(free_list, obj);
- ret = _file__mempool_remove(mempool->free_list, obj);
-
- DEBUG_WRAP(file_mempool_verify(mempool); );
+ DEBUG_WRAP(verify(); );
return ret;
}
* This can be called by a different thread calling
* file_mempool_alloc()
* *
- * Args:
- * FileMemPool: pointer to a FileMemPool struct
- * void *obj : memory object
- *
- * Return:
- * FILE_MEM_SUCCESS
- * FILE_MEM_FAIL
*/
-int file_mempool_release(FileMemPool* mempool, void* obj)
+int FileMemPool::m_release(void* obj)
{
- int ret;
-
- if (mempool == NULL)
- return FILE_MEM_FAIL;
+ std::lock_guard<std::mutex> lock(pool_mutex);
/*A writer that might from different thread*/
- ret = _file__mempool_remove(mempool->released_list, obj);
+ int ret = remove(released_list, obj);
- DEBUG_WRAP(file_mempool_verify(mempool); );
+ DEBUG_WRAP(verify(); );
return ret;
}
/* Returns number of elements allocated in current buffer*/
-uint64_t file_mempool_allocated(FileMemPool* mempool)
+uint64_t FileMemPool::allocated()
{
- uint64_t total_freed =
- file_mempool_released(mempool) + file_mempool_freed(mempool);
- return (mempool->total - total_freed);
+ uint64_t total_freed = released() + freed();
+ return (total - total_freed);
}
/* Returns number of elements freed in current buffer*/
-uint64_t file_mempool_freed(FileMemPool* mempool)
+uint64_t FileMemPool::freed()
{
- return (cbuffer_used(mempool->free_list));
+ return (cbuffer_used(free_list));
}
/* Returns number of elements released in current buffer*/
-uint64_t file_mempool_released(FileMemPool* mempool)
+uint64_t FileMemPool::released()
{
- return (cbuffer_used(mempool->released_list));
+ return (cbuffer_used(released_list));
}
#ifndef FILE_MEMPOOL_H
#define FILE_MEMPOOL_H
- // This mempool implementation has very efficient alloc/free operations.
- // In addition, it provides thread-safe alloc/free for one allocation/free
- // thread and one release thread.
- // One more bonus: Double free detection is also added into this library
- // This is a thread safe version of memory pool for one writer and one reader thread
+// This mempool implementation has very efficient alloc/free operations.
+// In addition, it provides thread-safe alloc/free for one allocation/free
+// thread and one release thread.
+// One more bonus: Double free detection is also added into this library
+// This is a thread safe version of memory pool for one writer and one reader thread
+
+#include <mutex>
#include "main/snort_types.h"
+#include "main/snort_debug.h"
#include "circular_buffer.h"
#define FILE_MEM_SUCCESS 0 // FIXIT-L use bool
#define FILE_MEM_FAIL -1
-typedef struct _FileMemPool
+class FileMemPool
{
- void** datapool; /* memory buffer */
+public:
- uint64_t total;
+ FileMemPool(uint64_t num_objects, size_t obj_size);
+ ~FileMemPool();
- CircularBuffer* free_list;
- CircularBuffer* released_list;
- size_t obj_size;
-} FileMemPool;
+ // Allocate a new object from the FileMemPool
+ // Note: Memory block will not be zeroed for performance
+ // Returns: a pointer to the FileMemPool object on success, nullptr on failure
+ void* m_alloc();
-// This must be called before file mempool is used
-// Return: FILE_MEM_SUCCESS or FILE_MEM_FAIL
-int file_mempool_init(FileMemPool* mempool, uint64_t num_objects, size_t obj_size);
+ // This must be called by the same thread calling file_mempool_alloc()
+ // Return: FILE_MEM_SUCCESS or FILE_MEM_FAIL
+ int m_free(void* obj);
-// This must be called during snort exits
-// Return: FILE_MEM_SUCCESS or FILE_MEM_FAIL
-int file_mempool_destroy(FileMemPool* mempool);
+ // This can be called by a different thread calling file_mempool_alloc()
+ // Return: FILE_MEM_SUCCESS or FILE_MEM_FAIL
+ int m_release(void* obj);
-// Allocate a new object from the FileMemPool
-// Note: Memory block will not be zeroed for performance
-// Returns: a pointer to the FileMemPool object on success, NULL on failure
-void* file_mempool_alloc(FileMemPool* mempool);
+ //Returns number of elements allocated
+ uint64_t allocated();
-// This must be called by the same thread calling file_mempool_alloc()
-// Return: FILE_MEM_SUCCESS or FILE_MEM_FAIL
-int file_mempool_free(FileMemPool* mempool, void* obj);
+ // Returns number of elements freed in current buffer
+ uint64_t freed();
-// This can be called by a different thread calling file_mempool_alloc()
-// Return: FILE_MEM_SUCCESS or FILE_MEM_FAIL
-int file_mempool_release(FileMemPool* mempool, void* obj);
+ // Returns number of elements released in current buffer
+ uint64_t released();
-//Returns number of elements allocated in current buffer
-uint64_t file_mempool_allocated(FileMemPool* mempool);
+ // Returns total number of elements in current buffer
+ uint64_t total_objects() { return total; }
-// Returns number of elements freed in current buffer
-uint64_t file_mempool_freed(FileMemPool* mempool);
+private:
-// Returns number of elements released in current buffer
-uint64_t file_mempool_released(FileMemPool* mempool);
+ void free_pools();
+ int remove(CircularBuffer* cb, void* obj);
+#ifdef DEBUG_MSGS
+ void verify();
+#endif
+
+ void** datapool; /* memory buffer */
+ uint64_t total;
+ CircularBuffer* free_list;
+ CircularBuffer* released_list;
+ size_t obj_size;
+ std::mutex pool_mutex;
+};
#endif
if (rule.use.capture_enabled)
{
- FileCapture *capture = file->get_file_capture();
- if (capture)
+ FileCapture *captured = nullptr;
+
+ if (file->reserve_file(captured) == FILE_CAPTURE_SUCCESS)
{
- if (capture->reserve_file(file) == FILE_CAPTURE_SUCCESS)
- {
- capture->store_file(file);
- capture->release_file();
- }
+ captured->store_file_async();
}
}
void FileService::post_init()
{
- FileConfig& file_config = snort_conf->file_config;
-
FilePolicy& fp = get_inspect();
fp.load();
- if ( file_capture_enabled)
- FileCapture::init_mempool(file_config.capture_memcap,
- file_config.capture_block_size);
+ if (file_capture_enabled)
+ FileCapture::init();
}
void FileService::close()
projects / thirdparty / snort3.git / commitdiff
