From 3ab9120821e2b5cbc5925470bcfa5bcfb53f246b Mon Sep 17 00:00:00 2001 From: Dana Helwig Date: Thu, 27 Apr 2017 11:17:16 -0600 Subject: [PATCH] source-pcap-file: Pcap Directory Mode (Feature #2222) https://redmine.openinfosecfoundation.org/issues/2222 Pcap file mode that when passed a directory will process all files in that directory. If --pcap-file-continuous or continuous option is passed in json, the directory will be monitored until the directory is moved/deleted, suricata is interrupted, or the pcap-interrupt command is used with unix command socket. Existing file implementation and new directory implementation has moved from source-pcap-file into pcap-file-helper and pcap-directory-helper. Engine state will not reset between files. Also satisfies: * https://redmine.openinfosecfoundation.org/issues/2299 * https://redmine.openinfosecfoundation.org/issues/724 * https://redmine.openinfosecfoundation.org/issues/1476 Co-Authors: Dana Helwig and Danny Browning --- doc/userguide/partials/options.rst | 10 +- doc/userguide/unix-socket.rst | 30 +- scripts/suricatasc/src/suricatasc.py | 20 +- src/Makefile.am | 2 + src/runmode-unix-socket.c | 259 +++++++++--- src/runmode-unix-socket.h | 2 +- src/source-pcap-file-directory-helper.c | 535 ++++++++++++++++++++++++ src/source-pcap-file-directory-helper.h | 83 ++++ src/source-pcap-file-helper.c | 235 +++++++++++ src/source-pcap-file-helper.h | 109 +++++ src/source-pcap-file.c | 439 +++++++++---------- src/suricata-common.h | 4 + src/suricata.c | 10 + 13 files changed, 1436 insertions(+), 302 deletions(-) create mode 100644 src/source-pcap-file-directory-helper.c create mode 100644 src/source-pcap-file-directory-helper.h create mode 100644 src/source-pcap-file-helper.c create mode 100644 src/source-pcap-file-helper.h diff --git a/doc/userguide/partials/options.rst b/doc/userguide/partials/options.rst index 70e2ae985d..d150fe5848 100644 --- a/doc/userguide/partials/options.rst +++ b/doc/userguide/partials/options.rst @@ -25,7 +25,15 @@ .. option:: -r - Run in pcap offline mode reading files from pcap file. + Run in pcap offline mode reading files from pcap file. If specifies + a directory, all files in that directory will be processed in order of + modified time maintaining flow state between files. + +.. option:: --pcap-file-continuous + + Used with the -r option to indicate that the mode should stay alive until + interrupted. This is useful with directories to add new files and not reset + flow state between files. .. option:: -i diff --git a/doc/userguide/unix-socket.rst b/doc/userguide/unix-socket.rst index 5704e7323d..36f9629350 100644 --- a/doc/userguide/unix-socket.rst +++ b/doc/userguide/unix-socket.rst @@ -48,7 +48,11 @@ example to write custom scripts: Commands in standard running mode --------------------------------- +You may need to install suricatasc if you have not done so, running the following command from scripts/suricatasc +:: + + sudo python setup.py install The set of existing commands is the following: @@ -152,14 +156,20 @@ treatment: Success: Successfully added file to list >>> pcap-file /home/benches/file2.pcap /tmp/file2 Success: Successfully added file to list + >>> pcap-file-continuous /home/pcaps /tmp/dirout + Success: Successfully added file to list You can add multiple files without waiting the result: they will be sequentially processed and the generated log/alert files will be put into the directory specified as second arguments of the pcap-file command. You need to provide absolute path to the files and directory -as suricata don’t know from where the script has been run. +as suricata doesn’t know from where the script has been run. If you pass +a directory instead of a file, all files in the directory will be processed. If +using ``pcap-file-continuous`` and passing in a directory, the directory will +be monitored for new files being added until you use ``pcap-interrupt`` or +delete/move the directory. -To know how much files are waiting to get processed, you can do: +To know how many files are waiting to get processed, you can do: :: @@ -181,6 +191,22 @@ To get current processed file: Success: "/tmp/test.pcap" +When passing in a directory, you can see last processed time (modified time of last file) in milliseconds since epoch: + +:: + + >>> pcap-last-processed + Success: + 1509138964000 + +To interrupt directory processing which terminates the current state: + +:: + + >>> pcap-interrupt + Success: + "Interrupted" + Build your own client --------------------- diff --git a/scripts/suricatasc/src/suricatasc.py b/scripts/suricatasc/src/suricatasc.py index ae62cf205e..71a942dd33 100644 --- a/scripts/suricatasc/src/suricatasc.py +++ b/scripts/suricatasc/src/suricatasc.py @@ -80,7 +80,7 @@ class SuricataCompleter: class SuricataSC: def __init__(self, sck_path, verbose=False): - self.cmd_list=['shutdown','quit','pcap-file','pcap-file-number','pcap-file-list','iface-list','iface-stat','register-tenant','unregister-tenant','register-tenant-handler','unregister-tenant-handler', 'add-hostbit', 'remove-hostbit', 'list-hostbit'] + self.cmd_list=['shutdown','quit','pcap-file','pcap-file-continuous','pcap-file-number','pcap-file-list','pcap-last-processed','pcap-interrupt','iface-list','iface-stat','register-tenant','unregister-tenant','register-tenant-handler','unregister-tenant-handler', 'add-hostbit', 'remove-hostbit', 'list-hostbit'] self.sck_path = sck_path self.verbose = verbose @@ -192,6 +192,24 @@ class SuricataSC: arguments["output-dir"] = output if tenant != None: arguments["tenant"] = int(tenant) + elif "pcap-file-continuous " in command: + try: + parts = command.split(' '); + except: + raise SuricataCommandException("Arguments to command '%s' is missing" % (command)) + cmd, filename, output = parts[0], parts[1], parts[2] + tenant = None + if len(parts) > 3: + tenant = parts[3] + if cmd != "pcap-file": + raise SuricataCommandException("Invalid command '%s'" % (command)) + else: + arguments = {} + arguments["filename"] = filename + arguments["output-dir"] = output + arguments["continuous"] = True + if tenant != None: + arguments["tenant"] = int(tenant) elif "iface-stat" in command: try: [cmd, iface] = command.split(' ', 1) diff --git a/src/Makefile.am b/src/Makefile.am index a97e4f05c8..76726125ed 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -344,6 +344,8 @@ source-nfq.c source-nfq.h \ source-nflog.c source-nflog.h \ source-pcap.c source-pcap.h \ source-pcap-file.c source-pcap-file.h \ +source-pcap-file-directory-helper.c source-pcap-file-directory-helper.h \ +source-pcap-file-helper.c source-pcap-file-helper.h \ source-pfring.c source-pfring.h \ stream.c stream.h \ stream-tcp.c stream-tcp.h stream-tcp-private.h \ diff --git a/src/runmode-unix-socket.c b/src/runmode-unix-socket.c index 79eb787c8e..48bbecf625 100644 --- a/src/runmode-unix-socket.c +++ b/src/runmode-unix-socket.c @@ -51,13 +51,16 @@ typedef struct PcapFiles_ { char *filename; char *output_dir; int tenant_id; + time_t delay; + time_t poll_interval; + bool continuous; TAILQ_ENTRY(PcapFiles_) next; } PcapFiles; typedef struct PcapCommand_ { TAILQ_HEAD(, PcapFiles_) files; int running; - char *currentfile; + PcapFiles *current_file; } PcapCommand; const char *RunModeUnixSocketGetDefaultMode(void) @@ -68,8 +71,10 @@ const char *RunModeUnixSocketGetDefaultMode(void) #ifdef BUILD_UNIX_SOCKET static int RunModeUnixSocketMaster(void); -static int unix_manager_file_task_running = 0; -static int unix_manager_file_task_failed = 0; +static int unix_manager_pcap_task_running = 0; +static int unix_manager_pcap_task_failed = 0; +static int unix_manager_pcap_task_interrupted = 0; +static struct timespec unix_manager_pcap_last_processed; /** * \brief return list of files in the queue @@ -124,15 +129,33 @@ static TmEcode UnixSocketPcapCurrent(json_t *cmd, json_t* answer, void *data) { PcapCommand *this = (PcapCommand *) data; - if (this->currentfile) { - json_object_set_new(answer, "message", json_string(this->currentfile)); + if (this->current_file && this->current_file->filename) { + json_object_set_new(answer, "message", + json_string(this->current_file->filename)); } else { json_object_set_new(answer, "message", json_string("None")); } return TM_ECODE_OK; } +static TmEcode UnixSocketPcapLastProcessed(json_t *cmd, json_t *answer, void *data) +{ + uint64_t epoch_millis = unix_manager_pcap_last_processed.tv_sec * 1000l + + unix_manager_pcap_last_processed.tv_nsec / 100000l; + json_object_set_new(answer, "message", + json_integer(epoch_millis)); + + return TM_ECODE_OK; +} + +static TmEcode UnixSocketPcapInterrupt(json_t *cmd, json_t *answer, void *data) +{ + unix_manager_pcap_task_interrupted = 1; + json_object_set_new(answer, "message", json_string("Interrupted")); + + return TM_ECODE_OK; +} static void PcapFilesFree(PcapFiles *cfile) { @@ -151,11 +174,22 @@ static void PcapFilesFree(PcapFiles *cfile) * \param this a UnixCommand:: structure * \param filename absolute filename * \param output_dir absolute name of directory where log will be put + * \param tenant_id Id of tenant associated with this file + * \param continuous If file should be run in continuous mode + * \param delay Delay required for file modified time before being processed + * \param poll_interval How frequently directory mode polls for new files * * \retval 0 in case of error, 1 in case of success */ -static TmEcode UnixListAddFile(PcapCommand *this, - const char *filename, const char *output_dir, int tenant_id) +static TmEcode UnixListAddFile( + PcapCommand *this, + const char *filename, + const char *output_dir, + int tenant_id, + bool continuous, + time_t delay, + time_t poll_interval +) { PcapFiles *cfile = NULL; if (filename == NULL || this == NULL) @@ -185,6 +219,9 @@ static TmEcode UnixListAddFile(PcapCommand *this, } cfile->tenant_id = tenant_id; + cfile->continuous = continuous; + cfile->delay = delay; + cfile->poll_interval = poll_interval; TAILQ_INSERT_TAIL(&this->files, cfile, next); return TM_ECODE_OK; @@ -200,10 +237,12 @@ static TmEcode UnixListAddFile(PcapCommand *this, static TmEcode UnixSocketAddPcapFile(json_t *cmd, json_t* answer, void *data) { PcapCommand *this = (PcapCommand *) data; - int ret; const char *filename; const char *output_dir; int tenant_id = 0; + bool continuous = false; + time_t delay = 30; + time_t poll_interval = 5; #ifdef OS_WIN32 struct _stat st; #else @@ -211,61 +250,100 @@ static TmEcode UnixSocketAddPcapFile(json_t *cmd, json_t* answer, void *data) #endif /* OS_WIN32 */ json_t *jarg = json_object_get(cmd, "filename"); - if(!json_is_string(jarg)) { - SCLogInfo("error: command is not a string"); - json_object_set_new(answer, "message", json_string("command is not a string")); + if (!json_is_string(jarg)) { + SCLogError(SC_ERR_INVALID_ARGUMENT, "filename is not a string"); + json_object_set_new(answer, "message", + json_string("filename is not a string")); return TM_ECODE_FAILED; } filename = json_string_value(jarg); #ifdef OS_WIN32 - if(_stat(filename, &st) != 0) { + if (_stat(filename, &st) != 0) { #else - if(stat(filename, &st) != 0) { + if (stat(filename, &st) != 0) { #endif /* OS_WIN32 */ - json_object_set_new(answer, "message", json_string("File does not exist")); + json_object_set_new(answer, "message", + json_string("filename does not exist")); return TM_ECODE_FAILED; } json_t *oarg = json_object_get(cmd, "output-dir"); if (oarg != NULL) { - if(!json_is_string(oarg)) { - SCLogInfo("error: output dir is not a string"); - json_object_set_new(answer, "message", json_string("output dir is not a string")); + if (!json_is_string(oarg)) { + SCLogError(SC_ERR_INVALID_ARGUMENT, "output-dir is not a string"); + + json_object_set_new(answer, "message", + json_string("output-dir is not a string")); return TM_ECODE_FAILED; } output_dir = json_string_value(oarg); } else { - SCLogInfo("error: can't get output-dir"); - json_object_set_new(answer, "message", json_string("output dir param is mandatory")); + SCLogError(SC_ERR_INVALID_ARGUMENT, "can't get output-dir"); + + json_object_set_new(answer, "message", + json_string("output-dir param is mandatory")); return TM_ECODE_FAILED; } #ifdef OS_WIN32 - if(_stat(output_dir, &st) != 0) { + if (_stat(output_dir, &st) != 0) { #else - if(stat(output_dir, &st) != 0) { + if (stat(output_dir, &st) != 0) { #endif /* OS_WIN32 */ - json_object_set_new(answer, "message", json_string("Output directory does not exist")); + json_object_set_new(answer, "message", + json_string("output-dir does not exist")); return TM_ECODE_FAILED; } json_t *targ = json_object_get(cmd, "tenant"); if (targ != NULL) { - if(!json_is_number(targ)) { - json_object_set_new(answer, "message", json_string("tenant is not a number")); + if (!json_is_integer(targ)) { + json_object_set_new(answer, "message", + json_string("tenant is not a number")); return TM_ECODE_FAILED; } tenant_id = json_number_value(targ); } - ret = UnixListAddFile(this, filename, output_dir, tenant_id); - switch(ret) { + json_t *cont_arg = json_object_get(cmd, "continuous"); + if (cont_arg != NULL) { + continuous = json_is_true(cont_arg); + } + + json_t *delay_arg = json_object_get(cmd, "delay"); + if (delay_arg != NULL) { + if (!json_is_integer(delay_arg)) { + SCLogError(SC_ERR_INVALID_ARGUMENT, "delay is not a integer"); + json_object_set_new(answer, "message", + json_string("delay is not a integer")); + return TM_ECODE_FAILED; + } + delay = json_integer_value(delay_arg); + } + + json_t *interval_arg = json_object_get(cmd, "poll-interval"); + if (interval_arg != NULL) { + if (!json_is_integer(interval_arg)) { + SCLogError(SC_ERR_INVALID_ARGUMENT, "poll-interval is not a integer"); + + json_object_set_new(answer, "message", + json_string("poll-interval is not a integer")); + return TM_ECODE_FAILED; + } + poll_interval = json_integer_value(interval_arg); + } + + switch (UnixListAddFile(this, filename, output_dir, tenant_id, continuous, + delay, poll_interval)) { case TM_ECODE_FAILED: - json_object_set_new(answer, "message", json_string("Unable to add file to list")); + case TM_ECODE_DONE: + json_object_set_new(answer, "message", + json_string("Unable to add file to list")); return TM_ECODE_FAILED; case TM_ECODE_OK: SCLogInfo("Added file '%s' to list", filename); - json_object_set_new(answer, "message", json_string("Successfully added file to list")); + json_object_set_new(answer, "message", + json_string("Successfully added file to list")); return TM_ECODE_OK; } return TM_ECODE_OK; @@ -287,21 +365,23 @@ static TmEcode UnixSocketAddPcapFile(json_t *cmd, json_t* answer, void *data) static TmEcode UnixSocketPcapFilesCheck(void *data) { PcapCommand *this = (PcapCommand *) data; - if (unix_manager_file_task_running == 1) { + if (unix_manager_pcap_task_running == 1) { return TM_ECODE_OK; } - if ((unix_manager_file_task_failed == 1) || (this->running == 1)) { - if (unix_manager_file_task_failed) { + if ((unix_manager_pcap_task_failed == 1) || (this->running == 1)) { + if (unix_manager_pcap_task_failed) { SCLogInfo("Preceeding task failed, cleaning the running mode"); } - unix_manager_file_task_failed = 0; + unix_manager_pcap_task_failed = 0; this->running = 0; - if (this->currentfile) { - SCFree(this->currentfile); - } - this->currentfile = NULL; + SCLogInfo("Resetting engine state"); PostRunDeinit(RUNMODE_PCAP_FILE, NULL /* no ts */); + + if (this->current_file) { + PcapFilesFree(this->current_file); + } + this->current_file = NULL; } if (TAILQ_EMPTY(&this->files)) { // nothing to do @@ -310,41 +390,73 @@ static TmEcode UnixSocketPcapFilesCheck(void *data) PcapFiles *cfile = TAILQ_FIRST(&this->files); TAILQ_REMOVE(&this->files, cfile, next); - SCLogInfo("Starting run for '%s'", cfile->filename); - unix_manager_file_task_running = 1; + + unix_manager_pcap_task_running = 1; this->running = 1; - if (ConfSet("pcap-file.file", cfile->filename) != 1) { - SCLogError(SC_ERR_INVALID_ARGUMENTS, - "Can not set working file to '%s'", cfile->filename); + + if (ConfSetFinal("pcap-file.file", cfile->filename) != 1) { + SCLogError(SC_ERR_INVALID_ARGUMENT, "Can not set working file to '%s'", + cfile->filename); PcapFilesFree(cfile); return TM_ECODE_FAILED; } - if (cfile->output_dir) { - if (ConfSet("default-log-dir", cfile->output_dir) != 1) { - SCLogError(SC_ERR_INVALID_ARGUMENTS, - "Can not set output dir to '%s'", cfile->output_dir); + + int set_res = 0; + if (cfile->continuous) { + set_res = ConfSetFinal("pcap-file.continuous", "true"); + } else { + set_res = ConfSetFinal("pcap-file.continuous", "false"); + } + if (set_res != 1) { + SCLogError(SC_ERR_INVALID_ARGUMENT, "Can not set continuous mode for pcap processing"); + PcapFilesFree(cfile); + return TM_ECODE_FAILED; + } + + if (cfile->delay > 0) { + char tstr[32]; + snprintf(tstr, sizeof(tstr), "%" PRIuMAX, (uintmax_t)cfile->delay); + if (ConfSetFinal("pcap-file.delay", tstr) != 1) { + SCLogError(SC_ERR_INVALID_ARGUMENT, "Can not set delay to '%s'", tstr); PcapFilesFree(cfile); return TM_ECODE_FAILED; } } + + if (cfile->poll_interval > 0) { + char tstr[32]; + snprintf(tstr, sizeof(tstr), "%" PRIuMAX, (uintmax_t)cfile->poll_interval); + if (ConfSetFinal("pcap-file.poll-interval", tstr) != 1) { + SCLogError(SC_ERR_INVALID_ARGUMENT, + "Can not set poll-interval to '%s'", tstr); + PcapFilesFree(cfile); + return TM_ECODE_FAILED; + } + } + if (cfile->tenant_id > 0) { char tstr[16]; snprintf(tstr, sizeof(tstr), "%d", cfile->tenant_id); - if (ConfSet("pcap-file.tenant-id", tstr) != 1) { - SCLogError(SC_ERR_INVALID_ARGUMENTS, - "Can not set working tenant-id to '%s'", tstr); + if (ConfSetFinal("pcap-file.tenant-id", tstr) != 1) { + SCLogError(SC_ERR_INVALID_ARGUMENT, + "Can not set working tenant-id to '%s'", tstr); PcapFilesFree(cfile); return TM_ECODE_FAILED; } } else { SCLogInfo("pcap-file.tenant-id not set"); } - this->currentfile = SCStrdup(cfile->filename); - if (unlikely(this->currentfile == NULL)) { - SCLogError(SC_ERR_MEM_ALLOC, "Failed file name allocation"); - return TM_ECODE_FAILED; + + if (cfile->output_dir) { + if (ConfSetFinal("default-log-dir", cfile->output_dir) != 1) { + SCLogError(SC_ERR_INVALID_ARGUMENT, + "Can not set output dir to '%s'", cfile->output_dir); + PcapFilesFree(cfile); + return TM_ECODE_FAILED; + } } - PcapFilesFree(cfile); + + this->current_file = cfile; PreRunInit(RUNMODE_PCAP_FILE); PreRunPostPrivsDropInit(RUNMODE_PCAP_FILE); @@ -353,6 +465,9 @@ static TmEcode UnixSocketPcapFilesCheck(void *data) /* Un-pause all the paused threads */ TmThreadWaitOnThreadInit(); TmThreadContinueThreads(); + + SCLogInfo("Starting run for '%s'", this->current_file->filename); + return TM_ECODE_OK; } #endif @@ -369,24 +484,34 @@ void RunModeUnixSocketRegister(void) RunModeUnixSocketMaster); default_mode = "autofp"; #endif - return; } -void UnixSocketPcapFile(TmEcode tm) +TmEcode UnixSocketPcapFile(TmEcode tm, struct timespec *last_processed) { #ifdef BUILD_UNIX_SOCKET + unix_manager_pcap_last_processed.tv_sec = last_processed->tv_sec; + unix_manager_pcap_last_processed.tv_nsec = last_processed->tv_nsec; switch (tm) { case TM_ECODE_DONE: - unix_manager_file_task_running = 0; - break; + SCLogInfo("Marking current task as done"); + unix_manager_pcap_task_running = 0; + return TM_ECODE_DONE; case TM_ECODE_FAILED: - unix_manager_file_task_running = 0; - unix_manager_file_task_failed = 1; - break; + SCLogInfo("Marking current task as failed"); + unix_manager_pcap_task_running = 0; + unix_manager_pcap_task_failed = 1; + //if we return failed, we can't stop the thread and suricata will fail to close + return TM_ECODE_DONE; case TM_ECODE_OK: - break; + if (unix_manager_pcap_task_interrupted == 1) { + SCLogInfo("Interrupting current run mode"); + return TM_ECODE_DONE; + } else { + return TM_ECODE_OK; + } } #endif + return TM_ECODE_FAILED; } #ifdef BUILD_UNIX_SOCKET @@ -591,9 +716,9 @@ TmEcode UnixSocketRegisterTenant(json_t *cmd, json_t* answer, void *data) } filename = json_string_value(jarg); #ifdef OS_WIN32 - if(_stat(filename, &st) != 0) { + if (_stat(filename, &st) != 0) { #else - if(stat(filename, &st) != 0) { + if (stat(filename, &st) != 0) { #endif /* OS_WIN32 */ json_object_set_new(answer, "message", json_string("file does not exist")); return TM_ECODE_FAILED; @@ -667,9 +792,9 @@ TmEcode UnixSocketReloadTenant(json_t *cmd, json_t* answer, void *data) } filename = json_string_value(jarg); #ifdef OS_WIN32 - if(_stat(filename, &st) != 0) { + if (_stat(filename, &st) != 0) { #else - if(stat(filename, &st) != 0) { + if (stat(filename, &st) != 0) { #endif /* OS_WIN32 */ json_object_set_new(answer, "message", json_string("file does not exist")); return TM_ECODE_FAILED; @@ -1029,11 +1154,13 @@ static int RunModeUnixSocketMaster(void) } TAILQ_INIT(&pcapcmd->files); pcapcmd->running = 0; - pcapcmd->currentfile = NULL; + pcapcmd->current_file = NULL; UnixManagerRegisterCommand("pcap-file", UnixSocketAddPcapFile, pcapcmd, UNIX_CMD_TAKE_ARGS); UnixManagerRegisterCommand("pcap-file-number", UnixSocketPcapFilesNumber, pcapcmd, 0); UnixManagerRegisterCommand("pcap-file-list", UnixSocketPcapFilesList, pcapcmd, 0); + UnixManagerRegisterCommand("pcap-last-processed", UnixSocketPcapLastProcessed, pcapcmd, 0); + UnixManagerRegisterCommand("pcap-interrupt", UnixSocketPcapInterrupt, pcapcmd, 0); UnixManagerRegisterCommand("pcap-current", UnixSocketPcapCurrent, pcapcmd, 0); UnixManagerRegisterBackgroundTask(UnixSocketPcapFilesCheck, pcapcmd); diff --git a/src/runmode-unix-socket.h b/src/runmode-unix-socket.h index 91de1c254f..352cc41373 100644 --- a/src/runmode-unix-socket.h +++ b/src/runmode-unix-socket.h @@ -28,7 +28,7 @@ const char *RunModeUnixSocketGetDefaultMode(void); int RunModeUnixSocketIsActive(void); -void UnixSocketPcapFile(TmEcode tm); +TmEcode UnixSocketPcapFile(TmEcode tm, struct timespec *last_processed); #ifdef BUILD_UNIX_SOCKET TmEcode UnixSocketRegisterTenantHandler(json_t *cmd, json_t* answer, void *data); diff --git a/src/source-pcap-file-directory-helper.c b/src/source-pcap-file-directory-helper.c new file mode 100644 index 0000000000..cd15a4786b --- /dev/null +++ b/src/source-pcap-file-directory-helper.c @@ -0,0 +1,535 @@ +/* Copyright (C) 2007-2016 Open Information Security Foundation + * + * You can copy, redistribute or modify this Program under the terms of + * the GNU General Public License version 2 as published by the Free + * Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * version 2 along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +/** + * \file + * + * \author Danny Browning + * + * Helper methods for directory based packet acquisition + */ + +#include "source-pcap-file-directory-helper.h" +#include "runmode-unix-socket.h" +#include "util-mem.h" + +static void GetTime(struct timespec *tm); +static void CopyTime(struct timespec *from, struct timespec *to); +static int CompareTimes(struct timespec *left, struct timespec *right); +static TmEcode PcapRunStatus(PcapFileDirectoryVars *); +static TmEcode PcapDirectoryFailure(PcapFileDirectoryVars *ptv); +static TmEcode PcapDirectoryDone(PcapFileDirectoryVars *ptv); +static int PcapDirectoryGetModifiedTime(char const * file, struct timespec * out); +static TmEcode PcapDirectoryInsertFile(PcapFileDirectoryVars *pv, + PendingFile *file_to_add, + struct timespec *file_to_add_modified_time); +static TmEcode PcapDirectoryPopulateBuffer(PcapFileDirectoryVars *ptv, + struct timespec * newer_than, + struct timespec * older_than); +static TmEcode PcapDirectoryDispatchForTimeRange(PcapFileDirectoryVars *pv, + struct timespec *newer_than, + struct timespec *older_than); + +void GetTime(struct timespec *tm) +{ + struct timeval now; + if(gettimeofday(&now, NULL) == 0) { + tm->tv_sec = now.tv_sec; + tm->tv_nsec = now.tv_usec * 1000; + } +} + +void CopyTime(struct timespec *from, struct timespec *to) +{ + to->tv_sec = from->tv_sec; + to->tv_nsec = from->tv_nsec; +} + +int CompareTimes(struct timespec *left, struct timespec *right) +{ + if (left->tv_sec < right->tv_sec) { + return -1; + } else if (left->tv_sec > right->tv_sec) { + return 1; + } else { + if (left->tv_nsec < right->tv_nsec) { + return -1; + } else if (left->tv_nsec > right->tv_nsec) { + return 1; + } else { + return 0; + } + } +} + +/** + * Pcap Folder Utilities + */ +TmEcode PcapRunStatus(PcapFileDirectoryVars *ptv) +{ + if (RunModeUnixSocketIsActive()) { + TmEcode done = UnixSocketPcapFile(TM_ECODE_OK, &ptv->shared->last_processed); + if ( (suricata_ctl_flags & SURICATA_STOP) || done != TM_ECODE_OK) { + SCReturnInt(TM_ECODE_DONE); + } + } else { + if (suricata_ctl_flags & SURICATA_STOP) { + SCReturnInt(TM_ECODE_DONE); + } + } + SCReturnInt(TM_ECODE_OK); +} + +void CleanupPendingFile(PendingFile *pending) { + if (pending != NULL) { + if (pending->filename != NULL) { + SCFree(pending->filename); + } + SCFree(pending); + } +} + +void CleanupPcapFileDirectoryVars(PcapFileDirectoryVars *ptv) +{ + if (ptv != NULL) { + if (ptv->current_file != NULL) { + CleanupPcapFileFileVars(ptv->current_file); + ptv->current_file = NULL; + } + if (ptv->directory != NULL) { + closedir(ptv->directory); + ptv->directory = NULL; + } + if (ptv->filename != NULL) { + SCFree(ptv->filename); + } + ptv->shared = NULL; + PendingFile *current_file = NULL; + while (!TAILQ_EMPTY(&ptv->directory_content)) { + current_file = TAILQ_FIRST(&ptv->directory_content); + TAILQ_REMOVE(&ptv->directory_content, current_file, next); + CleanupPendingFile(current_file); + } + SCFree(ptv); + } +} + +TmEcode PcapDirectoryFailure(PcapFileDirectoryVars *ptv) +{ + TmEcode status = TM_ECODE_FAILED; + + if (unlikely(ptv == NULL)) { + SCLogError(SC_ERR_INVALID_ARGUMENT, "Directory vars was null"); + SCReturnInt(TM_ECODE_FAILED); + } + if (unlikely(ptv->shared == NULL)) { + SCLogError(SC_ERR_INVALID_ARGUMENT, "Directory shared vars was null"); + SCReturnInt(TM_ECODE_FAILED); + } + + if (RunModeUnixSocketIsActive()) { + status = UnixSocketPcapFile(status, &ptv->shared->last_processed); + } + + SCReturnInt(status); +} + +TmEcode PcapDirectoryDone(PcapFileDirectoryVars *ptv) +{ + TmEcode status = TM_ECODE_DONE; + + if (unlikely(ptv == NULL)) { + SCLogError(SC_ERR_INVALID_ARGUMENT, "Directory vars was null"); + SCReturnInt(TM_ECODE_FAILED); + } + if (unlikely(ptv->shared == NULL)) { + SCLogError(SC_ERR_INVALID_ARGUMENT, "Directory shared vars was null"); + SCReturnInt(TM_ECODE_FAILED); + } + + if (RunModeUnixSocketIsActive()) { + status = UnixSocketPcapFile(status, &ptv->shared->last_processed); + } + + SCReturnInt(status); +} + +TmEcode PcapDetermineDirectoryOrFile(char *filename, DIR **directory) +{ + DIR *temp_dir = NULL; + TmEcode return_code = TM_ECODE_FAILED; + + temp_dir = opendir(filename); + + if (temp_dir == NULL) {//if null, our filename may just be a normal file + switch (errno) { + case EACCES: + SCLogError(SC_ERR_FOPEN, "%s: Permission denied", filename); + break; + + case EBADF: + SCLogError(SC_ERR_FOPEN, + "%s: Not a valid file descriptor opened for reading", + filename); + break; + + case EMFILE: + SCLogError(SC_ERR_FOPEN, + "%s: Per process open file descriptor limit reached", + filename); + break; + + case ENFILE: + SCLogError(SC_ERR_FOPEN, + "%s: System wide open file descriptor limit reached", + filename); + break; + + case ENOENT: + SCLogError(SC_ERR_FOPEN, + "%s: Does not exist, or name is an empty string", + filename); + break; + case ENOMEM: + SCLogError(SC_ERR_FOPEN, + "%s: Insufficient memory to complete the operation", + filename); + break; + + case ENOTDIR: //no error checking the directory, just is a plain file + SCLogInfo("%s: Plain file, not a directory", filename); + return_code = TM_ECODE_OK; + break; + + default: + SCLogError(SC_ERR_FOPEN, "%s: %" PRId32, filename, errno); + } + } else { + //no error, filename references a directory + *directory = temp_dir; + return_code = TM_ECODE_OK; + } + + return return_code; +} + +int PcapDirectoryGetModifiedTime(char const *file, struct timespec *out) +{ + struct stat buf; + int ret; + + if (file == NULL) + return -1; + + if ((ret = stat(file, &buf)) != 0) + return ret; + +#ifdef OS_DARWIN + *out = buf.st_mtimespec; +#else + *out = buf.st_mtim; +#endif + + return ret; +} + +TmEcode PcapDirectoryInsertFile(PcapFileDirectoryVars *pv, + PendingFile *file_to_add, + struct timespec *file_to_add_modified_time +) { + PendingFile *file_to_compare = NULL; + PendingFile *next_file_to_compare = NULL; + + if (unlikely(pv == NULL)) { + SCLogError(SC_ERR_INVALID_ARGUMENT, "No directory vars passed"); + SCReturnInt(TM_ECODE_FAILED); + } + + if (unlikely(file_to_add == NULL)) { + SCLogError(SC_ERR_INVALID_ARGUMENT, "File passed was null"); + SCReturnInt(TM_ECODE_FAILED); + } + + if (unlikely(file_to_add->filename == NULL)) { + SCLogError(SC_ERR_INVALID_ARGUMENT, "File was passed with null filename"); + SCReturnInt(TM_ECODE_FAILED); + } + + SCLogDebug("Inserting %s into directory buffer", file_to_add->filename); + + if (TAILQ_EMPTY(&pv->directory_content)) { + TAILQ_INSERT_TAIL(&pv->directory_content, file_to_add, next); + } else { + file_to_compare = TAILQ_FIRST(&pv->directory_content); + while(file_to_compare != NULL) { + struct timespec modified_time; + memset(&modified_time, 0, sizeof(struct timespec)); + + if (PcapDirectoryGetModifiedTime(file_to_compare->filename, + &modified_time) == TM_ECODE_FAILED) { + SCReturnInt(TM_ECODE_FAILED); + } + if (CompareTimes(file_to_add_modified_time, &modified_time) < 0) { + TAILQ_INSERT_BEFORE(file_to_compare, file_to_add, next); + file_to_compare = NULL; + } else { + next_file_to_compare = TAILQ_NEXT(file_to_compare, next); + if (next_file_to_compare == NULL) { + TAILQ_INSERT_AFTER(&pv->directory_content, file_to_compare, + file_to_add, next); + } + file_to_compare = next_file_to_compare; + } + } + } + + SCReturnInt(TM_ECODE_OK); +} + +TmEcode PcapDirectoryPopulateBuffer(PcapFileDirectoryVars *pv, + struct timespec *newer_than, + struct timespec *older_than +) { + if (unlikely(pv == NULL)) { + SCLogError(SC_ERR_INVALID_ARGUMENT, "No directory vars passed"); + SCReturnInt(TM_ECODE_FAILED); + } + if (unlikely(pv->filename == NULL)) { + SCLogError(SC_ERR_INVALID_ARGUMENT, "No directory filename was passed"); + SCReturnInt(TM_ECODE_FAILED); + } + struct dirent * dir = NULL; + PendingFile *file_to_add = NULL; + + while ((dir = readdir(pv->directory)) != NULL) { + if (dir->d_type != DT_REG) { + continue; + } + + char pathbuff[PATH_MAX] = {0}; + + int written = 0; + + written = snprintf(pathbuff, PATH_MAX, "%s/%s", pv->filename, dir->d_name); + + if (written <= 0 || written >= PATH_MAX) { + SCLogError(SC_ERR_SPRINTF, "Could not write path"); + + SCReturnInt(TM_ECODE_FAILED); + } else { + struct timespec temp_time; + memset(&temp_time, 0, sizeof(struct timespec)); + + if (PcapDirectoryGetModifiedTime(pathbuff, &temp_time) == 0) { + SCLogDebug("File %s time (%lu > %lu < %lu)", pathbuff, + newer_than->tv_sec, temp_time.tv_sec, older_than->tv_sec); + + // Skip files outside of our time range + if (CompareTimes(&temp_time, newer_than) < 0) { + SCLogDebug("Skipping old file %s", pathbuff); + continue; + } + else if (CompareTimes(&temp_time, older_than) >= 0) { + SCLogDebug("Skipping new file %s", pathbuff); + continue; + } + } else { + SCLogDebug("Unable to get modified time on %s, skipping", pathbuff); + continue; + } + + file_to_add = SCCalloc(1, sizeof(PendingFile)); + if (unlikely(file_to_add == NULL)) { + SCLogError(SC_ERR_MEM_ALLOC, "Failed to allocate pending file"); + + SCReturnInt(TM_ECODE_FAILED); + } + + file_to_add->filename = SCStrdup(pathbuff); + if (unlikely(file_to_add->filename == NULL)) { + SCLogError(SC_ERR_MEM_ALLOC, "Failed to copy filename"); + CleanupPendingFile(file_to_add); + + SCReturnInt(TM_ECODE_FAILED); + } + + SCLogDebug("Found \"%s\"", file_to_add->filename); + + if (PcapDirectoryInsertFile(pv, file_to_add, &temp_time) == TM_ECODE_FAILED) { + SCLogError(SC_ERR_INVALID_ARGUMENT, "Failed to add file"); + CleanupPendingFile(file_to_add); + + SCReturnInt(TM_ECODE_FAILED); + } + } + } + + SCReturnInt(TM_ECODE_OK); +} + + +TmEcode PcapDirectoryDispatchForTimeRange(PcapFileDirectoryVars *pv, + struct timespec *newer_than, + struct timespec *older_than) +{ + if (PcapDirectoryPopulateBuffer(pv, newer_than, older_than) == TM_ECODE_FAILED) { + SCLogError(SC_ERR_INVALID_ARGUMENT, "Failed to populate directory buffer"); + SCReturnInt(TM_ECODE_FAILED); + } + + TmEcode status = TM_ECODE_OK; + + if (TAILQ_EMPTY(&pv->directory_content)) { + SCLogInfo("Directory %s has no files to process", pv->filename); + GetTime(older_than); + older_than->tv_sec = older_than->tv_sec - pv->delay; + rewinddir(pv->directory); + status = TM_ECODE_OK; + } else { + PendingFile *current_file = NULL; + + while (status == TM_ECODE_OK && !TAILQ_EMPTY(&pv->directory_content)) { + current_file = TAILQ_FIRST(&pv->directory_content); + TAILQ_REMOVE(&pv->directory_content, current_file, next); + + if (unlikely(current_file == NULL)) { + SCLogWarning(SC_ERR_PCAP_DISPATCH, "Current file was null"); + } else if (unlikely(current_file->filename == NULL)) { + SCLogWarning(SC_ERR_PCAP_DISPATCH, "Current file filename was null"); + } else { + SCLogDebug("Processing file %s", current_file->filename); + + PcapFileFileVars *pftv = SCMalloc(sizeof(PcapFileFileVars)); + if (unlikely(pftv == NULL)) { + SCLogError(SC_ERR_MEM_ALLOC, "Failed to allocate PcapFileFileVars"); + SCReturnInt(TM_ECODE_FAILED); + } + memset(pftv, 0, sizeof(PcapFileFileVars)); + + pftv->filename = SCStrdup(current_file->filename); + if (unlikely(pftv->filename == NULL)) { + SCLogError(SC_ERR_MEM_ALLOC, "Failed to allocate filename"); + SCReturnInt(TM_ECODE_FAILED); + } + pftv->shared = pv->shared; + + if (InitPcapFile(pftv) == TM_ECODE_FAILED) { + SCLogWarning(SC_ERR_PCAP_DISPATCH, + "Failed to init pcap file %s, skipping", + current_file->filename); + CleanupPendingFile(current_file); + CleanupPcapFileFileVars(pftv); + status = TM_ECODE_OK; + } else { + pv->current_file = pftv; + + status = PcapFileDispatch(pftv); + + CleanupPcapFileFileVars(pftv); + + if (status == TM_ECODE_FAILED) { + CleanupPendingFile(current_file); + SCReturnInt(status); + } + + struct timespec temp_time; + memset(&temp_time, 0, sizeof(struct timespec)); + + if (PcapDirectoryGetModifiedTime(current_file->filename, + &temp_time) != 0) { + CopyTime(&temp_time, newer_than); + } + SCLogDebug("Processed file %s, processed up to %ld", + current_file->filename, temp_time.tv_sec); + CopyTime(&temp_time, &pv->shared->last_processed); + + CleanupPendingFile(current_file); + pv->current_file = NULL; + + status = PcapRunStatus(pv); + } + } + } + + *newer_than = *older_than; + } + GetTime(older_than); + older_than->tv_sec = older_than->tv_sec - pv->delay; + + SCReturnInt(status); +} + +TmEcode PcapDirectoryDispatch(PcapFileDirectoryVars *ptv) +{ + SCEnter(); + + DIR *directory_check = NULL; + + struct timespec newer_than; + memset(&newer_than, 0, sizeof(struct timespec)); + struct timespec older_than; + memset(&older_than, 0, sizeof(struct timespec)); + older_than.tv_sec = LONG_MAX; + uint32_t poll_seconds = (uint32_t)localtime(&ptv->poll_interval)->tm_sec; + + if (ptv->should_loop) { + GetTime(&older_than); + older_than.tv_sec = older_than.tv_sec - ptv->delay; + } + TmEcode status = TM_ECODE_OK; + + while (status == TM_ECODE_OK) { + //loop while directory is ok + SCLogInfo("Processing pcaps directory %s, files must be newer than %" PRIuMAX " and older than %" PRIuMAX, + ptv->filename, (uintmax_t)newer_than.tv_sec, (uintmax_t)older_than.tv_sec); + status = PcapDirectoryDispatchForTimeRange(ptv, &newer_than, &older_than); + if (ptv->should_loop && status == TM_ECODE_OK) { + sleep(poll_seconds); + //update our status based on suricata control flags or unix command socket + status = PcapRunStatus(ptv); + if (status == TM_ECODE_OK) { + SCLogDebug("Checking if directory %s still exists", ptv->filename); + //check directory + if (PcapDetermineDirectoryOrFile(ptv->filename, + &directory_check) == TM_ECODE_FAILED) { + SCLogInfo("Directory %s no longer exists, stopping", + ptv->filename); + status = TM_ECODE_DONE; + } else { + closedir(directory_check); + } + } + } else if (status == TM_ECODE_OK) { //not looping, mark done + SCLogDebug("Not looping, stopping directory mode"); + status = TM_ECODE_DONE; + } + } + + StatsSyncCountersIfSignalled(ptv->shared->tv); + + if (status == TM_ECODE_FAILED) { + SCLogError(SC_ERR_PCAP_DISPATCH, "Directory %s run mode failed", ptv->filename); + status = PcapDirectoryFailure(ptv); + } else { + SCLogInfo("Directory run mode complete"); + status = PcapDirectoryDone(ptv); + } + + SCReturnInt(status); +} + +/* eof */ diff --git a/src/source-pcap-file-directory-helper.h b/src/source-pcap-file-directory-helper.h new file mode 100644 index 0000000000..496e83793b --- /dev/null +++ b/src/source-pcap-file-directory-helper.h @@ -0,0 +1,83 @@ +/* Copyright (C) 2007-2010 Open Information Security Foundation + * + * You can copy, redistribute or modify this Program under the terms of + * the GNU General Public License version 2 as published by the Free + * Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * version 2 along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +/** + * \file + * + * \author Danny Browning + */ + +#include "suricata-common.h" +#include "source-pcap-file-helper.h" +#include "queue.h" + +#ifndef __SOURCE_PCAP_FILE_DIRECTORY_HELPER_H__ +#define __SOURCE_PCAP_FILE_DIRECTORY_HELPER_H__ + +typedef struct PendingFile_ +{ + char *filename; + TAILQ_ENTRY(PendingFile_) next; +} PendingFile; +/** + * Data specific to a directory of pcap files + */ +typedef struct PcapFileDirectoryVars_ +{ + char *filename; + DIR *directory; + PcapFileFileVars *current_file; + bool should_loop; + time_t delay; + time_t poll_interval; + + TAILQ_HEAD(PendingFiles, PendingFile_) directory_content; + + PcapFileSharedVars *shared; +} PcapFileDirectoryVars; + +/** + * Cleanup resources associated with a PendingFile object + * @param pending Object to be cleaned up + */ +void CleanupPendingFile(PendingFile *pending); + +/** + * Cleanup resources associated with a PcapFileDirectoryVars object + * @param ptv Object to be cleaned up + */ +void CleanupPcapFileDirectoryVars(PcapFileDirectoryVars *ptv); + +/** + * Determine if a given string represents a file or directory. If a directory, + * populate the directory object. + * @param filename String to check + * @param directory Directory point to populate if directory + * @return TM_ECODE_OK if string or directory + */ +TmEcode PcapDetermineDirectoryOrFile(char *filename, DIR **directory); + +/** + * Dispatch a directory for processing, where information for processing the + * directory is contained in a PcapFileDirectoryVars object + * @param ptv PcapFileDirectoryVars object containing information for processing + * the directory + * @return + */ +TmEcode PcapDirectoryDispatch(PcapFileDirectoryVars *ptv); + +#endif /* __SOURCE_PCAP_FILE_DIRECTORY_HELPER_H__ */ diff --git a/src/source-pcap-file-helper.c b/src/source-pcap-file-helper.c new file mode 100644 index 0000000000..40b753f52e --- /dev/null +++ b/src/source-pcap-file-helper.c @@ -0,0 +1,235 @@ +/* Copyright (C) 2007-2016 Open Information Security Foundation + * + * You can copy, redistribute or modify this Program under the terms of + * the GNU General Public License version 2 as published by the Free + * Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * version 2 along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +/** + * \file + * + * \author Danny Browning + * + * File based pcap packet acquisition support + */ + +#include "source-pcap-file-helper.h" +#include "util-checksum.h" +#include "util-profiling.h" + +#ifdef __SC_CUDA_SUPPORT__ + +#include "util-cuda.h" +#include "util-cuda-buffer.h" +#include "util-mpm-ac.h" +#include "util-cuda-handlers.h" +#include "detect-engine.h" +#include "detect-engine-mpm.h" +#include "util-cuda-vars.h" + +#endif /* __SC_CUDA_SUPPORT__ */ + +extern int max_pending_packets; +extern PcapFileGlobalVars pcap_g; + +static void PcapFileCallbackLoop(char *user, struct pcap_pkthdr *h, u_char *pkt); + +void CleanupPcapFileFileVars(PcapFileFileVars *pfv) +{ + if (pfv != NULL) { + if (pfv->pcap_handle != NULL) { + pcap_close(pfv->pcap_handle); + pfv->pcap_handle = NULL; + } + if (pfv->filename != NULL) { + SCFree(pfv->filename); + pfv->filename = NULL; + } + pfv->shared = NULL; + SCFree(pfv); + } +} + +void PcapFileCallbackLoop(char *user, struct pcap_pkthdr *h, u_char *pkt) +{ + SCEnter(); + + PcapFileFileVars *ptv = (PcapFileFileVars *)user; + Packet *p = PacketGetFromQueueOrAlloc(); + + if (unlikely(p == NULL)) { + SCReturn; + } + PACKET_PROFILING_TMM_START(p, TMM_RECEIVEPCAPFILE); + + PKT_SET_SRC(p, PKT_SRC_WIRE); + p->ts.tv_sec = h->ts.tv_sec; + p->ts.tv_usec = h->ts.tv_usec; + SCLogDebug("p->ts.tv_sec %"PRIuMAX"", (uintmax_t)p->ts.tv_sec); + p->datalink = ptv->datalink; + p->pcap_cnt = ++pcap_g.cnt; + + p->pcap_v.tenant_id = ptv->shared->tenant_id; + ptv->shared->pkts++; + ptv->shared->bytes += h->caplen; + + if (unlikely(PacketCopyData(p, pkt, h->caplen))) { + TmqhOutputPacketpool(ptv->shared->tv, p); + PACKET_PROFILING_TMM_END(p, TMM_RECEIVEPCAPFILE); + SCReturn; + } + + /* We only check for checksum disable */ + if (pcap_g.checksum_mode == CHECKSUM_VALIDATION_DISABLE) { + p->flags |= PKT_IGNORE_CHECKSUM; + } else if (pcap_g.checksum_mode == CHECKSUM_VALIDATION_AUTO) { + if (ChecksumAutoModeCheck(ptv->shared->pkts, p->pcap_cnt, + SC_ATOMIC_GET(pcap_g.invalid_checksums))) { + pcap_g.checksum_mode = CHECKSUM_VALIDATION_DISABLE; + p->flags |= PKT_IGNORE_CHECKSUM; + } + } + + PACKET_PROFILING_TMM_END(p, TMM_RECEIVEPCAPFILE); + + if (TmThreadsSlotProcessPkt(ptv->shared->tv, ptv->shared->slot, p) != TM_ECODE_OK) { + pcap_breakloop(ptv->pcap_handle); + ptv->shared->cb_result = TM_ECODE_FAILED; + } + + SCReturn; +} + +/** + * \brief Main PCAP file reading Loop function + */ +TmEcode PcapFileDispatch(PcapFileFileVars *ptv) +{ + SCEnter(); + + int packet_q_len = 64; + int r; + TmEcode loop_result = TM_ECODE_OK; + + while (loop_result == TM_ECODE_OK) { + if (suricata_ctl_flags & SURICATA_STOP) { + SCReturnInt(TM_ECODE_OK); + } + + /* make sure we have at least one packet in the packet pool, to prevent + * us from alloc'ing packets at line rate */ + PacketPoolWait(); + + /* Right now we just support reading packets one at a time. */ + r = pcap_dispatch(ptv->pcap_handle, packet_q_len, + (pcap_handler)PcapFileCallbackLoop, (u_char *)ptv); + if (unlikely(r == -1)) { + SCLogError(SC_ERR_PCAP_DISPATCH, "error code %" PRId32 " %s for %s", + r, pcap_geterr(ptv->pcap_handle), ptv->filename); + if (ptv->shared->cb_result == TM_ECODE_FAILED) { + SCReturnInt(TM_ECODE_FAILED); + } + loop_result = TM_ECODE_DONE; + } else if (unlikely(r == 0)) { + SCLogInfo("pcap file %s end of file reached (pcap err code %" PRId32 ")", + ptv->filename, r); + ptv->shared->files++; + loop_result = TM_ECODE_DONE; + } else if (ptv->shared->cb_result == TM_ECODE_FAILED) { + SCLogError(SC_ERR_PCAP_DISPATCH, + "Pcap callback PcapFileCallbackLoop failed for %s", ptv->filename); + loop_result = TM_ECODE_FAILED; + } + StatsSyncCountersIfSignalled(ptv->shared->tv); + } + + SCReturnInt(loop_result); +} + +TmEcode InitPcapFile(PcapFileFileVars *pfv) +{ + char errbuf[PCAP_ERRBUF_SIZE] = ""; + + if(unlikely(pfv->filename == NULL)) { + SCLogError(SC_ERR_INVALID_ARGUMENT, "Filename was null"); + SCReturnInt(TM_ECODE_FAILED); + } + + pfv->pcap_handle = pcap_open_offline(pfv->filename, errbuf); + if (pfv->pcap_handle == NULL) { + SCLogError(SC_ERR_FOPEN, "%s", errbuf); + if (!RunModeUnixSocketIsActive()) { + SCReturnInt(TM_ECODE_FAILED); + } else { + UnixSocketPcapFile(TM_ECODE_FAILED, 0); + SCReturnInt(TM_ECODE_DONE); + } + } + + if (pfv->shared != NULL && pfv->shared->bpf_string != NULL) { + SCLogInfo("using bpf-filter \"%s\"", pfv->shared->bpf_string); + + if (pcap_compile(pfv->pcap_handle, &pfv->filter, pfv->shared->bpf_string, 1, 0) < 0) { + SCLogError(SC_ERR_BPF, "bpf compilation error %s for %s", + pcap_geterr(pfv->pcap_handle), pfv->filename); + SCReturnInt(TM_ECODE_FAILED); + } + + if (pcap_setfilter(pfv->pcap_handle, &pfv->filter) < 0) { + SCLogError(SC_ERR_BPF,"could not set bpf filter %s for %s", + pcap_geterr(pfv->pcap_handle), pfv->filename); + SCReturnInt(TM_ECODE_FAILED); + } + } + + pfv->datalink = pcap_datalink(pfv->pcap_handle); + SCLogDebug("datalink %" PRId32 "", pfv->datalink); + + Decoder temp; + TmEcode validated = ValidateLinkType(pfv->datalink, &temp); + SCReturnInt(validated); +} + +TmEcode ValidateLinkType(int datalink, Decoder *decoder) +{ + switch (datalink) { + case LINKTYPE_LINUX_SLL: + *decoder = DecodeSll; + break; + case LINKTYPE_ETHERNET: + *decoder = DecodeEthernet; + break; + case LINKTYPE_PPP: + *decoder = DecodePPP; + break; + case LINKTYPE_IPV4: + case LINKTYPE_RAW: + case LINKTYPE_RAW2: + *decoder = DecodeRaw; + break; + case LINKTYPE_NULL: + *decoder = DecodeNull; + break; + + default: + SCLogError( + SC_ERR_UNIMPLEMENTED, + "datalink type %" PRId32 " not (yet) supported in module PcapFile.", + datalink + ); + SCReturnInt(TM_ECODE_FAILED); + } + + SCReturnInt(TM_ECODE_OK); +} +/* eof */ diff --git a/src/source-pcap-file-helper.h b/src/source-pcap-file-helper.h new file mode 100644 index 0000000000..6c9a705ffb --- /dev/null +++ b/src/source-pcap-file-helper.h @@ -0,0 +1,109 @@ +/* Copyright (C) 2007-2010 Open Information Security Foundation + * + * You can copy, redistribute or modify this Program under the terms of + * the GNU General Public License version 2 as published by the Free + * Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * version 2 along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +/** + * \file + * + * \author Danny Browning + */ + +#include "suricata-common.h" +#include "tm-threads.h" + +#ifndef __SOURCE_PCAP_FILE_HELPER_H__ +#define __SOURCE_PCAP_FILE_HELPER_H__ + +typedef struct PcapFileGlobalVars_ { + uint64_t cnt; /** packet counter */ + ChecksumValidationMode conf_checksum_mode; + ChecksumValidationMode checksum_mode; + SC_ATOMIC_DECLARE(unsigned int, invalid_checksums); +} PcapFileGlobalVars; + +/** + * Data that is shared amongst File, Directory, and Thread level vars + */ +typedef struct PcapFileSharedVars_ +{ + char *bpf_string; + + uint32_t tenant_id; + + struct timespec last_processed; + + ThreadVars *tv; + TmSlot *slot; + + /* counters */ + uint64_t pkts; + uint64_t bytes; + uint64_t files; + + uint8_t done; + uint32_t errs; + + /** callback result -- set if one of the thread module failed. */ + int cb_result; +} PcapFileSharedVars; + +/** + * Data specific to a single pcap file + */ +typedef struct PcapFileFileVars_ +{ + char *filename; + pcap_t *pcap_handle; + + int datalink; + struct bpf_program filter; + + PcapFileSharedVars *shared; +} PcapFileFileVars; + +typedef int (*Decoder)(ThreadVars *, DecodeThreadVars *, Packet *, uint8_t *, uint16_t, PacketQueue *); + +/** + * Dispatch a file for processing, where the information necessary to process that + * file is as PcapFileFileVars object. + * @param ptv PcapFileFileVars object to be processed + * @return + */ +TmEcode PcapFileDispatch(PcapFileFileVars *ptv); + +/** + * From a PcapFileFileVars, prepare the filename for processing by setting + * pcap_handle, datalink, and filter + * @param pfv PcapFileFileVars object to populate + * @return + */ +TmEcode InitPcapFile(PcapFileFileVars *pfv); + +/** + * Cleanup resources associated with a PcapFileFileVars object. + * @param pfv Object to be cleaned up + */ +void CleanupPcapFileFileVars(PcapFileFileVars *pfv); + +/** + * Determine if a datalink type is valid, setting a decoder function if valid. + * @param datalink Datalink type to validate + * @param decoder Pointer to decoder to set if valid + * @return TM_ECODE_OK if valid datalink type and decoder has been set. + */ +TmEcode ValidateLinkType(int datalink, Decoder *decoder); + +#endif /* __SOURCE_PCAP_FILE_HELPER_H__ */ diff --git a/src/source-pcap-file.c b/src/source-pcap-file.c index c71bb6bcca..cbb7bcef36 100644 --- a/src/source-pcap-file.c +++ b/src/source-pcap-file.c @@ -24,26 +24,11 @@ */ #include "suricata-common.h" -#include "suricata.h" -#include "decode.h" -#include "packet-queue.h" -#include "threads.h" -#include "threadvars.h" -#include "tm-queuehandlers.h" #include "source-pcap-file.h" -#include "util-time.h" -#include "util-debug.h" -#include "conf.h" -#include "util-error.h" -#include "util-privs.h" -#include "tmqh-packetpool.h" -#include "tm-threads.h" -#include "util-optimize.h" +#include "source-pcap-file-helper.h" +#include "source-pcap-file-directory-helper.h" #include "flow-manager.h" -#include "util-profiling.h" -#include "runmode-unix-socket.h" #include "util-checksum.h" -#include "util-atomic.h" #ifdef __SC_CUDA_SUPPORT__ @@ -58,49 +43,84 @@ #endif /* __SC_CUDA_SUPPORT__ */ extern int max_pending_packets; +PcapFileGlobalVars pcap_g; -typedef struct PcapFileGlobalVars_ { - pcap_t *pcap_handle; - int (*Decoder)(ThreadVars *, DecodeThreadVars *, Packet *, uint8_t *, uint16_t, PacketQueue *); - int datalink; - struct bpf_program filter; - uint64_t cnt; /** packet counter */ - ChecksumValidationMode conf_checksum_mode; - ChecksumValidationMode checksum_mode; - SC_ATOMIC_DECLARE(unsigned int, invalid_checksums); - -} PcapFileGlobalVars; +/** + * Union determining whether the behavior of the thread is file or directory + */ +typedef union PcapFileBehaviorVar_ +{ + PcapFileDirectoryVars *directory; + PcapFileFileVars *file; +} PcapFileBehaviorVar; +/** + * Data specific to the thread + */ typedef struct PcapFileThreadVars_ { - uint32_t tenant_id; + PcapFileBehaviorVar behavior; + bool is_directory; - /* counters */ - uint32_t pkts; - uint64_t bytes; - - ThreadVars *tv; - TmSlot *slot; + PcapFileSharedVars shared; +} PcapFileThreadVars; - /** callback result -- set if one of the thread module failed. */ - int cb_result; +static TmEcode ReceivePcapFileLoop(ThreadVars *, void *, void *); +static TmEcode ReceivePcapFileThreadInit(ThreadVars *, const void *, void **); +static void ReceivePcapFileThreadExitStats(ThreadVars *, void *); +static TmEcode ReceivePcapFileThreadDeinit(ThreadVars *, void *); - uint8_t done; - uint32_t errs; -} PcapFileThreadVars; +static TmEcode DecodePcapFile(ThreadVars *, Packet *, void *, PacketQueue *, + PacketQueue *); +static TmEcode DecodePcapFileThreadInit(ThreadVars *, const void *, void **); +static TmEcode DecodePcapFileThreadDeinit(ThreadVars *tv, void *data); -static PcapFileGlobalVars pcap_g; +static void CleanupPcapDirectoryFromThreadVars(PcapFileThreadVars *tv, + PcapFileDirectoryVars *ptv); +static void CleanupPcapFileFromThreadVars(PcapFileThreadVars *tv, PcapFileFileVars *pfv); +static void CleanupPcapFileThreadVars(PcapFileThreadVars *tv); -TmEcode ReceivePcapFileLoop(ThreadVars *, void *, void *); +void CleanupPcapFileFromThreadVars(PcapFileThreadVars *tv, PcapFileFileVars *pfv) +{ + CleanupPcapFileFileVars(pfv); + if (tv->is_directory == 0) { + tv->behavior.file = NULL; + } +} -TmEcode ReceivePcapFileThreadInit(ThreadVars *, const void *, void **); -void ReceivePcapFileThreadExitStats(ThreadVars *, void *); -TmEcode ReceivePcapFileThreadDeinit(ThreadVars *, void *); +void CleanupPcapDirectoryFromThreadVars(PcapFileThreadVars *tv, PcapFileDirectoryVars *ptv) +{ + CleanupPcapFileDirectoryVars(ptv); + if (tv->is_directory == 1) { + tv->behavior.directory = NULL; + } +} -TmEcode DecodePcapFile(ThreadVars *, Packet *, void *, PacketQueue *, PacketQueue *); -TmEcode DecodePcapFileThreadInit(ThreadVars *, const void *, void **); -TmEcode DecodePcapFileThreadDeinit(ThreadVars *tv, void *data); +void CleanupPcapFileThreadVars(PcapFileThreadVars *tv) +{ + if(tv != NULL) { + if (tv->is_directory == 0) { + if (tv->behavior.file != NULL) { + CleanupPcapFileFromThreadVars(tv, tv->behavior.file); + } + tv->behavior.file = NULL; + } else { + if (tv->behavior.directory != NULL) { + CleanupPcapDirectoryFromThreadVars(tv, tv->behavior.directory); + } + tv->behavior.directory = NULL; + } + if (tv->shared.bpf_string != NULL) { + SCFree(tv->shared.bpf_string); + tv->shared.bpf_string = NULL; + } + SCFree(tv); + } +} +/** + * Pcap File Functionality + */ void TmModuleReceivePcapFileRegister (void) { tmm_modules[TMM_RECEIVEPCAPFILE].name = "ReceivePcapFile"; @@ -133,139 +153,54 @@ void PcapFileGlobalInit() SC_ATOMIC_INIT(pcap_g.invalid_checksums); } -static void PcapFileCallbackLoop(char *user, struct pcap_pkthdr *h, u_char *pkt) -{ - SCEnter(); - - PcapFileThreadVars *ptv = (PcapFileThreadVars *)user; - Packet *p = PacketGetFromQueueOrAlloc(); - - if (unlikely(p == NULL)) { - SCReturn; - } - PACKET_PROFILING_TMM_START(p, TMM_RECEIVEPCAPFILE); - - PKT_SET_SRC(p, PKT_SRC_WIRE); - p->ts.tv_sec = h->ts.tv_sec; - p->ts.tv_usec = h->ts.tv_usec; - SCLogDebug("p->ts.tv_sec %"PRIuMAX"", (uintmax_t)p->ts.tv_sec); - p->datalink = pcap_g.datalink; - p->pcap_cnt = ++pcap_g.cnt; - - p->pcap_v.tenant_id = ptv->tenant_id; - ptv->pkts++; - ptv->bytes += h->caplen; - - if (unlikely(PacketCopyData(p, pkt, h->caplen))) { - TmqhOutputPacketpool(ptv->tv, p); - PACKET_PROFILING_TMM_END(p, TMM_RECEIVEPCAPFILE); - SCReturn; - } - - /* We only check for checksum disable */ - if (pcap_g.checksum_mode == CHECKSUM_VALIDATION_DISABLE) { - p->flags |= PKT_IGNORE_CHECKSUM; - } else if (pcap_g.checksum_mode == CHECKSUM_VALIDATION_AUTO) { - if (ChecksumAutoModeCheck(ptv->pkts, p->pcap_cnt, - SC_ATOMIC_GET(pcap_g.invalid_checksums))) { - pcap_g.checksum_mode = CHECKSUM_VALIDATION_DISABLE; - p->flags |= PKT_IGNORE_CHECKSUM; - } - } - - PACKET_PROFILING_TMM_END(p, TMM_RECEIVEPCAPFILE); - - if (TmThreadsSlotProcessPkt(ptv->tv, ptv->slot, p) != TM_ECODE_OK) { - pcap_breakloop(pcap_g.pcap_handle); - ptv->cb_result = TM_ECODE_FAILED; - } - - SCReturn; -} - -/** - * \brief Main PCAP file reading Loop function - */ TmEcode ReceivePcapFileLoop(ThreadVars *tv, void *data, void *slot) { SCEnter(); - int packet_q_len = 64; - PcapFileThreadVars *ptv = (PcapFileThreadVars *)data; - int r; + TmEcode status = TM_ECODE_OK; + PcapFileThreadVars *ptv = (PcapFileThreadVars *) data; TmSlot *s = (TmSlot *)slot; - ptv->slot = s->slot_next; - ptv->cb_result = TM_ECODE_OK; + ptv->shared.slot = s->slot_next; + ptv->shared.cb_result = TM_ECODE_OK; - while (1) { - if (suricata_ctl_flags & SURICATA_STOP) { - SCReturnInt(TM_ECODE_OK); - } - - /* make sure we have at least one packet in the packet pool, to prevent - * us from alloc'ing packets at line rate */ - PacketPoolWait(); - - /* Right now we just support reading packets one at a time. */ - r = pcap_dispatch(pcap_g.pcap_handle, packet_q_len, - (pcap_handler)PcapFileCallbackLoop, (u_char *)ptv); - if (unlikely(r == -1)) { - SCLogError(SC_ERR_PCAP_DISPATCH, "error code %" PRId32 " %s", - r, pcap_geterr(pcap_g.pcap_handle)); - if (ptv->cb_result == TM_ECODE_FAILED) { - SCReturnInt(TM_ECODE_FAILED); - } - if (! RunModeUnixSocketIsActive()) { - EngineStop(); - } else { - pcap_close(pcap_g.pcap_handle); - pcap_g.pcap_handle = NULL; - UnixSocketPcapFile(TM_ECODE_DONE); - SCReturnInt(TM_ECODE_DONE); - } - } else if (unlikely(r == 0)) { - SCLogInfo("pcap file end of file reached (pcap err code %" PRId32 ")", r); - if (! RunModeUnixSocketIsActive()) { - EngineStop(); - } else { - pcap_close(pcap_g.pcap_handle); - pcap_g.pcap_handle = NULL; - UnixSocketPcapFile(TM_ECODE_DONE); - SCReturnInt(TM_ECODE_DONE); - } - break; - } else if (ptv->cb_result == TM_ECODE_FAILED) { - SCLogError(SC_ERR_PCAP_DISPATCH, "Pcap callback PcapFileCallbackLoop failed"); - if (! RunModeUnixSocketIsActive()) { - SCReturnInt(TM_ECODE_FAILED); - } else { - pcap_close(pcap_g.pcap_handle); - pcap_g.pcap_handle = NULL; - UnixSocketPcapFile(TM_ECODE_DONE); - SCReturnInt(TM_ECODE_DONE); - } + if(ptv->is_directory == 0) { + SCLogInfo("Starting file run for %s", ptv->behavior.file->filename); + status = PcapFileDispatch(ptv->behavior.file); + if (!RunModeUnixSocketIsActive()) { + EngineStop(); + } else { + UnixSocketPcapFile(status, &ptv->shared.last_processed); } - StatsSyncCountersIfSignalled(tv); + CleanupPcapFileFromThreadVars(ptv, ptv->behavior.file); + } else { + SCLogInfo("Starting directory run for %s", ptv->behavior.directory->filename); + PcapDirectoryDispatch(ptv->behavior.directory); + CleanupPcapDirectoryFromThreadVars(ptv, ptv->behavior.directory); } - SCReturnInt(TM_ECODE_OK); + SCLogDebug("Pcap file loop complete with status %u", status); + + if(RunModeUnixSocketIsActive()) { + SCReturnInt(TM_ECODE_DONE); + } else { + EngineStop(); + SCReturnInt(TM_ECODE_OK); + } } TmEcode ReceivePcapFileThreadInit(ThreadVars *tv, const void *initdata, void **data) { SCEnter(); - const char *tmpbpfstring = NULL; const char *tmpstring = NULL; + const char *tmp_bpf_string = NULL; if (initdata == NULL) { SCLogError(SC_ERR_INVALID_ARGUMENT, "error: initdata == NULL"); SCReturnInt(TM_ECODE_FAILED); } - SCLogInfo("reading pcap file %s", (char *)initdata); - PcapFileThreadVars *ptv = SCMalloc(sizeof(PcapFileThreadVars)); if (unlikely(ptv == NULL)) SCReturnInt(TM_ECODE_FAILED); @@ -274,79 +209,114 @@ TmEcode ReceivePcapFileThreadInit(ThreadVars *tv, const void *initdata, void **d intmax_t tenant = 0; if (ConfGetInt("pcap-file.tenant-id", &tenant) == 1) { if (tenant > 0 && tenant < UINT_MAX) { - ptv->tenant_id = (uint32_t)tenant; - SCLogInfo("tenant %u", ptv->tenant_id); + ptv->shared.tenant_id = (uint32_t)tenant; + SCLogInfo("tenant %u", ptv->shared.tenant_id); } else { SCLogError(SC_ERR_INVALID_ARGUMENT, "tenant out of range"); } } - char errbuf[PCAP_ERRBUF_SIZE] = ""; - pcap_g.pcap_handle = pcap_open_offline((char *)initdata, errbuf); - if (pcap_g.pcap_handle == NULL) { - SCLogError(SC_ERR_FOPEN, "%s\n", errbuf); - SCFree(ptv); - if (! RunModeUnixSocketIsActive()) { - return TM_ECODE_FAILED; - } else { - UnixSocketPcapFile(TM_ECODE_FAILED); - SCReturnInt(TM_ECODE_DONE); + if (ConfGet("bpf-filter", &(tmp_bpf_string)) != 1) { + SCLogDebug("could not get bpf or none specified"); + } else { + ptv->shared.bpf_string = SCStrdup(tmp_bpf_string); + if (unlikely(ptv->shared.bpf_string == NULL)) { + SCLogError(SC_ERR_MEM_ALLOC, "Failed to allocate bpf_string"); + CleanupPcapFileThreadVars(ptv); + SCReturnInt(TM_ECODE_FAILED); } } - if (ConfGet("bpf-filter", &tmpbpfstring) != 1) { - SCLogDebug("could not get bpf or none specified"); + DIR *directory = NULL; + SCLogInfo("Checking file or directory %s", (char*)initdata); + if(PcapDetermineDirectoryOrFile((char *)initdata, &directory) == TM_ECODE_FAILED) { + CleanupPcapFileThreadVars(ptv); + SCReturnInt(TM_ECODE_FAILED); + } + + if(directory == NULL) { + SCLogInfo("Argument %s was a file", (char *)initdata); + PcapFileFileVars *pv = SCMalloc(sizeof(PcapFileFileVars)); + if (unlikely(pv == NULL)) { + SCLogError(SC_ERR_MEM_ALLOC, "Failed to allocate file vars"); + CleanupPcapFileThreadVars(ptv); + SCReturnInt(TM_ECODE_FAILED); + } + memset(pv, 0, sizeof(PcapFileFileVars)); + + pv->filename = SCStrdup((char *)initdata); + if (unlikely(pv->filename == NULL)) { + SCLogError(SC_ERR_MEM_ALLOC, "Failed to allocate filename"); + CleanupPcapFileFileVars(pv); + CleanupPcapFileThreadVars(ptv); + SCReturnInt(TM_ECODE_FAILED); + } + + TmEcode init_file_return = InitPcapFile(pv); + if(init_file_return == TM_ECODE_OK) { + pv->shared = &ptv->shared; + + ptv->is_directory = 0; + ptv->behavior.file = pv; + } else { + SCLogWarning(SC_ERR_PCAP_DISPATCH, + "Failed to init pcap file %s, skipping", (char *)initdata); + CleanupPcapFileFileVars(pv); + CleanupPcapFileThreadVars(ptv); + SCReturnInt(init_file_return); + } } else { - SCLogInfo("using bpf-filter \"%s\"", tmpbpfstring); + SCLogInfo("Argument %s was a directory", (char *)initdata); + PcapFileDirectoryVars *pv = SCMalloc(sizeof(PcapFileDirectoryVars)); + if (unlikely(pv == NULL)) { + SCLogError(SC_ERR_MEM_ALLOC, "Failed to allocate directory vars"); + CleanupPcapFileThreadVars(ptv); + SCReturnInt(TM_ECODE_FAILED); + } + memset(pv, 0, sizeof(PcapFileDirectoryVars)); + + pv->filename = SCStrdup((char*)initdata); + if (unlikely(pv->filename == NULL)) { + SCLogError(SC_ERR_MEM_ALLOC, "Failed to allocate filename"); + CleanupPcapFileDirectoryVars(pv); + CleanupPcapFileThreadVars(ptv); + SCReturnInt(TM_ECODE_FAILED); + } - if (pcap_compile(pcap_g.pcap_handle, &pcap_g.filter, (char *)tmpbpfstring, 1, 0) < 0) { - SCLogError(SC_ERR_BPF,"bpf compilation error %s", - pcap_geterr(pcap_g.pcap_handle)); - SCFree(ptv); - return TM_ECODE_FAILED; + int should_loop = 0; + pv->should_loop = false; + if (ConfGetBool("pcap-file.continuous", &should_loop) == 1) { + pv->should_loop = should_loop == 1; } - if (pcap_setfilter(pcap_g.pcap_handle, &pcap_g.filter) < 0) { - SCLogError(SC_ERR_BPF,"could not set bpf filter %s", pcap_geterr(pcap_g.pcap_handle)); - SCFree(ptv); - return TM_ECODE_FAILED; + pv->delay = 30; + intmax_t delay = 0; + if (ConfGetInt("pcap-file.delay", &delay) == 1) { + if (delay > 0 && delay < UINT_MAX) { + pv->delay = (time_t)delay; + SCLogDebug("delay %lu", pv->delay); + } else { + SCLogError(SC_ERR_INVALID_ARGUMENT, "delay out of range"); + } } - } - pcap_g.datalink = pcap_datalink(pcap_g.pcap_handle); - SCLogDebug("datalink %" PRId32 "", pcap_g.datalink); - - switch (pcap_g.datalink) { - case LINKTYPE_LINUX_SLL: - pcap_g.Decoder = DecodeSll; - break; - case LINKTYPE_ETHERNET: - pcap_g.Decoder = DecodeEthernet; - break; - case LINKTYPE_PPP: - pcap_g.Decoder = DecodePPP; - break; - case LINKTYPE_IPV4: - case LINKTYPE_RAW: - case LINKTYPE_RAW2: - pcap_g.Decoder = DecodeRaw; - break; - case LINKTYPE_NULL: - pcap_g.Decoder = DecodeNull; - break; - - default: - SCLogError(SC_ERR_UNIMPLEMENTED, "datalink type %" PRId32 " not " - "(yet) supported in module PcapFile.\n", pcap_g.datalink); - SCFree(ptv); - if (! RunModeUnixSocketIsActive()) { - SCReturnInt(TM_ECODE_FAILED); + pv->poll_interval = 5; + intmax_t poll_interval = 0; + if (ConfGetInt("pcap-file.poll-interval", &poll_interval) == 1) { + if (poll_interval > 0 && poll_interval < UINT_MAX) { + pv->poll_interval = (time_t)poll_interval; + SCLogDebug("poll-interval %lu", pv->delay); } else { - pcap_close(pcap_g.pcap_handle); - pcap_g.pcap_handle = NULL; - UnixSocketPcapFile(TM_ECODE_DONE); - SCReturnInt(TM_ECODE_DONE); + SCLogError(SC_ERR_INVALID_ARGUMENT, "poll-interval out of range"); } + } + + pv->shared = &ptv->shared; + pv->directory = directory; + TAILQ_INIT(&pv->directory_content); + + ptv->is_directory = 1; + ptv->behavior.directory = pv; } if (ConfGet("pcap-file.checksum-checks", &tmpstring) != 1) { @@ -362,7 +332,7 @@ TmEcode ReceivePcapFileThreadInit(ThreadVars *tv, const void *initdata, void **d } pcap_g.checksum_mode = pcap_g.conf_checksum_mode; - ptv->tv = tv; + ptv->shared.tv = tv; *data = (void *)ptv; SCReturnInt(TM_ECODE_OK); @@ -374,30 +344,32 @@ void ReceivePcapFileThreadExitStats(ThreadVars *tv, void *data) PcapFileThreadVars *ptv = (PcapFileThreadVars *)data; if (pcap_g.conf_checksum_mode == CHECKSUM_VALIDATION_AUTO && - pcap_g.cnt < CHECKSUM_SAMPLE_COUNT && - SC_ATOMIC_GET(pcap_g.invalid_checksums)) { + pcap_g.cnt < CHECKSUM_SAMPLE_COUNT && + SC_ATOMIC_GET(pcap_g.invalid_checksums)) { uint64_t chrate = pcap_g.cnt / SC_ATOMIC_GET(pcap_g.invalid_checksums); if (chrate < CHECKSUM_INVALID_RATIO) SCLogWarning(SC_ERR_INVALID_CHECKSUM, "1/%" PRIu64 "th of packets have an invalid checksum," - " consider setting pcap-file.checksum-checks variable to no" - " or use '-k none' option on command line.", + " consider setting pcap-file.checksum-checks variable to no" + " or use '-k none' option on command line.", chrate); else SCLogInfo("1/%" PRIu64 "th of packets have an invalid checksum", chrate); } - SCLogNotice("Pcap-file module read %" PRIu32 " packets, %" PRIu64 " bytes", ptv->pkts, ptv->bytes); - return; + SCLogNotice( + "Pcap-file module read %" PRIu64 " files, %" PRIu64 " packets, %" PRIu64 " bytes", + ptv->shared.files, + ptv->shared.pkts, + ptv->shared.bytes + ); } TmEcode ReceivePcapFileThreadDeinit(ThreadVars *tv, void *data) { SCEnter(); PcapFileThreadVars *ptv = (PcapFileThreadVars *)data; - if (ptv) { - SCFree(ptv); - } + CleanupPcapFileThreadVars(ptv); SCReturnInt(TM_ECODE_OK); } @@ -422,16 +394,22 @@ TmEcode DecodePcapFile(ThreadVars *tv, Packet *p, void *data, PacketQueue *pq, P FlowWakeupFlowManagerThread(); } - /* call the decoder */ - pcap_g.Decoder(tv, dtv, p, GET_PKT_DATA(p), GET_PKT_LEN(p), pq); + Decoder decoder; + if(ValidateLinkType(p->datalink, &decoder) == TM_ECODE_OK) { + + /* call the decoder */ + decoder(tv, dtv, p, GET_PKT_DATA(p), GET_PKT_LEN(p), pq); #ifdef DEBUG - BUG_ON(p->pkt_src != PKT_SRC_WIRE && p->pkt_src != PKT_SRC_FFR); + BUG_ON(p->pkt_src != PKT_SRC_WIRE && p->pkt_src != PKT_SRC_FFR); #endif - PacketDecodeFinalize(tv, dtv, p); + PacketDecodeFinalize(tv, dtv, p); - SCReturnInt(TM_ECODE_OK); + SCReturnInt(TM_ECODE_OK); + } else { + SCReturnInt(TM_ECODE_FAILED); + } } TmEcode DecodePcapFileThreadInit(ThreadVars *tv, const void *initdata, void **data) @@ -468,4 +446,3 @@ void PcapIncreaseInvalidChecksum() } /* eof */ - diff --git a/src/suricata-common.h b/src/suricata-common.h index 31c3993652..50a5ade86b 100644 --- a/src/suricata-common.h +++ b/src/suricata-common.h @@ -45,6 +45,10 @@ #define CLS 64 #endif +#if HAVE_DIRENT_H +#include +#endif + #if HAVE_STDIO_H #include #endif diff --git a/src/suricata.c b/src/suricata.c index 339801d350..09e8d23a49 100644 --- a/src/suricata.c +++ b/src/suricata.c @@ -655,6 +655,7 @@ static void PrintUsage(const char *progname) printf("\t--dump-config : show the running configuration\n"); printf("\t--build-info : display build information\n"); printf("\t--pcap[=] : run in pcap mode, no value select interfaces from suricata.yaml\n"); + printf("\t--pcap-file-continuous : when running in pcap mode with a directory, continue checking directory for pcaps until interrupted"); #ifdef HAVE_PCAP_SET_BUFF printf("\t--pcap-buffer-size : size of the pcap buffer value from 0 - %i\n",INT_MAX); #endif /* HAVE_SET_PCAP_BUFF */ @@ -1497,6 +1498,7 @@ static TmEcode ParseCommandLine(int argc, char** argv, SCInstance *suri) {"af-packet", optional_argument, 0, 0}, {"netmap", optional_argument, 0, 0}, {"pcap", optional_argument, 0, 0}, + {"pcap-file-continuous", 0, 0, 0}, {"simulate-ips", 0, 0 , 0}, {"no-random", 0, &g_disable_randomness, 1}, @@ -1873,6 +1875,13 @@ static TmEcode ParseCommandLine(int argc, char** argv, SCInstance *suri) } } } + else if (strcmp((long_opts[option_index]).name, "pcap-file-continuous") == 0) { + if(ConfSetFinal("pcap-file.continuous", "true") != 1) { + SCLogError(SC_ERR_CMD_LINE, "Failed to set pcap-file.continuous"); + return TM_ECODE_FAILED; + } + return TM_ECODE_OK; + } break; case 'c': suri->conf_filename = optarg; @@ -2010,6 +2019,7 @@ static TmEcode ParseCommandLine(int argc, char** argv, SCInstance *suri) fprintf(stderr, "ERROR: Failed to set pcap-file.file\n"); return TM_ECODE_FAILED; } + break; case 's': if (suri->sig_file != NULL) { -- 2.47.2