From: Danny Browning Date: Tue, 16 Jan 2018 16:24:16 +0000 (-0700) Subject: source-pcap-file: delete when done (2417) X-Git-Tag: suricata-4.1.0-rc1~18 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=2dc6b6ee14b0886783674fecaaa50e62328f9727;p=thirdparty%2Fsuricata.git source-pcap-file: delete when done (2417) https://redmine.openinfosecfoundation.org/issues/2417 Add option to have pcap files deleted after they have been processed. This option combines well with pcap file continuous and streaming files to a directory being processed. --- diff --git a/doc/userguide/partials/options.rst b/doc/userguide/partials/options.rst index a058fb2bbe..dd41057557 100644 --- a/doc/userguide/partials/options.rst +++ b/doc/userguide/partials/options.rst @@ -25,9 +25,9 @@ .. option:: -r - Run in pcap offline mode reading files from pcap file. If specifies - a directory, all files in that directory will be processed in order of - modified time maintaining flow state between files. + Run in pcap offline mode (replay mode) reading files from pcap file. If + specifies a directory, all files in that directory will be processed + in order of modified time maintaining flow state between files. .. option:: --pcap-file-continuous @@ -35,6 +35,13 @@ interrupted. This is useful with directories to add new files and not reset flow state between files. +.. option:: --pcap-file-delete + + Used with the -r option to indicate that the mode should delete pcap files + after they have been processed. This is useful with pcap-file-continuous to + continuously feed files to a directory and have them cleaned up when done. If + this option is not set, pcap files will not be deleted after processing. + .. option:: -i After the -i option you can enter the interface card you would like diff --git a/doc/userguide/unix-socket.rst b/doc/userguide/unix-socket.rst index 9a76783a23..cdd77d22df 100644 --- a/doc/userguide/unix-socket.rst +++ b/doc/userguide/unix-socket.rst @@ -256,9 +256,37 @@ In pcap-file mode, this gives: SND: {"command": "pcap-file-list"} RCV: {"message": {"count": 1, "files": ["/home/eric/git/oisf/benches/sandnet.pcap"]}, "return": "OK"} Success: {'count': 1, 'files': ['/home/eric/git/oisf/benches/sandnet.pcap']} + >>> pcap-file-continuous /home/eric/git/oisf/benches /tmp/bench 0 true + SND: {"command": "pcap-file", "arguments": {"output-dir": "/tmp/bench", "filename": "/home/eric/git/oisf/benches/sandnet.pcap", "tenant": 0, "delete-when-done": true}} + RCV: {"message": "Successfully added file to list", "return": "OK"} + Success: Successfully added file to list There is one thing to be careful about: a Suricata message is sent in multiple send operations. This result in possible incomplete read on client side. The worse workaround is to sleep a bit before trying a recv call. An other solution is to use non blocking socket and retry a recv if the previous one has failed. + +Pcap-file json format is: + +:: + + { + "command": "pcap-file", + "arguments": { + "output-dir": "path to output dir", + "filename": "path to file or directory to run", + "tenant": 0, + "continuous": false, + "delete-when-done": false + } + } + +`output-dir` and `filename` are required. `tenant` is optional and should be a +number, indicating which tenant the file or directory should run under. `continuous` +is optional and should be true/false, indicating that file or directory should be +run until `pcap-interrupt` is sent or ctrl-c is invoked. `delete-when-done` is +optional and should be true/false, indicating that the file or files under the +directory specified by `filename` should be deleted when processing is complete. +`delete-when-done` defaults to false, indicating files will be kept after +processing. diff --git a/python/suricata/sc/suricatasc.py b/python/suricata/sc/suricatasc.py index 22501afc25..2e3d14c38b 100644 --- a/python/suricata/sc/suricatasc.py +++ b/python/suricata/sc/suricatasc.py @@ -184,6 +184,9 @@ class SuricataSC: continuous = None if len(parts) > 4: continuous = parts[4] + delete_when_done = None + if len(parts) > 5: + delete_when_done = parts[5] if cmd != "pcap-file": raise SuricataCommandException("Invalid command '%s'" % (command)) else: @@ -194,6 +197,8 @@ class SuricataSC: arguments["tenant"] = int(tenant) if continuous != None: arguments["continuous"] = continuous + if delete_when_done != None: + arguments["delete-when-done"] = delete_when_done elif "pcap-file-continuous " in command: try: parts = command.split(' ') @@ -203,6 +208,9 @@ class SuricataSC: tenant = None if len(parts) > 3: tenant = parts[3] + delete_when_done = None + if len(parts) > 4: + delete_when_done = parts[4] if cmd != "pcap-file": raise SuricataCommandException("Invalid command '%s'" % (command)) else: @@ -212,6 +220,8 @@ class SuricataSC: arguments["continuous"] = True if tenant != None: arguments["tenant"] = int(tenant) + if delete_when_done != None: + arguments["delete-when-done"] = delete_when_done elif "iface-stat" in command: try: [cmd, iface] = command.split(' ', 1) diff --git a/src/runmode-unix-socket.c b/src/runmode-unix-socket.c index 31ded2c6e3..21b2e23c6f 100644 --- a/src/runmode-unix-socket.c +++ b/src/runmode-unix-socket.c @@ -59,6 +59,7 @@ typedef struct PcapFiles_ { time_t delay; time_t poll_interval; bool continuous; + bool should_delete; TAILQ_ENTRY(PcapFiles_) next; } PcapFiles; @@ -238,6 +239,7 @@ static void PcapFilesFree(PcapFiles *cfile) * \param output_dir absolute name of directory where log will be put * \param tenant_id Id of tenant associated with this file * \param continuous If file should be run in continuous mode + * \param delete If file should be deleted when done * \param delay Delay required for file modified time before being processed * \param poll_interval How frequently directory mode polls for new files * @@ -249,6 +251,7 @@ static TmEcode UnixListAddFile( const char *output_dir, int tenant_id, bool continuous, + bool should_delete, time_t delay, time_t poll_interval ) @@ -282,6 +285,7 @@ static TmEcode UnixListAddFile( cfile->tenant_id = tenant_id; cfile->continuous = continuous; + cfile->should_delete = should_delete; cfile->delay = delay; cfile->poll_interval = poll_interval; @@ -304,6 +308,7 @@ static TmEcode UnixSocketAddPcapFileImpl(json_t *cmd, json_t* answer, void *data const char *filename; const char *output_dir; int tenant_id = 0; + bool should_delete = false; time_t delay = 30; time_t poll_interval = 5; #ifdef OS_WIN32 @@ -368,6 +373,11 @@ static TmEcode UnixSocketAddPcapFileImpl(json_t *cmd, json_t* answer, void *data tenant_id = json_number_value(targ); } + json_t *delete_arg = json_object_get(cmd, "delete-when-done"); + if (delete_arg != NULL) { + should_delete = json_is_true(delete_arg); + } + json_t *delay_arg = json_object_get(cmd, "delay"); if (delay_arg != NULL) { if (!json_is_integer(delay_arg)) { @@ -392,7 +402,7 @@ static TmEcode UnixSocketAddPcapFileImpl(json_t *cmd, json_t* answer, void *data } switch (UnixListAddFile(this, filename, output_dir, tenant_id, continuous, - delay, poll_interval)) { + should_delete, delay, poll_interval)) { case TM_ECODE_FAILED: case TM_ECODE_DONE: json_object_set_new(answer, "message", @@ -502,6 +512,16 @@ static TmEcode UnixSocketPcapFilesCheck(void *data) PcapFilesFree(cfile); return TM_ECODE_FAILED; } + if (cfile->should_delete) { + set_res = ConfSetFinal("pcap-file.delete-when-done", "true"); + } else { + set_res = ConfSetFinal("pcap-file.delete-when-done", "false"); + } + if (set_res != 1) { + SCLogError(SC_ERR_INVALID_ARGUMENT, "Can not set delete mode for pcap processing"); + PcapFilesFree(cfile); + return TM_ECODE_FAILED; + } if (cfile->delay > 0) { char tstr[32]; diff --git a/src/source-pcap-file-helper.c b/src/source-pcap-file-helper.c index a8c8f67788..7a4f67803d 100644 --- a/src/source-pcap-file-helper.c +++ b/src/source-pcap-file-helper.c @@ -41,6 +41,13 @@ void CleanupPcapFileFileVars(PcapFileFileVars *pfv) pfv->pcap_handle = NULL; } if (pfv->filename != NULL) { + if (pfv->shared->should_delete) { + SCLogDebug("Deleting pcap file %s", pfv->filename); + if (unlink(pfv->filename) != 0) { + SCLogWarning(SC_ERR_PCAP_FILE_DELETE_FAILED, + "Failed to delete %s", pfv->filename); + } + } SCFree(pfv->filename); pfv->filename = NULL; } diff --git a/src/source-pcap-file-helper.h b/src/source-pcap-file-helper.h index 6c9a705ffb..e12005b8ad 100644 --- a/src/source-pcap-file-helper.h +++ b/src/source-pcap-file-helper.h @@ -45,6 +45,8 @@ typedef struct PcapFileSharedVars_ struct timespec last_processed; + bool should_delete; + ThreadVars *tv; TmSlot *slot; diff --git a/src/source-pcap-file.c b/src/source-pcap-file.c index 0857eb6b76..cd5cf0db7a 100644 --- a/src/source-pcap-file.c +++ b/src/source-pcap-file.c @@ -231,6 +231,12 @@ TmEcode ReceivePcapFileThreadInit(ThreadVars *tv, const void *initdata, void **d } } + int should_delete = 0; + ptv->shared.should_delete = false; + if (ConfGetBool("pcap-file.delete-when-done", &should_delete) == 1) { + ptv->shared.should_delete = should_delete == 1; + } + DIR *directory = NULL; SCLogInfo("Checking file or directory %s", (char*)initdata); if(PcapDetermineDirectoryOrFile((char *)initdata, &directory) == TM_ECODE_FAILED) { diff --git a/src/suricata.c b/src/suricata.c index 77a550d505..4124babe7d 100644 --- a/src/suricata.c +++ b/src/suricata.c @@ -622,6 +622,7 @@ static void PrintUsage(const char *progname) printf("\t--build-info : display build information\n"); printf("\t--pcap[=] : run in pcap mode, no value select interfaces from suricata.yaml\n"); printf("\t--pcap-file-continuous : when running in pcap mode with a directory, continue checking directory for pcaps until interrupted\n"); + printf("\t--pcap-file-delete : when running in replay mode (-r with directory or file), will delete pcap files that have been processed when done\n"); #ifdef HAVE_PCAP_SET_BUFF printf("\t--pcap-buffer-size : size of the pcap buffer value from 0 - %i\n",INT_MAX); #endif /* HAVE_SET_PCAP_BUFF */ @@ -1474,6 +1475,7 @@ static TmEcode ParseCommandLine(int argc, char** argv, SCInstance *suri) {"netmap", optional_argument, 0, 0}, {"pcap", optional_argument, 0, 0}, {"pcap-file-continuous", 0, 0, 0}, + {"pcap-file-delete", 0, 0, 0}, {"simulate-ips", 0, 0 , 0}, {"no-random", 0, &g_disable_randomness, 1}, @@ -1887,11 +1889,17 @@ static TmEcode ParseCommandLine(int argc, char** argv, SCInstance *suri) } } else if (strcmp((long_opts[option_index]).name, "pcap-file-continuous") == 0) { - if(ConfSetFinal("pcap-file.continuous", "true") != 1) { + if (ConfSetFinal("pcap-file.continuous", "true") != 1) { SCLogError(SC_ERR_CMD_LINE, "Failed to set pcap-file.continuous"); return TM_ECODE_FAILED; } } + else if (strcmp((long_opts[option_index]).name, "pcap-file-delete") == 0) { + if (ConfSetFinal("pcap-file.delete-when-done", "true") != 1) { + SCLogError(SC_ERR_CMD_LINE, "Failed to set pcap-file.delete-when-done"); + return TM_ECODE_FAILED; + } + } break; case 'c': suri->conf_filename = optarg;