]> git.ipfire.org Git - thirdparty/suricata.git/commitdiff
eve/ftp: Log FTP transactions
authorJeff Lucovsky <jeff@lucovsky.org>
Mon, 1 Apr 2019 22:14:28 +0000 (15:14 -0700)
committerVictor Julien <victor@inliniac.net>
Wed, 17 Jul 2019 06:21:54 +0000 (08:21 +0200)
This changeset includes changes that
1. Add transaction support to the FTP parser
2. Support eve json logging of FTP transactions

13 files changed:
doc/userguide/output/eve/eve-json-format.rst
src/Makefile.am
src/app-layer-ftp.c
src/app-layer-ftp.h
src/output-json-ftp.c [new file with mode: 0644]
src/output-json-ftp.h [new file with mode: 0644]
src/output-json.c
src/output-json.h
src/output.c
src/suricata-common.h
src/util-error.c
src/util-error.h
src/util-profiling.c

index 9bff6d94163d2e4396ef115edd41735b5a27edc8..5eaf0240c719e188853e33c06154cefeda9d0cc9 100644 (file)
@@ -533,6 +533,74 @@ Example of a old DNS answer with an IPv4 (resource record type 'A') return:
       "rdata": "199.16.156.6"
   }
 
+Event type: FTP
+---------------
+
+Fields
+~~~~~~
+
+* "command": The FTP command.
+* "command_data": The data accompanying the command.
+* "reply": The command reply, which may contain multiple lines, in array format.
+* "completion_code": The 3-digit completion code. The first digit indicates whether the response is good, bad or incomplete.
+* "dynamic_port": The dynamic port established for subsequent data transfers, when applicable, with a "PORT" or "EPRT" command.
+* "mode": The type of FTP connection. Most connections are "passive" but may be "active".
+
+
+Examples
+~~~~~~~~
+
+Example of regular FTP logging:
+
+::
+
+  "ftp": {
+    "command": "RETR",
+    "command_data": "index.html",
+    "reply": [
+      "Opening BINARY mode data connection for index.html (6712 bytes)",
+      "Transfer complete"
+    ],
+    "completion_code": "150"
+  }
+
+Example showing all fields
+
+::
+
+  "ftp": {
+    "command": "EPRT",
+    "command_data": "|2|2a01:e34:ee97:b130:8c3e:45ea:5ac6:e301|41813|",
+    "reply": [
+      "EPRT command successful. Consider using EPSV"
+    ],
+    "reply_code": "200",
+    "dynamic_port": 41813,
+    "mode": "active"
+  }
+
+Event type: FTP_DATA
+--------------------
+
+Fields
+~~~~~~
+
+* "command": The FTP command associated with the event.
+* "filename": The name of the involved file.
+
+
+Examples
+~~~~~~~~
+
+Example of FTP_DATA logging:
+
+::
+
+  "ftp_data": {
+    "filename": "temp.txt",
+    "command": "RETR"
+  }
+
 Event type: TLS
 ---------------
 
index b3a3cc720f392ff64fd9314b97c844f8032c7d59..0b497a9b785398c097214944a9120d61039ca1e7 100644 (file)
@@ -320,6 +320,7 @@ output-json-drop.c output-json-drop.h \
 output-json-email-common.c output-json-email-common.h \
 output-json-file.c output-json-file.h \
 output-json-flow.c output-json-flow.h \
+output-json-ftp.c output-json-ftp.h \
 output-json-netflow.c output-json-netflow.h \
 output-json-http.c output-json-http.h \
 output-json-smtp.c output-json-smtp.h \
index 08a25ede0db50d3086e26bbff632dacd662262b7..86a6e822f63c39fcbe9edfc05dafb00ea20d5c22 100644 (file)
@@ -20,6 +20,7 @@
  *
  * \author Pablo Rincon Crespo <pablo.rincon.crespo@gmail.com>
  * \author Eric Leblond <eric@regit.org>
+ * \author Jeff Lucovsky <jeff@lucovsky.org>
  *
  * App Layer Parser for FTP
  */
 
 #include "output-json.h"
 
+const FtpCommand FtpCommands[FTP_COMMAND_MAX + 1] = {
+    /* Parsed and handled */
+    { FTP_COMMAND_PORT, "PORT", "port", 4},
+    { FTP_COMMAND_EPRT, "EPRT", "eprt", 4},
+    { FTP_COMMAND_AUTH_TLS, "AUTH TLS", "auth tls", 8},
+    { FTP_COMMAND_PASV, "PASV", "pasv", 4},
+    { FTP_COMMAND_RETR, "RETR", "retr", 4},
+    { FTP_COMMAND_EPSV, "EPSV", "epsv", 4},
+    { FTP_COMMAND_STOR, "STOR", "stor", 4},
+
+    /* Parsed, but not handled */
+    { FTP_COMMAND_ABOR, "ABOR", "abor", 4},
+    { FTP_COMMAND_ACCT, "ACCT", "acct", 4},
+    { FTP_COMMAND_ALLO, "ALLO", "allo", 4},
+    { FTP_COMMAND_APPE, "APPE", "appe", 4},
+    { FTP_COMMAND_CDUP, "CDUP", "cdup", 4},
+    { FTP_COMMAND_CHMOD, "CHMOD", "chmod", 5},
+    { FTP_COMMAND_CWD, "CWD", "cwd", 3},
+    { FTP_COMMAND_DELE, "DELE", "dele", 4},
+    { FTP_COMMAND_HELP, "HELP", "help", 4},
+    { FTP_COMMAND_IDLE, "IDLE", "idle", 4},
+    { FTP_COMMAND_LIST, "LIST", "list", 4},
+    { FTP_COMMAND_MAIL, "MAIL", "mail", 4},
+    { FTP_COMMAND_MDTM, "MDTM", "mdtm", 4},
+    { FTP_COMMAND_MKD, "MKD", "mkd", 3},
+    { FTP_COMMAND_MLFL, "MLFL", "mlfl", 4},
+    { FTP_COMMAND_MODE, "MODE", "mode", 4},
+    { FTP_COMMAND_MRCP, "MRCP", "mrcp", 4},
+    { FTP_COMMAND_MRSQ, "MRSQ", "mrsq", 4},
+    { FTP_COMMAND_MSAM, "MSAM", "msam", 4},
+    { FTP_COMMAND_MSND, "MSND", "msnd", 4},
+    { FTP_COMMAND_MSOM, "MSOM", "msom", 4},
+    { FTP_COMMAND_NLST, "NLST", "nlst", 4},
+    { FTP_COMMAND_NOOP, "NOOP", "noop", 4},
+    { FTP_COMMAND_PASS, "PASS", "pass", 4},
+    { FTP_COMMAND_PWD, "PWD", "pwd", 3},
+    { FTP_COMMAND_QUIT, "QUIT", "quit", 4},
+    { FTP_COMMAND_REIN, "REIN", "rein", 4},
+    { FTP_COMMAND_REST, "REST", "rest", 4},
+    { FTP_COMMAND_RMD, "RMD", "rmd", 3},
+    { FTP_COMMAND_RNFR, "RNFR", "rnfr", 4},
+    { FTP_COMMAND_RNTO, "RNTO", "rnto", 4},
+    { FTP_COMMAND_SITE, "SITE", "site", 4},
+    { FTP_COMMAND_SIZE, "SIZE", "size", 4},
+    { FTP_COMMAND_SMNT, "SMNT", "smnt", 4},
+    { FTP_COMMAND_STAT, "STAT", "stat", 4},
+    { FTP_COMMAND_STOU, "STOU", "stou", 4},
+    { FTP_COMMAND_STRU, "STRU", "stru", 4},
+    { FTP_COMMAND_SYST, "SYST", "syst", 4},
+    { FTP_COMMAND_TYPE, "TYPE", "type", 4},
+    { FTP_COMMAND_UMASK, "UMASK", "umask", 5},
+    { FTP_COMMAND_USER, "USER", "user", 4},
+    { FTP_COMMAND_UNKNOWN, NULL, NULL, 0}
+};
 uint64_t ftp_config_memcap = 0;
 
 SC_ATOMIC_DECLARE(uint64_t, ftp_memuse);
@@ -190,6 +245,61 @@ static void FTPFree(void *ptr, size_t size)
     FTPDecrMemuse((uint64_t)size);
 }
 
+static FTPString *FTPStringAlloc(void)
+{
+    return FTPCalloc(1, sizeof(FTPString));
+}
+
+static void FTPStringFree(FTPString *str)
+{
+    if (str->str) {
+        FTPFree(str->str, str->len);
+    }
+
+    FTPFree(str, sizeof(FTPString));
+}
+
+static FTPTransaction *FTPTransactionCreate(FtpState *state)
+{
+    SCEnter();
+    FTPTransaction *tx = FTPCalloc(1, sizeof(*tx));
+    if (tx == NULL) {
+        return NULL;
+    }
+
+    TAILQ_INSERT_TAIL(&state->tx_list, tx, next);
+    tx->tx_id = state->tx_cnt++;
+
+    TAILQ_INIT(&tx->response_list);
+
+    SCLogDebug("new transaction %p (state tx cnt %"PRIu64")", tx, state->tx_cnt);
+    return tx;
+}
+
+static void FTPTransactionFree(FTPTransaction *tx)
+{
+    SCEnter();
+
+    if (tx->de_state != NULL) {
+        DetectEngineStateFree(tx->de_state);
+    }
+
+    if (tx->request) {
+        FTPFree(tx->request, tx->request_length);
+    }
+    if (tx->response) {
+        FTPFree(tx->response, tx->response_length);
+    }
+
+    FTPString *str = NULL;
+    while ((str = TAILQ_FIRST(&tx->response_list))) {
+        TAILQ_REMOVE(&tx->response_list, str, next);
+        FTPStringFree(str);
+    }
+
+    SCFree(tx);
+}
+
 static int FTPGetLineForDirection(FtpState *state, FtpLineState *line_state)
 {
     void *ptmp;
@@ -312,49 +422,32 @@ static int FTPGetLine(FtpState *state)
 
 /**
  * \brief This function is called to determine and set which command is being
- * transfered to the ftp server
- * \param ftp_state the ftp state structure for the parser
+ * transferred to the ftp server
  * \param input input line of the command
  * \param len of the command
+ * \param cmd_descriptor when the command has been parsed
  *
  * \retval 1 when the command is parsed, 0 otherwise
  */
-static int FTPParseRequestCommand(void *ftp_state, uint8_t *input,
-                                  uint32_t input_len)
+static int FTPParseRequestCommand(uint8_t *input, uint32_t input_len, const FtpCommand **cmd_descriptor)
 {
     SCEnter();
-    FtpState *fstate = (FtpState *)ftp_state;
-    fstate->command = FTP_COMMAND_UNKNOWN;
-
-    if (input_len >= 4 && SCMemcmpLowercase("port", input, 4) == 0) {
-        fstate->command = FTP_COMMAND_PORT;
-    }
-
-    if (input_len >= 4 && SCMemcmpLowercase("eprt", input, 4) == 0) {
-        fstate->command = FTP_COMMAND_EPRT;
-    }
-
-    if (input_len >= 8 && SCMemcmpLowercase("auth tls", input, 8) == 0) {
-        fstate->command = FTP_COMMAND_AUTH_TLS;
-    }
 
-    if (input_len >= 4 && SCMemcmpLowercase("pasv", input, 4) == 0) {
-        fstate->command = FTP_COMMAND_PASV;
-    }
+    *cmd_descriptor = NULL;
 
-    if (input_len > 5 && SCMemcmpLowercase("retr", input, 4) == 0) {
-        fstate->command = FTP_COMMAND_RETR;
-    }
-
-    if (input_len >= 4 && SCMemcmpLowercase("epsv", input, 4) == 0) {
-        fstate->command = FTP_COMMAND_EPSV;
-    }
+    for (int i = 0; i < FTP_COMMAND_MAX - 1; i++) {
+        if (!FtpCommands[i].command_length) {
+            break;
+        }
+        if (input_len >= FtpCommands[i].command_length &&
+                SCMemcmpLowercase(FtpCommands[i].command_name_lower,
+                                  input, FtpCommands[i].command_length) == 0) {
 
-    if (input_len > 5 && SCMemcmpLowercase("stor", input, 4) == 0) {
-        fstate->command = FTP_COMMAND_STOR;
+            *cmd_descriptor = &FtpCommands[i];
+            return 1;
+        }
     }
-
-    return 1;
+    return 0;
 }
 
 struct FtpTransferCmd {
@@ -378,6 +471,26 @@ static void FtpTransferCmdFree(void *data)
     FTPFree(cmd, sizeof(struct FtpTransferCmd));
 }
 
+static uint32_t CopyCommandLine(uint8_t **dest, uint8_t *src, uint32_t length)
+{
+    if (likely(length)) {
+        uint8_t *where = FTPCalloc(length, sizeof(char));
+        if (unlikely(where == NULL)) {
+            return 0;
+        }
+        memcpy(where, src, length);
+
+        /* Remove trailing newlines/carriage returns */
+        if (isspace((unsigned char)where[length - 1])) {
+            while(length && isspace((unsigned char)where[--length]));
+            where[length] = '\0';
+        }
+        *dest = where;
+    }
+    /* either 0 or actual */
+    return length;
+}
+
 static uint16_t ftp_validate_port(int computed_port_value)
 {
     unsigned int port_val = computed_port_value;
@@ -475,8 +588,26 @@ static int FTPParseRequest(Flow *f, void *ftp_state,
 
     int direction = STREAM_TOSERVER;
     while (FTPGetLine(state) >= 0) {
-        FTPParseRequestCommand(state,
-                               state->current_line, state->current_line_len);
+        const FtpCommand *cmd_descriptor;
+
+        if (!FTPParseRequestCommand(state->current_line, state->current_line_len, &cmd_descriptor)) {
+            state->command = FTP_COMMAND_UNKNOWN;
+            continue;
+        }
+
+        state->command = cmd_descriptor->command;
+
+        FTPTransaction *tx = state->curr_tx;
+        if (tx == NULL || tx->done) {
+            tx = FTPTransactionCreate(state);
+            if (unlikely(tx == NULL))
+                return -1;
+            state->curr_tx = tx;
+        }
+
+        tx->command_descriptor = cmd_descriptor;
+        tx->request_length = CopyCommandLine(&tx->request, state->current_line, state->current_line_len);
+
         switch (state->command) {
             case FTP_COMMAND_EPRT:
                 // fallthrough
@@ -569,6 +700,8 @@ static int FTPParsePassiveResponse(Flow *f, FtpState *state, uint8_t *input, uin
     SCLogDebug("FTP passive mode (v4): dynamic port %"PRIu16"", dyn_port);
     state->active = false;
     state->dyn_port = dyn_port;
+    state->curr_tx->dyn_port = dyn_port;
+    state->curr_tx->active = false;
 
     return 0;
 }
@@ -587,9 +720,26 @@ static int FTPParsePassiveResponseV6(Flow *f, FtpState *state, uint8_t *input, u
     SCLogDebug("FTP passive mode (v6): dynamic port %"PRIu16"", dyn_port);
     state->active = false;
     state->dyn_port = dyn_port;
+    state->curr_tx->dyn_port = dyn_port;
+    state->curr_tx->active = false;
     return 0;
 }
 
+/**
+ * \brief  Handle preliminary replies -- keep tx open
+ * \retval: True for a positive preliminary reply; false otherwise
+ *
+ * 1yz   Positive Preliminary reply
+ *
+ *                The requested action is being initiated; expect another
+ *                               reply before proceeding with a new command
+ */
+static inline bool FTPIsPPR(uint8_t *input, uint32_t input_len)
+{
+    return input_len >= 4 && isdigit(input[0]) && input[0] == '1' &&
+           isdigit(input[1]) && isdigit(input[2]) && isspace(input[3]);
+}
+
 /**
  * \brief This function is called to retrieve a ftp response
  * \param ftp_state the ftp state structure for the parser
@@ -604,6 +754,12 @@ static int FTPParseResponse(Flow *f, void *ftp_state, AppLayerParserState *pstat
                             void *local_data, const uint8_t flags)
 {
     FtpState *state = (FtpState *)ftp_state;
+    FTPTransaction *tx = state->curr_tx;
+    int retcode = 1;
+
+    if (state->command == FTP_COMMAND_UNKNOWN) {
+        return 1;
+    }
 
     if (state->command == FTP_COMMAND_AUTH_TLS) {
         if (input_len >= 4 && SCMemcmp("234 ", input, 4) == 0) {
@@ -614,10 +770,13 @@ static int FTPParseResponse(Flow *f, void *ftp_state, AppLayerParserState *pstat
     if (state->command == FTP_COMMAND_EPRT) {
         uint16_t dyn_port = FTPGetV6PortNumber(state->port_line, state->port_line_len);
         if (dyn_port == 0) {
-            return 0;
+            retcode = 0;
+            goto tx_complete;
         }
         state->dyn_port = dyn_port;
         state->active = true;
+        tx->dyn_port = dyn_port;
+        tx->active = true;
         SCLogDebug("FTP active mode (v6): dynamic port %"PRIu16"", dyn_port);
     }
 
@@ -625,13 +784,17 @@ static int FTPParseResponse(Flow *f, void *ftp_state, AppLayerParserState *pstat
         if ((flags & STREAM_TOCLIENT)) {
             uint16_t dyn_port = FTPGetV4PortNumber(state->port_line, state->port_line_len);
             if (dyn_port == 0) {
-                return 0;
+                retcode = 0;
+                goto tx_complete;
             }
             state->dyn_port = dyn_port;
             state->active = true;
+            tx->dyn_port = state->dyn_port;
+            tx->active = true;
             SCLogDebug("FTP active mode (v4): dynamic port %"PRIu16"", dyn_port);
         }
     }
+
     if (state->command == FTP_COMMAND_PASV) {
         if (input_len >= 4 && SCMemcmp("227 ", input, 4) == 0) {
             FTPParsePassiveResponse(f, ftp_state, input, input_len);
@@ -644,7 +807,18 @@ static int FTPParseResponse(Flow *f, void *ftp_state, AppLayerParserState *pstat
         }
     }
 
-    return 1;
+    if (likely(input_len)) {
+        FTPString *response = FTPStringAlloc();
+        if (likely(response)) {
+            response->len = CopyCommandLine(&response->str, input, input_len);
+            if (response->str)
+                TAILQ_INSERT_TAIL(&tx->response_list, response, next);
+        }
+    }
+
+tx_complete:
+    tx->done = true;
+    return retcode;
 }
 
 #ifdef DEBUG
@@ -655,11 +829,12 @@ static uint64_t ftp_state_memcnt = 0;
 
 static void *FTPStateAlloc(void)
 {
-    void *s = FTPMalloc(sizeof(FtpState));
+    void *s = FTPCalloc(1, sizeof(FtpState));
     if (unlikely(s == NULL))
         return NULL;
 
-    memset(s, 0, sizeof(FtpState));
+    FtpState *ftp_state = (FtpState *) s;
+    TAILQ_INIT(&ftp_state->tx_list);
 
 #ifdef DEBUG
     SCMutexLock(&ftp_state_mem_lock);
@@ -682,8 +857,10 @@ static void FTPStateFree(void *s)
 
     //AppLayerDecoderEventsFreeEvents(&s->decoder_events);
 
-    if (fstate->de_state != NULL) {
-        DetectEngineStateFree(fstate->de_state);
+    FTPTransaction *tx = NULL;
+    while ((tx = TAILQ_FIRST(&fstate->tx_list))) {
+        TAILQ_REMOVE(&fstate->tx_list, tx, next);
+        FTPTransactionFree(tx);
     }
 
     FTPFree(s, sizeof(FtpState));
@@ -697,32 +874,85 @@ static void FTPStateFree(void *s)
 
 static int FTPSetTxDetectState(void *vtx, DetectEngineState *de_state)
 {
-    FtpState *ftp_state = (FtpState *)vtx;
-    ftp_state->de_state = de_state;
+    FTPTransaction *tx = (FTPTransaction *)vtx;
+    tx->de_state = de_state;
     return 0;
 }
 
+static void *FTPGetTx(void *state, uint64_t tx_id)
+{
+    FtpState *ftp_state = (FtpState *)state;
+    if (ftp_state) {
+        FTPTransaction *tx = NULL;
+
+        if (ftp_state->curr_tx == NULL)
+            return NULL;
+        if (ftp_state->curr_tx->tx_id == tx_id)
+            return ftp_state->curr_tx;
+
+        TAILQ_FOREACH(tx, &ftp_state->tx_list, next) {
+            if (tx->tx_id == tx_id)
+                return tx;
+        }
+    }
+    return NULL;
+
+}
+
 static DetectEngineState *FTPGetTxDetectState(void *vtx)
 {
-    FtpState *ftp_state = (FtpState *)vtx;
-    return ftp_state->de_state;
+    FTPTransaction *tx = (FTPTransaction *)vtx;
+    return tx->de_state;
 }
 
-static void FTPStateTransactionFree(void *state, uint64_t tx_id)
+
+static uint64_t FTPGetTxDetectFlags(void *vtx, uint8_t dir)
 {
-    /* do nothing */
+    FTPTransaction *tx = (FTPTransaction *)vtx;
+    if (dir & STREAM_TOSERVER) {
+        return tx->detect_flags_ts;
+    } else {
+        return tx->detect_flags_tc;
+    }
 }
 
-static void *FTPGetTx(void *state, uint64_t tx_id)
+static void FTPSetTxDetectFlags(void *vtx, uint8_t dir, uint64_t flags)
 {
-    FtpState *ftp_state = (FtpState *)state;
-    return ftp_state;
+    FTPTransaction *tx = (FTPTransaction *)vtx;
+    if (dir & STREAM_TOSERVER) {
+        tx->detect_flags_ts = flags;
+    } else {
+        tx->detect_flags_tc = flags;
+    }
+}
+
+static void FTPStateTransactionFree(void *state, uint64_t tx_id)
+{
+    FtpState *ftp_state = state;
+    FTPTransaction *tx = NULL;
+    TAILQ_FOREACH(tx, &ftp_state->tx_list, next) {
+        if (tx_id < tx->tx_id)
+            break;
+        else if (tx_id > tx->tx_id)
+            continue;
+
+        if (tx == ftp_state->curr_tx)
+            ftp_state->curr_tx = NULL;
+        TAILQ_REMOVE(&ftp_state->tx_list, tx, next);
+        FTPTransactionFree(tx);
+        break;
+    }
 }
 
 static uint64_t FTPGetTxCnt(void *state)
 {
-    /* single tx */
-    return 1;
+    uint64_t cnt = 0;
+    FtpState *ftp_state = state;
+    if (ftp_state) {
+        cnt = ftp_state->tx_cnt;
+    }
+    SCLogDebug("returning state %p %"PRIu64, state, cnt);
+    return cnt;
 }
 
 static int FTPGetAlstateProgressCompletionStatus(uint8_t direction)
@@ -730,12 +960,13 @@ static int FTPGetAlstateProgressCompletionStatus(uint8_t direction)
     return FTP_STATE_FINISHED;
 }
 
-static int FTPGetAlstateProgress(void *tx, uint8_t direction)
+static int FTPGetAlstateProgress(void *vtx, uint8_t direction)
 {
-    FtpState *ftp_state = (FtpState *)tx;
+    SCLogDebug("tx %p", vtx);
+    FTPTransaction *tx = vtx;
 
     if (direction == STREAM_TOSERVER &&
-        ftp_state->command == FTP_COMMAND_PORT) {
+        tx->command_descriptor->command == FTP_COMMAND_PORT) {
         return FTP_STATE_PORT_DONE;
     }
 
@@ -868,6 +1099,17 @@ out:
     return ret;
 }
 
+static void FTPStateSetTxLogged(void *state, void *vtx, LoggerId logged)
+{
+    FTPTransaction *tx = vtx;
+    tx->logged = logged;
+}
+
+static LoggerId FTPStateGetTxLogged(void *state, void *vtx)
+{
+    FTPTransaction *tx = vtx;
+    return tx->logged;
+}
 static int FTPDataParseRequest(Flow *f, void *ftp_state,
         AppLayerParserState *pstate,
         uint8_t *input, uint32_t input_len,
@@ -894,12 +1136,12 @@ static uint64_t ftpdata_state_memcnt = 0;
 
 static void *FTPDataStateAlloc(void)
 {
-    void *s = FTPMalloc(sizeof(FtpDataState));
+    void *s = FTPCalloc(1, sizeof(FtpDataState));
     if (unlikely(s == NULL))
         return NULL;
 
-    memset(s, 0, sizeof(FtpDataState));
-    ((FtpDataState *)s)->state = FTPDATA_STATE_IN_PROGRESS;
+    FtpDataState *state = (FtpDataState *) s;
+    state->state = FTPDATA_STATE_IN_PROGRESS;
 
 #ifdef DEBUG
     SCMutexLock(&ftpdata_state_mem_lock);
@@ -1009,7 +1251,12 @@ void RegisterFTPParsers(void)
         AppLayerParserRegisterDetectStateFuncs(IPPROTO_TCP, ALPROTO_FTP,
                 FTPGetTxDetectState, FTPSetTxDetectState);
 
+        AppLayerParserRegisterDetectFlagsFuncs(IPPROTO_TCP, ALPROTO_FTP,
+                                               FTPGetTxDetectFlags, FTPSetTxDetectFlags);
+
         AppLayerParserRegisterGetTx(IPPROTO_TCP, ALPROTO_FTP, FTPGetTx);
+        AppLayerParserRegisterLoggerFuncs(IPPROTO_TCP, ALPROTO_FTP, FTPStateGetTxLogged,
+                                          FTPStateSetTxLogged);
 
         AppLayerParserRegisterGetTxCnt(IPPROTO_TCP, ALPROTO_FTP, FTPGetTxCnt);
 
@@ -1153,7 +1400,7 @@ end:
     return result;
 }
 
-/** \test Send a splitted get request. */
+/** \test Send a split get request. */
 static int FTPParserTest03(void)
 {
     int result = 1;
index 144937065cf83f5b572dad3a9554d8d52e11669e..35e633d674333317031c62621bb26e1ea37eb836 100644 (file)
@@ -79,9 +79,21 @@ typedef enum {
     FTP_COMMAND_TYPE,
     FTP_COMMAND_UMASK,
     FTP_COMMAND_USER,
-    FTP_COMMAND_EPRT
+    FTP_COMMAND_EPRT,
+
+    /* must be last */
+    FTP_COMMAND_MAX
     /** \todo more if missing.. */
 } FtpRequestCommand;
+
+typedef struct FtpCommand_ {
+    FtpRequestCommand command;
+    const char *command_name_upper;
+    const char *command_name_lower;
+    const uint8_t command_length;
+} FtpCommand;
+extern const FtpCommand FtpCommands[FTP_COMMAND_MAX + 1];
+
 typedef uint32_t FtpRequestCommandArgOfs;
 
 typedef uint16_t FtpResponseCode;
@@ -111,6 +123,42 @@ typedef struct FtpLineState_ {
     uint8_t current_line_lf_seen;
 } FtpLineState;
 
+typedef struct FTPString_ {
+    uint8_t *str;
+    uint16_t len;
+    TAILQ_ENTRY(FTPString_) next;
+} FTPString;
+
+typedef struct FTPTransaction_  {
+    /** id of this tx, starting at 0 */
+    uint64_t tx_id;
+
+    uint64_t detect_flags_ts;
+    uint64_t detect_flags_tc;
+
+    /** indicates loggers done logging */
+    uint32_t logged;
+    bool done;
+
+    const FtpCommand *command_descriptor;
+
+    uint8_t direction;
+    uint16_t dyn_port;
+    bool active;
+
+    uint8_t *request;
+    uint32_t request_length;
+
+    /* Handle multiple responses */
+    TAILQ_HEAD(, FTPString_) response_list;
+    uint8_t *response;
+    uint32_t response_length;
+
+    DetectEngineState *de_state;
+
+    TAILQ_ENTRY(FTPTransaction_) next;
+} FTPTransaction;
+
 /** FTP State for app layer parser */
 typedef struct FtpState_ {
     uint8_t *input;
@@ -118,6 +166,10 @@ typedef struct FtpState_ {
     uint8_t direction;
     bool active;
 
+    FTPTransaction *curr_tx;
+    TAILQ_HEAD(, FTPTransaction_) tx_list;  /**< transaction list */
+    uint64_t tx_cnt;
+
     /* --parser details-- */
     /** current line extracted by the parser from the call to FTPGetline() */
     uint8_t *current_line;
@@ -138,7 +190,6 @@ typedef struct FtpState_ {
     /* specifies which loggers are done logging */
     uint32_t logged;
 
-    DetectEngineState *de_state;
 } FtpState;
 
 enum {
diff --git a/src/output-json-ftp.c b/src/output-json-ftp.c
new file mode 100644 (file)
index 0000000..605654f
--- /dev/null
@@ -0,0 +1,247 @@
+/* Copyright (C) 2017 Open Information Security Foundation
+ *
+ * You can copy, redistribute or modify this Program under the terms of
+ * the GNU General Public License version 2 as published by the Free
+ * Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * version 2 along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ */
+
+/**
+ * \file
+ *
+ * \author Jeff Lucovsky <jeff@lucovsky.org>
+ *
+ * Implement JSON/eve logging app-layer FTP.
+ */
+
+
+#include "suricata-common.h"
+#include "debug.h"
+#include "detect.h"
+#include "pkt-var.h"
+#include "conf.h"
+
+#include "threads.h"
+#include "threadvars.h"
+#include "tm-threads.h"
+
+#include "util-unittest.h"
+#include "util-buffer.h"
+#include "util-debug.h"
+#include "util-byte.h"
+
+#include "output.h"
+#include "output-json.h"
+
+#include "app-layer.h"
+#include "app-layer-parser.h"
+
+#include "app-layer-ftp.h"
+#include "output-json-ftp.h"
+
+#ifdef HAVE_LIBJANSSON
+
+typedef struct LogFTPFileCtx_ {
+    LogFileCtx *file_ctx;
+    OutputJsonCommonSettings cfg;
+} LogFTPFileCtx;
+
+typedef struct LogFTPLogThread_ {
+    LogFTPFileCtx *ftplog_ctx;
+    uint32_t            count;
+    MemBuffer          *buffer;
+} LogFTPLogThread;
+
+static void JsonFTPLogJSON(json_t *tjs, Flow *f, FTPTransaction *tx)
+{
+    json_t *cjs = NULL;
+    if (f->alproto == ALPROTO_FTPDATA) {
+        cjs = JsonFTPDataAddMetadata(f);
+    } else if (tx->command_descriptor->command != FTP_COMMAND_UNKNOWN) {
+        cjs = json_object();
+        if (cjs) {
+            FTPString *response;
+            json_object_set_new(cjs, "command", json_string(tx->command_descriptor->command_name_upper));
+            uint32_t min_length = tx->command_descriptor->command_length + 1; /* command + space */
+            if (tx->request_length >= min_length) {
+                json_object_set_new(cjs, "command_data",
+                                    JsonAddStringN((const char *)tx->request + min_length,
+                                                   tx->request_length - min_length));
+            }
+            if (!TAILQ_EMPTY(&tx->response_list)) {
+                json_t *js_resplist = json_array();
+                if (likely(js_resplist != NULL)) {
+                    json_t *resp_code = NULL;
+                    TAILQ_FOREACH(response, &tx->response_list, next) {
+                        if (!resp_code && response->len >= 3)  {
+                            /* the first completion codes with multiple response lines is definitive */
+                            resp_code = JsonAddStringN((const char *)response->str, 3);
+                        }
+                        /* move past 3 character completion code */
+                        if (response->len > 4) {
+                            json_array_append_new(js_resplist,
+                                                  JsonAddStringN((const char *)response->str + 4,
+                                                                 response->len - 4));
+                        }
+                    }
+
+                    json_object_set_new(cjs, "reply", js_resplist);
+
+                    if (resp_code) {
+                        json_object_set_new(cjs, "completion_code", resp_code);
+                    }
+                }
+            }
+            if (tx->dyn_port) {
+                json_object_set_new(cjs, "dynamic_port", json_integer(tx->dyn_port));
+            }
+            if (tx->command_descriptor->command == FTP_COMMAND_PORT ||
+                tx->command_descriptor->command == FTP_COMMAND_EPRT) {
+                json_object_set_new(cjs, "mode",
+                        json_string((char*)(tx->active ? "active" : "passive")));
+            }
+        }
+    }
+
+    if (cjs) {
+        json_object_set_new(tjs, f->alproto == ALPROTO_FTP ? "ftp" : "ftp_data", cjs);
+    }
+}
+
+static int JsonFTPLogger(ThreadVars *tv, void *thread_data,
+    const Packet *p, Flow *f, void *state, void *vtx, uint64_t tx_id)
+{
+    SCEnter();
+
+    FTPTransaction *tx = vtx;
+    LogFTPLogThread *thread = thread_data;
+    LogFTPFileCtx *ftp_ctx = thread->ftplog_ctx;
+
+    json_t *js = CreateJSONHeaderWithTxId(p, LOG_DIR_FLOW,
+                                          f->alproto == ALPROTO_FTP ? "ftp" : "ftp_data",
+                                          tx_id);
+    if (likely(js)) {
+        JsonAddCommonOptions(&ftp_ctx->cfg, p, f, js);
+        JsonFTPLogJSON(js, f, tx);
+
+        MemBufferReset(thread->buffer);
+        OutputJSONBuffer(js, thread->ftplog_ctx->file_ctx, &thread->buffer);
+
+        json_object_clear(js);
+        json_decref(js);
+    }
+    return TM_ECODE_OK;
+}
+
+static void OutputFTPLogDeInitCtxSub(OutputCtx *output_ctx)
+{
+    LogFTPFileCtx *ftplog_ctx = (LogFTPFileCtx *)output_ctx->data;
+    SCFree(ftplog_ctx);
+    SCFree(output_ctx);
+}
+
+
+static OutputInitResult OutputFTPLogInitSub(ConfNode *conf,
+    OutputCtx *parent_ctx)
+{
+    OutputInitResult result = { NULL, false };
+    OutputJsonCtx *ajt = parent_ctx->data;
+
+    LogFTPFileCtx *ftplog_ctx = SCCalloc(1, sizeof(*ftplog_ctx));
+    if (unlikely(ftplog_ctx == NULL)) {
+        return result;
+    }
+    ftplog_ctx->file_ctx = ajt->file_ctx;
+    ftplog_ctx->cfg = ajt->cfg;
+
+    OutputCtx *output_ctx = SCCalloc(1, sizeof(*output_ctx));
+    if (unlikely(output_ctx == NULL)) {
+        SCFree(ftplog_ctx);
+        return result;
+    }
+    output_ctx->data = ftplog_ctx;
+    output_ctx->DeInit = OutputFTPLogDeInitCtxSub;
+
+    SCLogDebug("FTP log sub-module initialized.");
+
+    AppLayerParserRegisterLogger(IPPROTO_TCP, ALPROTO_FTP);
+    AppLayerParserRegisterLogger(IPPROTO_TCP, ALPROTO_FTPDATA);
+
+    result.ctx = output_ctx;
+    result.ok = true;
+    return result;
+}
+
+#define OUTPUT_BUFFER_SIZE 65535
+static TmEcode JsonFTPLogThreadInit(ThreadVars *t, const void *initdata, void **data)
+{
+    LogFTPLogThread *thread = SCCalloc(1, sizeof(*thread));
+    if (unlikely(thread == NULL)) {
+        return TM_ECODE_FAILED;
+    }
+
+    if (initdata == NULL) {
+        SCLogDebug("Error getting context for EveLogFTP.  \"initdata\" is NULL.");
+        SCFree(thread);
+        return TM_ECODE_FAILED;
+    }
+
+    thread->buffer = MemBufferCreateNew(OUTPUT_BUFFER_SIZE);
+    if (unlikely(thread->buffer == NULL)) {
+        SCFree(thread);
+        return TM_ECODE_FAILED;
+    }
+
+    thread->ftplog_ctx = ((OutputCtx *)initdata)->data;
+    *data = (void *)thread;
+
+    return TM_ECODE_OK;
+}
+
+static TmEcode JsonFTPLogThreadDeinit(ThreadVars *t, void *data)
+{
+    LogFTPLogThread *thread = (LogFTPLogThread *)data;
+    if (thread == NULL) {
+        return TM_ECODE_OK;
+    }
+    if (thread->buffer != NULL) {
+        MemBufferFree(thread->buffer);
+    }
+    /* clear memory */
+    memset(thread, 0, sizeof(LogFTPLogThread));
+    SCFree(thread);
+    return TM_ECODE_OK;
+}
+
+void JsonFTPLogRegister(void)
+{
+    /* Register as an eve sub-module. */
+    OutputRegisterTxSubModule(LOGGER_JSON_FTP, "eve-log", "JsonFTPLog",
+                              "eve-log.ftp", OutputFTPLogInitSub,
+                              ALPROTO_FTP, JsonFTPLogger,
+                              JsonFTPLogThreadInit, JsonFTPLogThreadDeinit,
+                              NULL);
+    OutputRegisterTxSubModule(LOGGER_JSON_FTP, "eve-log", "JsonFTPLog",
+                              "eve-log.ftp", OutputFTPLogInitSub,
+                              ALPROTO_FTPDATA, JsonFTPLogger,
+                              JsonFTPLogThreadInit, JsonFTPLogThreadDeinit,
+                              NULL);
+
+    SCLogDebug("FTP JSON logger registered.");
+}
+#else /* HAVE_LIBJANSSON */
+
+void JsonFTPLogRegister(void)
+{
+}
+
+#endif /* HAVE_LIBJANSSON */
diff --git a/src/output-json-ftp.h b/src/output-json-ftp.h
new file mode 100644 (file)
index 0000000..acba553
--- /dev/null
@@ -0,0 +1,29 @@
+/* Copyright (C) 2019 Open Information Security Foundation
+ *
+ * You can copy, redistribute or modify this Program under the terms of
+ * the GNU General Public License version 2 as published by the Free
+ * Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * version 2 along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ */
+
+/**
+ * \file
+ *
+ * \author Jeff Lucovsky <jeff@lucovsky.org>
+ */
+
+#ifndef __OUTPUT_JSON_FTP_H__
+#define __OUTPUT_JSON_FTP_H__
+
+void JsonFTPLogRegister(void);
+
+#endif /* __OUTPUT_JSON_FTP_H__ */
index a9355cd2cee2e17396722417ae1dfadd8418ba3f..0efe67044038f605bc0d5d01d5aff9e352f8f212 100644 (file)
@@ -148,6 +148,22 @@ json_t *SCJsonString(const char *val)
 /* Default Sensor ID value */
 static int64_t sensor_id = -1; /* -1 = not defined */
 
+/**
+ * \brief Create a JSON string from a character sequence
+ *
+ * \param Pointer to character sequence
+ * \param Number of characters to use from the sequence
+ * \retval JSON object for the character sequence
+ */
+json_t *JsonAddStringN(const char *string, size_t size)
+{
+    char tmpbuf[size + 1];
+
+    memcpy(tmpbuf, string, size);
+    tmpbuf[size] = '\0';
+
+    return SCJsonString(tmpbuf);
+}
 static void JsonAddPacketvars(const Packet *p, json_t *js_vars)
 {
     if (p == NULL || p->pktvar == NULL) {
index 73eddf88ae0a46ab178979053ebb4e43374b8b63..a7d996e4277aa854fd24321aa1957dab0024c8f6 100644 (file)
@@ -88,6 +88,7 @@ typedef struct OutputJsonThreadCtx_ {
 
 json_t *SCJsonBool(int val);
 json_t *SCJsonString(const char *val);
+json_t *JsonAddStringN(const char *string, size_t size);
 void SCJsonDecref(json_t *js);
 
 void JsonAddCommonOptions(const OutputJsonCommonSettings *cfg,
index a735fd74cbccabcbc34ad88c9d3eb1613ac82b96..1c27a758cfc29971c837a40a02ffdd6c834bf184 100644 (file)
@@ -68,6 +68,7 @@
 #include "log-stats.h"
 #include "output-json.h"
 #include "output-json-nfs.h"
+#include "output-json-ftp.h"
 #include "output-json-tftp.h"
 #include "output-json-smb.h"
 #include "output-json-ikev2.h"
@@ -1096,6 +1097,8 @@ void OutputRegisterLoggers(void)
     JsonNFSLogRegister();
     /* TFTP JSON logger. */
     JsonTFTPLogRegister();
+    /* FTP JSON logger. */
+    JsonFTPLogRegister();
     /* SMB JSON logger. */
     JsonSMBLogRegister();
     /* IKEv2 JSON logger. */
index dc05e19d0ae7267857da8ec0c135de574fb7a352..31764c5ae060eb55249a32ae050bade7a8c96b32 100644 (file)
@@ -431,6 +431,7 @@ typedef enum {
     LOGGER_JSON_TLS,
     LOGGER_JSON_NFS,
     LOGGER_JSON_TFTP,
+    LOGGER_JSON_FTP,
     LOGGER_JSON_DNP3_TS,
     LOGGER_JSON_DNP3_TC,
     LOGGER_JSON_SSH,
index a77519271be5cc3b327d9d1f027e4ca53f06063d..592659d9467b195132de93f7959f97da0fb3d455 100644 (file)
@@ -106,6 +106,7 @@ const char * SCErrorToString(SCError err)
         CASE_CODE (SC_ERR_DEBUG_LOG_GENERIC);
         CASE_CODE (SC_ERR_UNIFIED_LOG_GENERIC);
         CASE_CODE (SC_ERR_HTTP_LOG_GENERIC);
+        CASE_CODE (SC_ERR_FTP_LOG_GENERIC);
         CASE_CODE (SC_ERR_UNIFIED_ALERT_GENERIC);
         CASE_CODE (SC_ERR_UNIFIED2_ALERT_GENERIC);
         CASE_CODE (SC_ERR_FWRITE);
index 76debeca71ef8119b81d5d0c1258e8efbfcb68ce..ee8363857e75b8991ea8e44ae0e2f603fc52c40a 100644 (file)
@@ -350,6 +350,7 @@ typedef enum {
     SC_WARN_DEFAULT_WILL_CHANGE,
     SC_WARN_EVE_MISSING_EVENTS,
     SC_ERR_PLEDGE_FAILED,
+    SC_ERR_FTP_LOG_GENERIC,
 
     SC_ERR_MAX,
 } SCError;
index 7262b3a5b145be9726355c3dc0f7ca267eb86389..2a11d77805eeed54de4a5411707bf17cf6587ea9 100644 (file)
@@ -1313,6 +1313,7 @@ const char * PacketProfileLoggertIdToString(LoggerId id)
         CASE_CODE (LOGGER_JSON_DHCP);
         CASE_CODE (LOGGER_JSON_KRB5);
         CASE_CODE (LOGGER_JSON_IKEV2);
+        CASE_CODE (LOGGER_JSON_FTP);
         CASE_CODE (LOGGER_JSON_TFTP);
         CASE_CODE (LOGGER_JSON_SMTP);
         CASE_CODE (LOGGER_JSON_SNMP);