src/app-layer-htp.h

   1 /* Copyright (C) 2007-2020 Open Information Security Foundation
   2  *
   3  * You can copy, redistribute or modify this Program under the terms of
   4  * the GNU General Public License version 2 as published by the Free
   5  * Software Foundation.
   6  *
   7  * This program is distributed in the hope that it will be useful,
   8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
   9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  10  * GNU General Public License for more details.
  11  *
  12  * You should have received a copy of the GNU General Public License
  13  * version 2 along with this program; if not, write to the Free Software
  14  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  15  * 02110-1301, USA.
  16  */
  17
  18 /**
  19  * \defgroup httplayer HTTP layer support
  20  *
  21  * @{
  22  */
  23
  24 /**
  25  * \file
  26  *
  27  * \author Gurvinder Singh <gurvindersinghdahiya@gmail.com>
  28  * \author Pablo Rincon <pablo.rincon.crespo@gmail.com>
  29  *
  30  * This file provides a HTTP protocol support for the engine using HTP library.
  31  */
  32
  33 #ifndef __APP_LAYER_HTP_H__
  34 #define __APP_LAYER_HTP_H__
  35
  36 #include "util-radix-tree.h"
  37 #include "util-file.h"
  38 #include "app-layer-htp-mem.h"
  39 #include "detect-engine-state.h"
  40 #include "util-streaming-buffer.h"
  41 #include "app-layer-htp-range.h"
  42 #include "rust.h"
  43
  44 #include <htp/htp.h>
  45
  46 /* default request body limit */
  47 #define HTP_CONFIG_DEFAULT_REQUEST_BODY_LIMIT           4096U
  48 #define HTP_CONFIG_DEFAULT_RESPONSE_BODY_LIMIT          4096U
  49 #define HTP_CONFIG_DEFAULT_REQUEST_INSPECT_MIN_SIZE     32768U
  50 #define HTP_CONFIG_DEFAULT_REQUEST_INSPECT_WINDOW       4096U
  51 #define HTP_CONFIG_DEFAULT_RESPONSE_INSPECT_MIN_SIZE    32768U
  52 #define HTP_CONFIG_DEFAULT_RESPONSE_INSPECT_WINDOW      4096U
  53 #define HTP_CONFIG_DEFAULT_FIELD_LIMIT_SOFT             9000U
  54 #define HTP_CONFIG_DEFAULT_FIELD_LIMIT_HARD             18000U
  55
  56 #define HTP_CONFIG_DEFAULT_LZMA_LAYERS 0U
  57 /* default libhtp lzma limit, taken from libhtp. */
  58 #define HTP_CONFIG_DEFAULT_LZMA_MEMLIMIT                1048576U
  59 #define HTP_CONFIG_DEFAULT_COMPRESSION_BOMB_LIMIT       1048576U
  60 // 100000 usec is 0.1 sec
  61 #define HTP_CONFIG_DEFAULT_COMPRESSION_TIME_LIMIT 100000
  62
  63 #define HTP_CONFIG_DEFAULT_RANDOMIZE                    1
  64 #define HTP_CONFIG_DEFAULT_RANDOMIZE_RANGE              10
  65
  66 /** a boundary should be smaller in size */
  67 #define HTP_BOUNDARY_MAX                            200U
  68
  69 // 0x0001 not used
  70 #define HTP_FLAG_STATE_CLOSED_TS    0x0002    /**< Flag to indicate that HTTP
  71                                              connection is closed */
  72 #define HTP_FLAG_STATE_CLOSED_TC    0x0004    /**< Flag to indicate that HTTP
  73                                              connection is closed */
  74 #define HTP_FLAG_STORE_FILES_TS     0x0040
  75 #define HTP_FLAG_STORE_FILES_TC     0x0080
  76 #define HTP_FLAG_STORE_FILES_TX_TS  0x0100
  77 #define HTP_FLAG_STORE_FILES_TX_TC  0x0200
  78
  79 enum {
  80     HTP_BODY_REQUEST_NONE = 0,
  81     HTP_BODY_REQUEST_MULTIPART, /* POST, MP */
  82     HTP_BODY_REQUEST_POST,      /* POST, no MP */
  83     HTP_BODY_REQUEST_PUT,
  84 };
  85
  86 enum {
  87     /* libhtp errors/warnings */
  88     HTTP_DECODER_EVENT_UNKNOWN_ERROR,
  89     HTTP_DECODER_EVENT_GZIP_DECOMPRESSION_FAILED,
  90     HTTP_DECODER_EVENT_REQUEST_FIELD_MISSING_COLON,
  91     HTTP_DECODER_EVENT_RESPONSE_FIELD_MISSING_COLON,
  92     HTTP_DECODER_EVENT_INVALID_REQUEST_CHUNK_LEN,
  93     HTTP_DECODER_EVENT_INVALID_RESPONSE_CHUNK_LEN,
  94     HTTP_DECODER_EVENT_INVALID_TRANSFER_ENCODING_VALUE_IN_REQUEST,
  95     HTTP_DECODER_EVENT_INVALID_TRANSFER_ENCODING_VALUE_IN_RESPONSE,
  96     HTTP_DECODER_EVENT_INVALID_CONTENT_LENGTH_FIELD_IN_REQUEST,
  97     HTTP_DECODER_EVENT_INVALID_CONTENT_LENGTH_FIELD_IN_RESPONSE,
  98     HTTP_DECODER_EVENT_DUPLICATE_CONTENT_LENGTH_FIELD_IN_REQUEST,
  99     HTTP_DECODER_EVENT_DUPLICATE_CONTENT_LENGTH_FIELD_IN_RESPONSE,
 100     HTTP_DECODER_EVENT_100_CONTINUE_ALREADY_SEEN,
 101     HTTP_DECODER_EVENT_UNABLE_TO_MATCH_RESPONSE_TO_REQUEST,
 102     HTTP_DECODER_EVENT_INVALID_SERVER_PORT_IN_REQUEST,
 103     HTTP_DECODER_EVENT_INVALID_AUTHORITY_PORT,
 104     HTTP_DECODER_EVENT_REQUEST_HEADER_INVALID,
 105     HTTP_DECODER_EVENT_RESPONSE_HEADER_INVALID,
 106     HTTP_DECODER_EVENT_MISSING_HOST_HEADER,
 107     HTTP_DECODER_EVENT_HOST_HEADER_AMBIGUOUS,
 108     HTTP_DECODER_EVENT_INVALID_REQUEST_FIELD_FOLDING,
 109     HTTP_DECODER_EVENT_INVALID_RESPONSE_FIELD_FOLDING,
 110     HTTP_DECODER_EVENT_REQUEST_FIELD_TOO_LONG,
 111     HTTP_DECODER_EVENT_RESPONSE_FIELD_TOO_LONG,
 112     HTTP_DECODER_EVENT_REQUEST_SERVER_PORT_TCP_PORT_MISMATCH,
 113     HTTP_DECODER_EVENT_URI_HOST_INVALID,
 114     HTTP_DECODER_EVENT_HEADER_HOST_INVALID,
 115     HTTP_DECODER_EVENT_METHOD_DELIM_NON_COMPLIANT,
 116     HTTP_DECODER_EVENT_URI_DELIM_NON_COMPLIANT,
 117     HTTP_DECODER_EVENT_REQUEST_LINE_LEADING_WHITESPACE,
 118     HTTP_DECODER_EVENT_TOO_MANY_ENCODING_LAYERS,
 119     HTTP_DECODER_EVENT_ABNORMAL_CE_HEADER,
 120     HTTP_DECODER_EVENT_AUTH_UNRECOGNIZED,
 121     HTTP_DECODER_EVENT_REQUEST_HEADER_REPETITION,
 122     HTTP_DECODER_EVENT_RESPONSE_HEADER_REPETITION,
 123     HTTP_DECODER_EVENT_RESPONSE_MULTIPART_BYTERANGES,
 124     HTTP_DECODER_EVENT_RESPONSE_ABNORMAL_TRANSFER_ENCODING,
 125     HTTP_DECODER_EVENT_RESPONSE_CHUNKED_OLD_PROTO,
 126     HTTP_DECODER_EVENT_RESPONSE_INVALID_PROTOCOL,
 127     HTTP_DECODER_EVENT_RESPONSE_INVALID_STATUS,
 128     HTTP_DECODER_EVENT_REQUEST_LINE_INCOMPLETE,
 129     HTTP_DECODER_EVENT_DOUBLE_ENCODED_URI,
 130     HTTP_DECODER_EVENT_REQUEST_LINE_INVALID,
 131     HTTP_DECODER_EVENT_REQUEST_BODY_UNEXPECTED,
 132
 133     HTTP_DECODER_EVENT_LZMA_MEMLIMIT_REACHED,
 134     HTTP_DECODER_EVENT_COMPRESSION_BOMB,
 135
 136     HTTP_DECODER_EVENT_RANGE_INVALID,
 137
 138     /* suricata errors/warnings */
 139     HTTP_DECODER_EVENT_MULTIPART_GENERIC_ERROR,
 140     HTTP_DECODER_EVENT_MULTIPART_NO_FILEDATA,
 141     HTTP_DECODER_EVENT_MULTIPART_INVALID_HEADER,
 142
 143     HTTP_DECODER_EVENT_TOO_MANY_WARNINGS,
 144 };
 145
 146 typedef enum HtpSwfCompressType_ {
 147     HTTP_SWF_COMPRESSION_NONE = 0,
 148     HTTP_SWF_COMPRESSION_ZLIB,
 149     HTTP_SWF_COMPRESSION_LZMA,
 150     HTTP_SWF_COMPRESSION_BOTH,
 151 } HtpSwfCompressType;
 152
 153 typedef struct HTPCfgDir_ {
 154     uint32_t body_limit;
 155     uint32_t inspect_min_size;
 156     uint32_t inspect_window;
 157     StreamingBufferConfig sbcfg;
 158 } HTPCfgDir;
 159
 160 /** Need a linked list in order to keep track of these */
 161 typedef struct HTPCfgRec_ {
 162     htp_cfg_t           *cfg;
 163     struct HTPCfgRec_   *next;
 164
 165     int                 uri_include_all; /**< use all info in uri (bool) */
 166
 167     /** max size of the client body we inspect */
 168     int                 randomize;
 169     int                 randomize_range;
 170     int                 http_body_inline;
 171
 172     int                 swf_decompression_enabled;
 173     HtpSwfCompressType  swf_compression_type;
 174     uint32_t            swf_decompress_depth;
 175     uint32_t            swf_compress_depth;
 176
 177     HTPCfgDir request;
 178     HTPCfgDir response;
 179 } HTPCfgRec;
 180
 181 /** Struct used to hold chunks of a body on a request */
 182 struct HtpBodyChunk_ {
 183     struct HtpBodyChunk_ *next; /**< Pointer to the next chunk */
 184     int logged;
 185     StreamingBufferSegment sbseg;
 186 } __attribute__((__packed__));
 187 typedef struct HtpBodyChunk_ HtpBodyChunk;
 188
 189 /** Struct used to hold all the chunks of a body on a request */
 190 typedef struct HtpBody_ {
 191     HtpBodyChunk *first; /**< Pointer to the first chunk */
 192     HtpBodyChunk *last;  /**< Pointer to the last chunk */
 193
 194     StreamingBuffer *sb;
 195
 196     /* Holds the length of the htp request body seen so far */
 197     uint64_t content_len_so_far;
 198     /* parser tracker */
 199     uint64_t body_parsed;
 200     /* inspection tracker */
 201     uint64_t body_inspected;
 202 } HtpBody;
 203
 204 #define HTP_CONTENTTYPE_SET     BIT_U8(0)    /**< We have the content type */
 205 #define HTP_BOUNDARY_SET        BIT_U8(1)    /**< We have a boundary string */
 206 #define HTP_BOUNDARY_OPEN       BIT_U8(2)    /**< We have a boundary string */
 207 #define HTP_FILENAME_SET        BIT_U8(3)    /**< filename is registered in the flow */
 208 #define HTP_DONTSTORE           BIT_U8(4)    /**< not storing this file */
 209 #define HTP_STREAM_DEPTH_SET    BIT_U8(5)    /**< stream-depth is set */
 210
 211 /** Now the Body Chunks will be stored per transaction, at
 212   * the tx user data */
 213 typedef struct HtpTxUserData_ {
 214     /* Body of the request (if any) */
 215     uint8_t request_body_init;
 216     uint8_t response_body_init;
 217
 218     uint8_t request_has_trailers;
 219     uint8_t response_has_trailers;
 220
 221     HtpBody request_body;
 222     HtpBody response_body;
 223
 224     bstr *request_uri_normalized;
 225
 226     uint8_t *request_headers_raw;
 227     uint8_t *response_headers_raw;
 228     uint32_t request_headers_raw_len;
 229     uint32_t response_headers_raw_len;
 230
 231     /** Holds the boundary identification string if any (used on
 232      *  multipart/form-data only)
 233      */
 234     uint8_t *boundary;
 235     uint8_t boundary_len;
 236
 237     uint8_t tsflags;
 238     uint8_t tcflags;
 239
 240     uint8_t request_body_type;
 241
 242     AppLayerTxData tx_data;
 243 } HtpTxUserData;
 244
 245 typedef struct HtpState_ {
 246     /* Connection parser structure for each connection */
 247     htp_connp_t *connp;
 248     /* Connection structure for each connection */
 249     htp_conn_t *conn;
 250     Flow *f;                /**< Needed to retrieve the original flow when using HTPLib callbacks */
 251     uint64_t transaction_cnt;
 252     uint64_t store_tx_id;
 253     FileContainer *files_ts;
 254     FileContainer *files_tc;
 255     const struct HTPCfgRec_ *cfg;
 256     uint16_t flags;
 257     uint16_t events;
 258     uint16_t htp_messages_offset; /**< offset into conn->messages list */
 259     uint32_t file_track_id;             /**< used to assign file track ids to files */
 260     HttpRangeContainerBlock *file_range; /**< used to assign track ids to range file */
 261     uint64_t last_request_data_stamp;
 262     uint64_t last_response_data_stamp;
 263 } HtpState;
 264
 265 /** part of the engine needs the request body (e.g. http_client_body keyword) */
 266 #define HTP_REQUIRE_REQUEST_BODY        (1 << 0)
 267 /** part of the engine needs the request body multipart header (e.g. filename
 268  *  and / or fileext keywords) */
 269 #define HTP_REQUIRE_REQUEST_MULTIPART   (1 << 1)
 270 /** part of the engine needs the request file (e.g. log-file module) */
 271 #define HTP_REQUIRE_REQUEST_FILE        (1 << 2)
 272 /** part of the engine needs the request body (e.g. file_data keyword) */
 273 #define HTP_REQUIRE_RESPONSE_BODY       (1 << 3)
 274
 275 SC_ATOMIC_EXTERN(uint32_t, htp_config_flags);
 276
 277 void RegisterHTPParsers(void);
 278 void HTPAtExitPrintStats(void);
 279 void HTPFreeConfig(void);
 280
 281 void HtpBodyPrint(HtpBody *);
 282 void HtpBodyFree(HtpBody *);
 283 /* To free the state from unittests using app-layer-htp */
 284 void HTPStateFree(void *);
 285 void AppLayerHtpEnableRequestBodyCallback(void);
 286 void AppLayerHtpEnableResponseBodyCallback(void);
 287 void AppLayerHtpNeedFileInspection(void);
 288 void AppLayerHtpPrintStats(void);
 289
 290 void HTPConfigure(void);
 291
 292 void HtpConfigCreateBackup(void);
 293 void HtpConfigRestoreBackup(void);
 294
 295 void *HtpGetTxForH2(void *);
 296
 297 #endif  /* __APP_LAYER_HTP_H__ */
 298
 299 /**
 300  * @}
 301  */