From: Marcin Siodelski Date: Wed, 20 Dec 2017 13:23:42 +0000 (+0100) Subject: [5451] Implemented HTTP response parser. X-Git-Tag: trac5457_base~4^2~8 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=d63def48e29ed7f51993f348aa5ba2e91bc362e1;p=thirdparty%2Fkea.git [5451] Implemented HTTP response parser. --- diff --git a/src/lib/http/Makefile.am b/src/lib/http/Makefile.am index 42937507f5..ebdaa25f13 100644 --- a/src/lib/http/Makefile.am +++ b/src/lib/http/Makefile.am @@ -39,6 +39,7 @@ libkea_http_la_SOURCES += request.cc request.h libkea_http_la_SOURCES += request_context.h libkea_http_la_SOURCES += request_parser.cc request_parser.h libkea_http_la_SOURCES += response.cc response.h +libkea_http_la_SOURCES += response_parser.cc response_parser.h libkea_http_la_SOURCES += response_context.h libkea_http_la_SOURCES += response_creator.cc response_creator.h libkea_http_la_SOURCES += response_creator_factory.h diff --git a/src/lib/http/http_message_parser_base.cc b/src/lib/http/http_message_parser_base.cc index 25178ecb63..a7c01bf85f 100644 --- a/src/lib/http/http_message_parser_base.cc +++ b/src/lib/http/http_message_parser_base.cc @@ -186,7 +186,7 @@ HttpMessageParserBase::parseEndedHandler() { transition(END_ST, END_EVT); break; case HTTP_PARSE_FAILED_EVT: - abortModel("HTTP request parsing failed"); + abortModel("HTTP message parsing failed"); break; default: diff --git a/src/lib/http/post_request_json.h b/src/lib/http/post_request_json.h index 57c6d922b1..3a6f37ae0d 100644 --- a/src/lib/http/post_request_json.h +++ b/src/lib/http/post_request_json.h @@ -80,6 +80,8 @@ public: protected: + /// @brief Interprets body as JSON, which can be later retrieved using + /// data element objects. void parseBodyAsJson(); /// @brief Pointer to the parsed JSON body. diff --git a/src/lib/http/request_parser.cc b/src/lib/http/request_parser.cc index dd5e095d76..fe9716646e 100644 --- a/src/lib/http/request_parser.cc +++ b/src/lib/http/request_parser.cc @@ -6,7 +6,6 @@ #include #include -#include #include using namespace isc::util; diff --git a/src/lib/http/request_parser.h b/src/lib/http/request_parser.h index f65572de5a..4cd278e181 100644 --- a/src/lib/http/request_parser.h +++ b/src/lib/http/request_parser.h @@ -142,7 +142,7 @@ public: /// @param request Reference to the @ref HttpRequest object or its /// derivation that should be used to validate the parsed request and /// to be used as a container for the parsed request. - HttpRequestParser(HttpRequest& request); + explicit HttpRequestParser(HttpRequest& request); /// @brief Initialize the state model for parsing. /// @@ -240,6 +240,8 @@ private: /// @brief Handler for HTTP_BODY_ST. void bodyHandler(); + //@} + /// @brief Reference to the request object specified in the constructor. HttpRequest& request_; diff --git a/src/lib/http/response.cc b/src/lib/http/response.cc index 292d8f8683..13fd4a5b2c 100644 --- a/src/lib/http/response.cc +++ b/src/lib/http/response.cc @@ -54,7 +54,7 @@ HttpResponse::HttpResponse(const HttpVersion& version, : HttpMessage(OUTBOUND), context_(new HttpResponseContext()) { context_->http_version_major_ = version.major_; context_->http_version_minor_ = version.minor_; - context_->status_code_ = static_cast(status_code); + context_->status_code_ = static_cast(status_code); if (generic_body.set_) { // This currently does nothing, but it is useful to have it here as @@ -145,6 +145,12 @@ HttpResponse::getStatusCode() const { return (static_cast(context_->status_code_)); } +std::string +HttpResponse::getStatusPhrase() const { + checkCreated(); + return (context_->phrase_); +} + std::string HttpResponse::getBody() const { checkFinalized(); diff --git a/src/lib/http/response.h b/src/lib/http/response.h index eabf476489..921af82367 100644 --- a/src/lib/http/response.h +++ b/src/lib/http/response.h @@ -7,11 +7,13 @@ #ifndef HTTP_RESPONSE_H #define HTTP_RESPONSE_H +#include #include #include #include #include #include +#include #include namespace isc { @@ -150,9 +152,29 @@ public: /// @brief Returns HTTP status code. HttpStatusCode getStatusCode() const; + /// @brief Returns HTTP status phrase. + std::string getStatusPhrase() const; + /// @brief Returns HTTP response body as string. virtual std::string getBody() const; + /// @brief Retrieves JSON body. + /// + /// @return Pointer to the root element of the JSON structure. + /// @throw HttpResponseJsonError if an error occurred. + data::ConstElementPtr getBodyAsJson() const; + + /// @brief Retrieves a single JSON element. + /// + /// The element must be at top level of the JSON structure. + /// + /// @param element_name Element name. + /// + /// @return Pointer to the specified element or NULL if such element + /// doesn't exist. + /// @throw HttpResponseJsonError if an error occurred. + data::ConstElementPtr getJsonElement(const std::string& element_name) const; + /// @brief Checks if the status code indicates client error. /// /// @param status_code HTTP status code. @@ -220,6 +242,8 @@ private: /// generated. void setGenericBody(const HttpStatusCode& /*status_code*/) { }; +protected: + /// @brief Pointer to the @ref HttpResponseContext holding parsed /// data. HttpResponseContextPtr context_; diff --git a/src/lib/http/response_context.h b/src/lib/http/response_context.h index 5154dc4c6d..a8214776eb 100644 --- a/src/lib/http/response_context.h +++ b/src/lib/http/response_context.h @@ -26,7 +26,9 @@ struct HttpResponseContext { /// @brief HTTP minor version number. unsigned int http_version_minor_; /// @brief HTTP status code. - uint16_t status_code_; + unsigned int status_code_; + /// @brief HTTP status phrase. + std::string phrase_; /// @brief Collection of HTTP headers. std::vector headers_; /// @brief HTTP request body. diff --git a/src/lib/http/response_json.cc b/src/lib/http/response_json.cc index 231d5beabc..aa29d49b1f 100644 --- a/src/lib/http/response_json.cc +++ b/src/lib/http/response_json.cc @@ -5,6 +5,7 @@ // file, You can obtain one at http://mozilla.org/MPL/2.0/. #include +#include using namespace isc::data; @@ -48,6 +49,29 @@ HttpResponseJson::setGenericBody(const HttpStatusCode& status_code) { } } +void +HttpResponseJson::finalize() { + if (!created_) { + create(); + } + + // Parse JSON body and store. + parseBodyAsJson(); + finalized_ = true; +} + +void +HttpResponseJson::reset() { + HttpResponse::reset(); + json_.reset(); +} + +ConstElementPtr +HttpResponseJson::getBodyAsJson() const { + checkFinalized(); + return (json_); +} + void HttpResponseJson::setBodyAsJson(const ConstElementPtr& json_body) { if (json_body) { @@ -60,6 +84,37 @@ HttpResponseJson::setBodyAsJson(const ConstElementPtr& json_body) { json_ = json_body; } +ConstElementPtr +HttpResponseJson::getJsonElement(const std::string& element_name) const { + try { + ConstElementPtr body = getBodyAsJson(); + if (body) { + const std::map& map_value = body->mapValue(); + auto map_element = map_value.find(element_name); + if (map_element != map_value.end()) { + return (map_element->second); + } + } + + } catch (const std::exception& ex) { + isc_throw(HttpResponseJsonError, "unable to get JSON element " + << element_name << ": " << ex.what()); + } + return (ConstElementPtr()); +} + +void +HttpResponseJson::parseBodyAsJson() { + try { + // Only parse the body if it hasn't been parsed yet. + if (!json_ && !context_->body_.empty()) { + json_ = Element::fromJSON(context_->body_); + } + } catch (const std::exception& ex) { + isc_throw(HttpResponseJsonError, "unable to parse the body of the HTTP" + " response: " << ex.what()); + } +} } // namespace http } // namespace isc diff --git a/src/lib/http/response_json.h b/src/lib/http/response_json.h index 4b40548636..5c43d0b31c 100644 --- a/src/lib/http/response_json.h +++ b/src/lib/http/response_json.h @@ -8,11 +8,19 @@ #define HTTP_RESPONSE_JSON_H #include +#include #include namespace isc { namespace http { +/// @brief Exception thrown when body of the HTTP message is not JSON. +class HttpResponseJsonError : public HttpResponseError { +public: + HttpResponseJsonError(const char* file, size_t line, const char* what) : + HttpResponseError(file, line, what) { }; +}; + class HttpResponseJson; /// @brief Pointer to the @ref HttpResponseJson object. @@ -43,11 +51,37 @@ public: const CallSetGenericBody& generic_body = CallSetGenericBody::yes()); + /// @brief Completes creation of the HTTP response. + /// + /// This method marks the response as finalized. The JSON structure is + /// created and can be used to retrieve the parsed data. + virtual void finalize(); + + /// @brief Reset the state of the object. + virtual void reset(); + + /// @brief Retrieves JSON body. + /// + /// @return Pointer to the root element of the JSON structure. + /// @throw HttpRequestJsonError if an error occurred. + data::ConstElementPtr getBodyAsJson() const; + /// @brief Generates JSON content from the data structures represented /// as @ref data::ConstElementPtr. /// /// @param json_body A data structure representing JSON content. - virtual void setBodyAsJson(const data::ConstElementPtr& json_body); + void setBodyAsJson(const data::ConstElementPtr& json_body); + + /// @brief Retrieves a single JSON element. + /// + /// The element must be at top level of the JSON structure. + /// + /// @param element_name Element name. + /// + /// @return Pointer to the specified element or NULL if such element + /// doesn't exist. + /// @throw HttpRequestJsonError if an error occurred. + data::ConstElementPtr getJsonElement(const std::string& element_name) const; private: @@ -64,6 +98,10 @@ private: protected: + /// @brief Interprets body as JSON, which can be later retrieved using + /// data element objects. + void parseBodyAsJson(); + /// @brief Pointer to the parsed JSON body. data::ConstElementPtr json_; }; diff --git a/src/lib/http/response_parser.cc b/src/lib/http/response_parser.cc new file mode 100644 index 0000000000..677c3aa764 --- /dev/null +++ b/src/lib/http/response_parser.cc @@ -0,0 +1,452 @@ +// Copyright (C) 2017 Internet Systems Consortium, Inc. ("ISC") +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +#include +#include +#include + +using namespace isc::util; + +namespace isc { +namespace http { + +const int HttpResponseParser::RECEIVE_START_ST; +const int HttpResponseParser::HTTP_VERSION_H_ST; +const int HttpResponseParser::HTTP_VERSION_T1_ST; +const int HttpResponseParser::HTTP_VERSION_T2_ST; +const int HttpResponseParser::HTTP_VERSION_P_ST; +const int HttpResponseParser::HTTP_VERSION_SLASH_ST; +const int HttpResponseParser::HTTP_VERSION_MAJOR_START_ST; +const int HttpResponseParser::HTTP_VERSION_MAJOR_ST; +const int HttpResponseParser::HTTP_VERSION_MINOR_START_ST; +const int HttpResponseParser::HTTP_VERSION_MINOR_ST; +const int HttpResponseParser::HTTP_STATUS_CODE_START_ST; +const int HttpResponseParser::HTTP_STATUS_CODE_ST; +const int HttpResponseParser::HTTP_PHRASE_START_ST; +const int HttpResponseParser::HTTP_PHRASE_ST; +const int HttpResponseParser::EXPECTING_NEW_LINE1_ST; +const int HttpResponseParser::HEADER_LINE_START_ST; +const int HttpResponseParser::HEADER_LWS_ST; +const int HttpResponseParser::HEADER_NAME_ST; +const int HttpResponseParser::SPACE_BEFORE_HEADER_VALUE_ST; +const int HttpResponseParser::HEADER_VALUE_ST; +const int HttpResponseParser::EXPECTING_NEW_LINE2_ST; +const int HttpResponseParser::EXPECTING_NEW_LINE3_ST; +const int HttpResponseParser::HTTP_BODY_ST; + +HttpResponseParser::HttpResponseParser(HttpResponse& response) + : HttpMessageParserBase(response), response_(response), + context_(response.context()) { +} + +void +HttpResponseParser::initModel() { + // Initialize dictionaries of events and states. + initDictionaries(); + + // Set the current state to starting state and enter the run loop. + setState(RECEIVE_START_ST); + + // Parsing starts from here. + postNextEvent(START_EVT); +} + +void +HttpResponseParser::defineStates() { + // Call parent class implementation first. + HttpMessageParserBase::defineStates(); + + // Define HTTP parser specific states. + defineState(RECEIVE_START_ST, "RECEIVE_START_ST", + boost::bind(&HttpResponseParser::receiveStartHandler, this)); + + defineState(HTTP_VERSION_T1_ST, "HTTP_VERSION_T1_ST", + boost::bind(&HttpResponseParser::versionHTTPHandler, this, 'T', + HTTP_VERSION_T2_ST)); + + defineState(HTTP_VERSION_T2_ST, "HTTP_VERSION_T2_ST", + boost::bind(&HttpResponseParser::versionHTTPHandler, this, 'T', + HTTP_VERSION_P_ST)); + + defineState(HTTP_VERSION_P_ST, "HTTP_VERSION_P_ST", + boost::bind(&HttpResponseParser::versionHTTPHandler, this, 'P', + HTTP_VERSION_SLASH_ST)); + + defineState(HTTP_VERSION_SLASH_ST, "HTTP_VERSION_SLASH_ST", + boost::bind(&HttpResponseParser::versionHTTPHandler, this, '/', + HTTP_VERSION_MAJOR_ST)); + + defineState(HTTP_VERSION_MAJOR_START_ST, "HTTP_VERSION_MAJOR_START_ST", + boost::bind(&HttpResponseParser::numberStartHandler, this, + HTTP_VERSION_MAJOR_ST, + "HTTP version", + &context_->http_version_major_)); + + defineState(HTTP_VERSION_MAJOR_ST, "HTTP_VERSION_MAJOR_ST", + boost::bind(&HttpResponseParser::numberHandler, this, + '.', HTTP_VERSION_MINOR_START_ST, + "HTTP version", + &context_->http_version_major_)); + + defineState(HTTP_VERSION_MINOR_START_ST, "HTTP_VERSION_MINOR_START_ST", + boost::bind(&HttpResponseParser::numberStartHandler, this, + HTTP_VERSION_MINOR_ST, + "HTTP version", + &context_->http_version_minor_)); + + defineState(HTTP_VERSION_MINOR_ST, "HTTP_VERSION_MINOR_ST", + boost::bind(&HttpResponseParser::numberHandler, this, + ' ', HTTP_STATUS_CODE_START_ST, + "HTTP version", + &context_->http_version_minor_)); + + defineState(HTTP_STATUS_CODE_START_ST, "HTTP_STATUS_CODE_START_ST", + boost::bind(&HttpResponseParser::numberStartHandler, this, + HTTP_STATUS_CODE_ST, + "HTTP status code", + &context_->status_code_)); + + defineState(HTTP_STATUS_CODE_ST, "HTTP_STATUS_CODE_ST", + boost::bind(&HttpResponseParser::numberHandler, this, + ' ', HTTP_PHRASE_START_ST, + "HTTP status code", + &context_->status_code_)); + + defineState(HTTP_PHRASE_START_ST, "HTTP_PHRASE_START_ST", + boost::bind(&HttpResponseParser::phraseStartHandler, this)); + + defineState(HTTP_PHRASE_ST, "HTTP_PHRASE_ST", + boost::bind(&HttpResponseParser::phraseHandler, this)); + + defineState(EXPECTING_NEW_LINE1_ST, "EXPECTING_NEW_LINE1_ST", + boost::bind(&HttpResponseParser::expectingNewLineHandler, this, + HEADER_LINE_START_ST)); + + defineState(HEADER_LINE_START_ST, "HEADER_LINE_START_ST", + boost::bind(&HttpResponseParser::headerLineStartHandler, this)); + + defineState(HEADER_LWS_ST, "HEADER_LWS_ST", + boost::bind(&HttpResponseParser::headerLwsHandler, this)); + + defineState(HEADER_NAME_ST, "HEADER_NAME_ST", + boost::bind(&HttpResponseParser::headerNameHandler, this)); + + defineState(SPACE_BEFORE_HEADER_VALUE_ST, "SPACE_BEFORE_HEADER_VALUE_ST", + boost::bind(&HttpResponseParser::spaceBeforeHeaderValueHandler, this)); + + defineState(HEADER_VALUE_ST, "HEADER_VALUE_ST", + boost::bind(&HttpResponseParser::headerValueHandler, this)); + + defineState(EXPECTING_NEW_LINE2_ST, "EXPECTING_NEW_LINE2", + boost::bind(&HttpResponseParser::expectingNewLineHandler, this, + HEADER_LINE_START_ST)); + + defineState(EXPECTING_NEW_LINE3_ST, "EXPECTING_NEW_LINE3_ST", + boost::bind(&HttpResponseParser::expectingNewLineHandler, this, + HTTP_PARSE_OK_ST)); + + defineState(HTTP_BODY_ST, "HTTP_BODY_ST", + boost::bind(&HttpResponseParser::bodyHandler, this)); +} + +void +HttpResponseParser::receiveStartHandler() { + char c = getNextFromBuffer(); + if (getNextEvent() != NEED_MORE_DATA_EVT) { + switch(getNextEvent()) { + case START_EVT: + if (c == 'H') { + transition(HTTP_VERSION_T1_ST, DATA_READ_OK_EVT); + + } else { + parseFailure("unexpected first character " + std::string(1, c) + + ": expected \'H\'"); + } + break; + + default: + invalidEventError("receiveStartHandler", getNextEvent()); + } + } +} + +void +HttpResponseParser::versionHTTPHandler(const char expected_letter, + const unsigned int next_state) { + stateWithReadHandler("versionHTTPHandler", + [this, expected_letter, next_state](const char c) { + // We're handling one of the letters: 'H', 'T' or 'P'. + if (c == expected_letter) { + // The HTTP version is specified as "HTTP/X.Y". If the current + // character is a slash we're starting to parse major HTTP version + // number. Let's reset the version numbers. + if (c == '/') { + context_->http_version_major_ = 0; + context_->http_version_minor_ = 0; + } + // In all cases, let's transition to next specified state. + transition(next_state, DATA_READ_OK_EVT); + + } else { + // Unexpected character found. Parsing fails. + parseFailure("unexpected character " + std::string(1, c) + + " in HTTP version string"); + } + }); +} + +void +HttpResponseParser::numberStartHandler(const unsigned int next_state, + const std::string& number_name, + unsigned int* storage) { + stateWithReadHandler("numberStartHandler", + [this, next_state, number_name, storage](const char c) mutable { + // HTTP version number must be a digit. + if (isdigit(c)) { + // Update the version number using new digit being parsed. + *storage = *storage * 10 + c - '0'; + transition(next_state, DATA_READ_OK_EVT); + + } else { + parseFailure("expected digit in " + number_name + ", found " + + std::string(1, c)); + } + }); +} + +void +HttpResponseParser::numberHandler(const char following_character, + const unsigned int next_state, + const std::string& number_name, + unsigned int* const storage) { + stateWithReadHandler("numberHandler", + [this, following_character, number_name, next_state, storage](const char c) + mutable { + // We're getting to the end of the version number, let's transition + // to next state. + if (c == following_character) { + transition(next_state, DATA_READ_OK_EVT); + + } else if (isdigit(c)) { + // Current character is a digit, so update the version number. + *storage = *storage * 10 + c - '0'; + + } else { + parseFailure("expected digit in " + number_name + ", found " + + std::string(1, c)); + } + }); +} + +void +HttpResponseParser::phraseStartHandler() { + stateWithReadHandler("phraseStartHandler", [this](const char c) { + if (!isChar(c) || isCtl(c)) { + parseFailure("invalid first character " + std::string(1, c) + + " in HTTP phrase"); + } else { + context_->phrase_.push_back(c); + transition(HTTP_PHRASE_ST, DATA_READ_OK_EVT); + } + }); +} + +void +HttpResponseParser::phraseHandler() { + stateWithReadHandler("phraseHandler", [this](const char c) { + if (c == '\r') { + transition(EXPECTING_NEW_LINE1_ST, DATA_READ_OK_EVT); + + } else if (!isChar(c) || isCtl(c)) { + parseFailure("invalid character " + std::string(1, c) + + " in HTTP phrase"); + + } else { + context_->phrase_.push_back(c); + transition(HTTP_PHRASE_ST, DATA_READ_OK_EVT); + } + }); +} + +void +HttpResponseParser::expectingNewLineHandler(const unsigned int next_state) { + stateWithReadHandler("expectingNewLineHandler", [this, next_state](const char c) { + // Only a new line character is allowed in this state. + if (c == '\n') { + // If next state is HTTP_PARSE_OK_ST it means that we're + // parsing 3rd new line in the HTTP request message. This + // terminates the HTTP request (if there is no body) or marks the + // beginning of the body. + if (next_state == HTTP_PARSE_OK_ST) { + // Whether there is a body in this message or not, we should + // parse the HTTP headers to validate it and to check if there + // is "Content-Length" specified. The "Content-Length" is + // required for parsing body. + response_.create(); + try { + // This will throw exception if there is no Content-Length. + uint64_t content_length = + response_.getHeaderValueAsUint64("Content-Length"); + if (content_length > 0) { + // There is body in this request, so let's parse it. + transition(HTTP_BODY_ST, DATA_READ_OK_EVT); + } + } catch (const std::exception& ex) { + // There is no body in this message. If the body is required + // parsing fails. + if (response_.requiresBody()) { + parseFailure("HTTP message lacks a body"); + + } else { + // Body not required so simply terminate parsing. + transition(HTTP_PARSE_OK_ST, HTTP_PARSE_OK_EVT); + } + } + + } else { + // This is 1st or 2nd new line, so let's transition to the + // next state required by this handler. + transition(next_state, DATA_READ_OK_EVT); + } + } else { + parseFailure("expecting new line after CR, found " + + std::string(1, c)); + } + }); +} + +void +HttpResponseParser::headerLineStartHandler() { + stateWithReadHandler("headerLineStartHandler", [this](const char c) { + // If we're parsing HTTP headers and we found CR it marks the + // end of headers section. + if (c == '\r') { + transition(EXPECTING_NEW_LINE3_ST, DATA_READ_OK_EVT); + + } else if (!context_->headers_.empty() && ((c == ' ') || (c == '\t'))) { + // New line in headers section followed by space or tab is an LWS, + // a line break within header value. + transition(HEADER_LWS_ST, DATA_READ_OK_EVT); + + } else if (!isChar(c) || isCtl(c) || isSpecial(c)) { + parseFailure("invalid character " + std::string(1, c) + + " in header name"); + + } else { + // Update header name with the parse letter. + context_->headers_.push_back(HttpHeaderContext()); + context_->headers_.back().name_.push_back(c); + transition(HEADER_NAME_ST, DATA_READ_OK_EVT); + } + }); +} + +void +HttpResponseParser::headerLwsHandler() { + stateWithReadHandler("headerLwsHandler", [this](const char c) { + if (c == '\r') { + // Found CR during parsing a header value. Next value + // should be new line. + transition(EXPECTING_NEW_LINE2_ST, DATA_READ_OK_EVT); + + } else if ((c == ' ') || (c == '\t')) { + // Space and tab is used to mark LWS. Simply swallow + // this character. + transition(getCurrState(), DATA_READ_OK_EVT); + + } else if (isCtl(c)) { + parseFailure("control character found in the HTTP header " + + context_->headers_.back().name_); + + } else { + // We're parsing header value, so let's update it. + context_->headers_.back().value_.push_back(c); + transition(HEADER_VALUE_ST, DATA_READ_OK_EVT); + } + }); +} + +void +HttpResponseParser::headerNameHandler() { + stateWithReadHandler("headerNameHandler", [this](const char c) { + // Colon follows header name and it has its own state. + if (c == ':') { + transition(SPACE_BEFORE_HEADER_VALUE_ST, DATA_READ_OK_EVT); + + } else if (!isChar(c) || isCtl(c) || isSpecial(c)) { + parseFailure("invalid character " + std::string(1, c) + + " found in the HTTP header name"); + + } else { + // Parsing a header name, so update it. + context_->headers_.back().name_.push_back(c); + transition(getCurrState(), DATA_READ_OK_EVT); + } + }); +} + +void +HttpResponseParser::spaceBeforeHeaderValueHandler() { + stateWithReadHandler("spaceBeforeHeaderValueHandler", [this](const char c) { + if (c == ' ') { + // Remove leading whitespace from the header value. + transition(getCurrState(), DATA_READ_OK_EVT); + + } else if (c == '\r') { + // If CR found during parsing header value, it marks the end + // of this value. + transition(EXPECTING_NEW_LINE2_ST, DATA_READ_OK_EVT); + + } else if (isCtl(c)) { + parseFailure("control character found in the HTTP header " + + context_->headers_.back().name_); + + } else { + // Still parsing the value, so let's update it. + context_->headers_.back().value_.push_back(c); + transition(HEADER_VALUE_ST, DATA_READ_OK_EVT); + } + }); +} + +void +HttpResponseParser::headerValueHandler() { + stateWithReadHandler("headerValueHandler", [this](const char c) { + // If CR found during parsing header value, it marks the end + // of this value. + if (c == '\r') { + transition(EXPECTING_NEW_LINE2_ST, DATA_READ_OK_EVT); + + } else if (isCtl(c)) { + parseFailure("control character found in the HTTP header " + + context_->headers_.back().name_); + + } else { + // Still parsing the value, so let's update it. + context_->headers_.back().value_.push_back(c); + transition(HEADER_VALUE_ST, DATA_READ_OK_EVT); + } + }); +} + +void +HttpResponseParser::bodyHandler() { + stateWithReadHandler("bodyHandler", [this](const char c) { + // We don't validate the body at this stage. Simply record the + // number of characters specified within "Content-Length". + context_->body_.push_back(c); + if (context_->body_.length() < + response_.getHeaderValueAsUint64("Content-Length")) { + transition(HTTP_BODY_ST, DATA_READ_OK_EVT); + } else { + transition(HTTP_PARSE_OK_ST, HTTP_PARSE_OK_EVT); + } + }); +} + + +} // end of namespace isc::http +} // end of namespace isc diff --git a/src/lib/http/response_parser.h b/src/lib/http/response_parser.h new file mode 100644 index 0000000000..074ffa4f4a --- /dev/null +++ b/src/lib/http/response_parser.h @@ -0,0 +1,253 @@ +// Copyright (C) 2017 Internet Systems Consortium, Inc. ("ISC") +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +#ifndef HTTP_RESPONSE_PARSER_H +#define HTTP_RESPONSE_PARSER_H + +#include +#include +#include + +namespace isc { +namespace http { + +/// @brief Exception thrown when an error during parsing HTTP response +/// has occurred. +/// +/// The most common errors are due to receiving malformed response. +class HttpResponseParserError : public HttpMessageParserBaseError { +public: + HttpResponseParserError(const char* file, size_t line, const char* what) : + HttpMessageParserBaseError(file, line, what) { }; +}; + +class HttpResponseParser; + +/// @brief Pointer to the @ref HttpResponseParser. +typedef boost::shared_ptr HttpResponseParserPtr; + +/// @brief A generic parser for HTTP responses. +/// +/// This class implements a parser for HTTP responses. The parser derives +/// from the @ref HttpMessageParserBase class and implements its own state +/// machine on top of it. The states of the parser reflect various parts of +/// the HTTP message being parsed, e.g. parsing HTTP version, status code, +/// message body etc. The descriptions of all parser states are provided +/// below together with the constants defining these states. +/// +/// The response parser validates the syntax of the received message as it +/// progresses with parsing the data. Though, it doesn't interpret the +/// received data until the whole message is parsed. In most cases we want +/// to apply some restrictions on the message content, e.g. responses to +/// control commands include JSON body. The parser doesn't verify if the +/// message meets such restrictions until the whole message is parsed, +/// i.e. stored in the @ref HttpResponseContext object. This object is +/// associated with @ref HttpResponse object (or its derivation). When +/// the parsing is completed, the @ref HttpResponse::create method is +/// called to retrieve and interpret the data from the context. In +/// particular, the @ref HttpResponse or its derivation checks if the +/// received message meets the desired restrictions. +class HttpResponseParser : public HttpMessageParserBase { +public: + + /// @name States supported by the HttpResponseParser. + /// + //@{ + + /// @brief State indicating a beginning of parsing. + static const int RECEIVE_START_ST = SM_DERIVED_STATE_MIN + 101; + + /// @brief Parsing letter "H" of "HTTP". + static const int HTTP_VERSION_H_ST = SM_DERIVED_STATE_MIN + 102; + + /// @brief Parsing first occurrence of "T" in "HTTP". + static const int HTTP_VERSION_T1_ST = SM_DERIVED_STATE_MIN + 103; + + /// @brief Parsing second occurrence of "T" in "HTTP". + static const int HTTP_VERSION_T2_ST = SM_DERIVED_STATE_MIN + 104; + + /// @brief Parsing letter "P" in "HTTP". + static const int HTTP_VERSION_P_ST = SM_DERIVED_STATE_MIN + 105; + + /// @brief Parsing slash character in "HTTP/Y.X" + static const int HTTP_VERSION_SLASH_ST = SM_DERIVED_STATE_MIN + 106; + + /// @brief Starting to parse major HTTP version number. + static const int HTTP_VERSION_MAJOR_START_ST = SM_DERIVED_STATE_MIN + 107; + + /// @brief Parsing major HTTP version number. + static const int HTTP_VERSION_MAJOR_ST = SM_DERIVED_STATE_MIN + 108; + + /// @brief Starting to parse minor HTTP version number. + static const int HTTP_VERSION_MINOR_START_ST = SM_DERIVED_STATE_MIN + 109; + + /// @brief Parsing minor HTTP version number. + static const int HTTP_VERSION_MINOR_ST = SM_DERIVED_STATE_MIN + 110; + + /// @brief Starting to parse HTTP status code. + static const int HTTP_STATUS_CODE_START_ST = SM_DERIVED_STATE_MIN + 111; + + /// @brief Parsing HTTP status code. + static const int HTTP_STATUS_CODE_ST = SM_DERIVED_STATE_MIN + 112; + + /// @brief Starting to parse HTTP status phrase. + static const int HTTP_PHRASE_START_ST = SM_DERIVED_STATE_MIN + 113; + + /// @brief Parsing HTTP status phrase. + static const int HTTP_PHRASE_ST = SM_DERIVED_STATE_MIN + 114; + + /// @brief Parsing first new line (after HTTP status phrase). + static const int EXPECTING_NEW_LINE1_ST = SM_DERIVED_STATE_MIN + 115; + + static const int HEADER_LINE_START_ST = SM_DERIVED_STATE_MIN + 116; + + /// @brief Parsing LWS (Linear White Space), i.e. new line with a space + /// or tab character while parsing a HTTP header. + static const int HEADER_LWS_ST = SM_DERIVED_STATE_MIN + 117; + + /// @brief Parsing header name. + static const int HEADER_NAME_ST = SM_DERIVED_STATE_MIN + 118; + + /// @brief Parsing space before header value. + static const int SPACE_BEFORE_HEADER_VALUE_ST = SM_DERIVED_STATE_MIN + 119; + + /// @brief Parsing header value. + static const int HEADER_VALUE_ST = SM_DERIVED_STATE_MIN + 120; + + /// @brief Expecting new line after parsing header value. + static const int EXPECTING_NEW_LINE2_ST = SM_DERIVED_STATE_MIN + 121; + + /// @brief Expecting second new line marking end of HTTP headers. + static const int EXPECTING_NEW_LINE3_ST = SM_DERIVED_STATE_MIN + 122; + + /// @brief Parsing body of a HTTP message. + static const int HTTP_BODY_ST = SM_DERIVED_STATE_MIN + 123; + + //@} + + /// @brief Constructor. + /// + /// Creates new instance of the parser. + /// + /// @param response Reference to the @ref HttpResponse object or its + /// derivation that should be used to validate the parsed response and + /// to be used as a container for the parsed response. + explicit HttpResponseParser(HttpResponse& response); + + /// @brief Initialize the state model for parsing. + /// + /// This method must be called before parsing the response, i.e. before + /// calling @ref HttpResponseParser::poll. It initializes dictionaries of + /// states and events, and sets the initial model state to RECEIVE_START_ST. + void initModel(); + +private: + + /// @brief Defines states of the parser. + virtual void defineStates(); + + /// @name State handlers. + /// + //@{ + + /// @brief Handler for RECEIVE_START_ST. + void receiveStartHandler(); + + /// @brief Handler for states parsing "HTTP" string within the first line + /// of the HTTP request. + /// + /// @param expected_letter One of the 'H', 'T', 'P'. + /// @param next_state A state to which the parser should transition after + /// parsing the character. + void versionHTTPHandler(const char expected_letter, + const unsigned int next_state); + + /// @brief Handler for states in which parser begins to read numeric values. + /// + /// This handler calculates version number using the following equation: + /// @code + /// storage = storage * 10 + c - '0'; + /// @endcode + /// + /// @param next_state State to which the parser should transition. + /// @param number_name Name of the parsed numeric value, e.g. HTTP version or + /// HTTP status code (used for error logging). + /// @param [out] storage Reference to a number holding current product of + /// parsing major or minor version number. + void numberStartHandler(const unsigned int next_state, + const std::string& number_name, + unsigned int* const storage); + + /// @brief Handler for states in which pareser reads numeric values. + /// + /// This handler calculates version number using the following equation: + /// @code + /// storage = storage * 10 + c - '0'; + /// @endcode + /// + /// @param following_character Character following the version number, i.e. + /// '.' for major version, \r for minor version. + /// @param next_state State to which the parser should transition. + /// @param number_name Name of the parsed numeric value, e.g. HTTP version or + /// HTTP status code (used for error logging). + /// @param [out] storage Pointer to a number holding current product of + /// parsing major or minor version number. + void numberHandler(const char following_character, + const unsigned int next_state, + const std::string& number_name, + unsigned int* const storage); + + /// @brief Handler for HTTP_PHRASE_START_ST. + void phraseStartHandler(); + + /// @brief Handler for HTTP_PHRASE_ST. + void phraseHandler(); + + /// @brief Handler for states related to new lines. + /// + /// If the next_state is HTTP_PARSE_OK_ST it indicates that the parsed + /// value is a 3rd new line within request HTTP message. In this case the + /// handler calls @ref HttpRequest::create to validate the received message + /// (excluding body). The hander then reads the "Content-Length" header to + /// check if the request contains a body. If the "Content-Length" is greater + /// than zero, the parser transitions to HTTP_BODY_ST. If the + /// "Content-Length" doesn't exist the parser transitions to + /// HTTP_PARSE_OK_ST. + /// + /// @param next_state A state to which parser should transition. + void expectingNewLineHandler(const unsigned int next_state); + + /// @brief Handler for HEADER_LINE_START_ST. + void headerLineStartHandler(); + + /// @brief Handler for HEADER_LWS_ST. + void headerLwsHandler(); + + /// @brief Handler for HEADER_NAME_ST. + void headerNameHandler(); + + /// @brief Handler for SPACE_BEFORE_HEADER_VALUE_ST. + void spaceBeforeHeaderValueHandler(); + + /// @brief Handler for HEADER_VALUE_ST. + void headerValueHandler(); + + /// @brief Handler for HTTP_BODY_ST. + void bodyHandler(); + + //@} + + /// @brief Reference to the response object specified in the constructor. + HttpResponse& response_; + + /// @brief Pointer to the internal context of the @ref HttpResponse object. + HttpResponseContextPtr context_; +}; + +} // end of namespace isc::http +} // end of namespace isc + +#endif diff --git a/src/lib/http/tests/Makefile.am b/src/lib/http/tests/Makefile.am index a9c637deea..3a0a937d25 100644 --- a/src/lib/http/tests/Makefile.am +++ b/src/lib/http/tests/Makefile.am @@ -29,6 +29,7 @@ libhttp_unittests_SOURCES += post_request_json_unittests.cc libhttp_unittests_SOURCES += request_parser_unittests.cc libhttp_unittests_SOURCES += request_test.h libhttp_unittests_SOURCES += response_creator_unittests.cc +libhttp_unittests_SOURCES += response_parser_unittests.cc libhttp_unittests_SOURCES += response_test.h libhttp_unittests_SOURCES += request_unittests.cc libhttp_unittests_SOURCES += response_unittests.cc diff --git a/src/lib/http/tests/response_parser_unittests.cc b/src/lib/http/tests/response_parser_unittests.cc new file mode 100644 index 0000000000..b4dacfe890 --- /dev/null +++ b/src/lib/http/tests/response_parser_unittests.cc @@ -0,0 +1,292 @@ +// Copyright (C) 2017 Internet Systems Consortium, Inc. ("ISC") +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +#include + +#include +#include +#include +#include +#include + +using namespace isc; +using namespace isc::data; +using namespace isc::http; + +namespace { + +/// @brief Test fixture class for @ref HttpResponseParser. +class HttpResponseParserTest : public ::testing::Test { +public: + + /// @brief Creates HTTP response string. + /// + /// @param preamble A string including HTTP response's first line + /// and all headers except "Content-Length". + /// @param payload A string containing HTTP response payload. + std::string createResponseString(const std::string& preamble, + const std::string& payload) { + std::ostringstream s; + s << preamble; + s << "Content-Length: " << payload.length() << "\r\n\r\n" + << payload; + return (s.str()); + } + + /// @brief Parses the HTTP response and checks that parsing was + /// successful. + /// + /// @param http_resp HTTP response string. + void doParse(const std::string& http_resp) { + HttpResponseParser parser(response_); + ASSERT_NO_THROW(parser.initModel()); + + parser.postBuffer(&http_resp[0], http_resp.size()); + ASSERT_NO_THROW(parser.poll()); + + ASSERT_FALSE(parser.needData()); + ASSERT_TRUE(parser.httpParseOk()); + EXPECT_TRUE(parser.getErrorMessage().empty()); + } + + /// @brief Tests that parsing fails when malformed HTTP response + /// is received. + /// + /// @param http_resp HTTP response string. + void testInvalidHttpResponse(const std::string& http_resp) { + HttpResponseParser parser(response_); + ASSERT_NO_THROW(parser.initModel()); + + parser.postBuffer(&http_resp[0], http_resp.size()); + ASSERT_NO_THROW(parser.poll()); + + EXPECT_FALSE(parser.needData()); + EXPECT_FALSE(parser.httpParseOk()); + EXPECT_FALSE(parser.getErrorMessage().empty()); + } + + /// @brief Instance of the HttpResponse used by the unit tests. + HttpResponse response_; +}; + +// Test test verifies that an HTTP response including JSON body is parsed +// successfully. +TEST_F(HttpResponseParserTest, responseWithJson) { + std::string http_resp = "HTTP/1.1 408 Request Timeout\r\n" + "Content-Type: application/json\r\n"; + std::string json = "{ \"result\": 0, \"text\": \"All ok\" }"; + + http_resp = createResponseString(http_resp, json); + + // Create HTTP response which accepts JSON as a body. + HttpResponseJson response; + + // Create a parser and make it use the response we created. + HttpResponseParser parser(response); + ASSERT_NO_THROW(parser.initModel()); + + // Simulate receiving HTTP response in chunks. + const unsigned chunk_size = 10; + while (!http_resp.empty()) { + size_t chunk = http_resp.size() % chunk_size; + if (chunk == 0) { + chunk = chunk_size; + } + + parser.postBuffer(&http_resp[0], chunk); + http_resp.erase(0, chunk); + parser.poll(); + if (chunk < chunk_size) { + ASSERT_TRUE(parser.needData()); + } + } + + // Parser should have parsed the response and should expect no more data. + ASSERT_FALSE(parser.needData()); + // Parsing should be successful. + ASSERT_TRUE(parser.httpParseOk()) << parser.getErrorMessage(); + // There should be no error message. + EXPECT_TRUE(parser.getErrorMessage().empty()); + + // Verify HTTP version, status code and phrase. + EXPECT_EQ(1, response.getHttpVersion().major_); + EXPECT_EQ(1, response.getHttpVersion().minor_); + EXPECT_EQ(HttpStatusCode::REQUEST_TIMEOUT, response.getStatusCode()); + EXPECT_EQ("Request Timeout", response.getStatusPhrase()); + + // Try to retrieve values carried in JSON payload. + ConstElementPtr json_element; + ASSERT_NO_THROW(json_element = response.getJsonElement("result")); + EXPECT_EQ(0, json_element->intValue()); + + ASSERT_NO_THROW(json_element = response.getJsonElement("text")); + EXPECT_EQ("All ok", json_element->stringValue()); +} + +// This test verifies that extraneous data in the response will not cause +// an error if "Content-Length" value refers to the length of the valid +// part of the response. +TEST_F(HttpResponseParserTest, extraneousDataInResponse) { + std::string http_resp = "HTTP/1.0 200 OK\r\n" + "Content-Type: application/json\r\n"; + std::string json = "{ \"service\": \"dhcp4\", \"command\": \"shutdown\" }"; + + // Create valid response. + http_resp = createResponseString(http_resp, json); + + // Add some garbage at the end. + http_resp += "some stuff which, if parsed, will cause errors"; + + // Create HTTP response which accepts JSON as a body. + HttpResponseJson response; + + // Create a parser and make it use the response we created. + HttpResponseParser parser(response); + ASSERT_NO_THROW(parser.initModel()); + + // Feed the parser with the response containing some garbage at the end. + parser.postBuffer(&http_resp[0], http_resp.size()); + ASSERT_NO_THROW(parser.poll()); + + // The parser should only parse the valid part of the response as indicated + // by the Content-Length. + ASSERT_FALSE(parser.needData()); + ASSERT_TRUE(parser.httpParseOk()); + // There should be no error message. + EXPECT_TRUE(parser.getErrorMessage().empty()); + + // Do another poll() to see if the parser will parse the garbage. We + // expect that it doesn't. + ASSERT_NO_THROW(parser.poll()); + EXPECT_FALSE(parser.needData()); + EXPECT_TRUE(parser.httpParseOk()); + EXPECT_TRUE(parser.getErrorMessage().empty()); +} + +// This test verifies that LWS is parsed correctly. The LWS marks line breaks +// in the HTTP header values. +TEST_F(HttpResponseParserTest, getLWS) { + // "User-Agent" header contains line breaks with whitespaces in the new + // lines to mark continuation of the header value. + std::string http_resp = "HTTP/1.1 200 OK\r\n" + "Content-Type: text/html\r\n" + "User-Agent: Kea/1.2 Command \r\n" + " Control \r\n" + "\tClient\r\n\r\n"; + + ASSERT_NO_FATAL_FAILURE(doParse(http_resp)); + + // Verify parsed values. + EXPECT_EQ(1, response_.getHttpVersion().major_); + EXPECT_EQ(1, response_.getHttpVersion().minor_); + EXPECT_EQ(HttpStatusCode::OK, response_.getStatusCode()); + EXPECT_EQ("OK", response_.getStatusPhrase()); + EXPECT_EQ("text/html", response_.getHeaderValue("Content-Type")); + EXPECT_EQ("Kea/1.2 Command Control Client", + response_.getHeaderValue("User-Agent")); +} + +// This test verifies that the HTTP response with no headers is +// parsed correctly. +TEST_F(HttpResponseParserTest, noHeaders) { + std::string http_resp = "HTTP/1.1 204 No Content\r\n\r\n"; + + ASSERT_NO_FATAL_FAILURE(doParse(http_resp)); + + // Verify the values. + EXPECT_EQ(1, response_.getHttpVersion().major_); + EXPECT_EQ(1, response_.getHttpVersion().minor_); + EXPECT_EQ(HttpStatusCode::NO_CONTENT, response_.getStatusCode()); +} + +// This test verifies that headers are case insensitive. +TEST_F(HttpResponseParserTest, headersCaseInsensitive) { + std::string http_resp = "HTTP/1.1 200 OK\r\n" + "Content-type: text/html\r\n" + "connection: clOSe\r\n\r\n"; + + ASSERT_NO_FATAL_FAILURE(doParse(http_resp)); + + EXPECT_EQ("text/html", response_.getHeader("Content-Type")->getValue()); + EXPECT_EQ("close", response_.getHeader("Connection")->getLowerCaseValue()); + EXPECT_EQ(1, response_.getHttpVersion().major_); + EXPECT_EQ(1, response_.getHttpVersion().minor_); + EXPECT_EQ(HttpStatusCode::OK, response_.getStatusCode()); + EXPECT_EQ("OK", response_.getStatusPhrase()); +} + +// This test verifies that the header with no whitespace between the +// colon and header value is accepted. +TEST_F(HttpResponseParserTest, noHeaderWhitespace) { + std::string http_resp = "HTTP/1.0 200 OK\r\n" + "Content-Type:text/html\r\n\r\n"; + + ASSERT_NO_FATAL_FAILURE(doParse(http_resp)); + + EXPECT_EQ("text/html", response_.getHeaderValue("Content-Type")); + EXPECT_EQ(1, response_.getHttpVersion().major_); + EXPECT_EQ(0, response_.getHttpVersion().minor_); + EXPECT_EQ(HttpStatusCode::OK, response_.getStatusCode()); + EXPECT_EQ("OK", response_.getStatusPhrase()); +} + +// This test verifies that the header value preceded with multiple +// whitespaces is accepted. +TEST_F(HttpResponseParserTest, multipleLeadingHeaderWhitespaces) { + std::string http_resp = "HTTP/1.0 200 OK\r\n" + "Content-Type: text/html\r\n\r\n"; + + ASSERT_NO_FATAL_FAILURE(doParse(http_resp)); + + EXPECT_EQ("text/html", response_.getHeaderValue("Content-Type")); + EXPECT_EQ(1, response_.getHttpVersion().major_); + EXPECT_EQ(0, response_.getHttpVersion().minor_); + EXPECT_EQ(HttpStatusCode::OK, response_.getStatusCode()); + EXPECT_EQ("OK", response_.getStatusPhrase()); +} + +// This test verifies that the response containing a typo in the +// HTTP version string causes parsing error. +TEST_F(HttpResponseParserTest, invalidHTTPString) { + std::string http_resp = "HTLP/2.0 100 OK\r\n" + "Content-Type: text/html\r\n\r\n"; + testInvalidHttpResponse(http_resp); +} + +// This test verifies that error is reported when the HTTP version +// string doesn't contain a slash character. +TEST_F(HttpResponseParserTest, invalidHttpVersionNoSlash) { + std::string http_resp = "HTTP 1.1 100 OK\r\n" + "Content-Type: text/html\r\n\r\n"; + testInvalidHttpResponse(http_resp); +} + +// This test verifies that error is reported when HTTP version string +// doesn't contain the minor version number. +TEST_F(HttpResponseParserTest, invalidHttpNoMinorVersion) { + std::string http_resp = "HTTP/1 200 OK\r\n" + "Content-Type: text/html\r\n\r\n"; + testInvalidHttpResponse(http_resp); +} + +// This test verifies that error is reported when HTTP header name +// contains an invalid character. +TEST_F(HttpResponseParserTest, invalidHeaderName) { + std::string http_resp = "HTTP/1.1 200 OK\r\n" + "Content-;: text/html\r\n\r\n"; + testInvalidHttpResponse(http_resp); +} + +// This test verifies that error is reported when HTTP header value +// is not preceded with the colon character. +TEST_F(HttpResponseParserTest, noColonInHttpHeader) { + std::string http_resp = "HTTP/1.1 200 OK\r\n" + "Content-Type text/html\r\n\r\n"; + testInvalidHttpResponse(http_resp); +} + + +}