From: JINMEI Tatuya Date: Wed, 28 Nov 2012 23:12:28 +0000 (-0800) Subject: [2506] rename: MasterLexer::Token to MasterToken. X-Git-Tag: bind10-1.0.0-beta-release~46^2~10 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=c6c7cdee4088d6be3728ffaf07a25b920b0e8b5c;p=thirdparty%2Fkea.git [2506] rename: MasterLexer::Token to MasterToken. it's a kind of hack for the convenience of the planned change of this branch, but the revised name doesn't seem really bad anyway. --- diff --git a/src/lib/dns/master_lexer.cc b/src/lib/dns/master_lexer.cc index d223d7a71c..4db2bf8527 100644 --- a/src/lib/dns/master_lexer.cc +++ b/src/lib/dns/master_lexer.cc @@ -36,7 +36,7 @@ using namespace master_lexer_internal; struct MasterLexer::MasterLexerImpl { - MasterLexerImpl() : source_(NULL), token_(Token::NOT_STARTED), + MasterLexerImpl() : source_(NULL), token_(MasterToken::NOT_STARTED), paren_count_(0), last_was_eol_(false), has_previous_(false), previous_paren_count_(0), @@ -82,7 +82,7 @@ struct MasterLexer::MasterLexerImpl { std::vector sources_; InputSource* source_; // current source (NULL if sources_ is empty) - Token token_; // currently recognized token (set by a state) + MasterToken token_; // currently recognized token (set by a state) std::vector data_; // placeholder for string data // These are used in states, and defined here only as a placeholder. @@ -165,7 +165,7 @@ MasterLexer::getSourceLine() const { return (impl_->sources_.back()->getCurrentLine()); } -const MasterLexer::Token& +const MasterToken& MasterLexer::getNextToken(Options options) { if (impl_->source_ == NULL) { isc_throw(isc::InvalidOperation, "No source to read tokens from"); @@ -177,7 +177,7 @@ MasterLexer::getNextToken(Options options) { impl_->has_previous_ = true; // Reset the token now. This is to check a token was actually produced. // This is debugging aid. - impl_->token_ = Token(Token::NO_TOKEN_PRODUCED); + impl_->token_ = MasterToken(MasterToken::NO_TOKEN_PRODUCED); // And get the token // This actually handles EOF internally too. @@ -187,8 +187,8 @@ MasterLexer::getNextToken(Options options) { } // Make sure a token was produced. Since this Can Not Happen, we assert // here instead of throwing. - assert(impl_->token_.getType() != Token::ERROR || - impl_->token_.getErrorCode() != Token::NO_TOKEN_PRODUCED); + assert(impl_->token_.getType() != MasterToken::ERROR || + impl_->token_.getErrorCode() != MasterToken::NO_TOKEN_PRODUCED); return (impl_->token_); } @@ -217,10 +217,10 @@ const size_t error_text_max_count = sizeof(error_text) / sizeof(error_text[0]); } // end unnamed namespace std::string -MasterLexer::Token::getErrorText() const { +MasterToken::getErrorText() const { if (type_ != ERROR) { isc_throw(InvalidOperation, - "Token::getErrorText() for non error type"); + "MasterToken::getErrorText() for non error type"); } // The class integrity ensures the following: @@ -233,14 +233,12 @@ namespace master_lexer_internal { // Note that these need to be defined here so that they can refer to // the details of MasterLexerImpl. -typedef MasterLexer::Token Token; // convenience shortcut - bool State::wasLastEOL(const MasterLexer& lexer) const { return (lexer.impl_->last_was_eol_); } -const MasterLexer::Token& +const MasterToken& State::getToken(const MasterLexer& lexer) const { return (lexer.impl_->token_); } @@ -270,7 +268,7 @@ public: if (c != '\n') { getLexerImpl(lexer)->source_->ungetChar(); } - getLexerImpl(lexer)->token_ = Token(Token::END_OF_LINE); + getLexerImpl(lexer)->token_ = MasterToken(MasterToken::END_OF_LINE); getLexerImpl(lexer)->last_was_eol_ = true; } }; @@ -341,24 +339,24 @@ State::start(MasterLexer& lexer, MasterLexer::Options options) { if (c == InputSource::END_OF_STREAM) { lexerimpl.last_was_eol_ = false; if (paren_count != 0) { - lexerimpl.token_ = Token(Token::UNBALANCED_PAREN); + lexerimpl.token_ = MasterToken(MasterToken::UNBALANCED_PAREN); paren_count = 0; // reset to 0; this helps in lenient mode. return (NULL); } - lexerimpl.token_ = Token(Token::END_OF_FILE); + lexerimpl.token_ = MasterToken(MasterToken::END_OF_FILE); return (NULL); } else if (c == ' ' || c == '\t') { // If requested and we are not in (), recognize the initial space. if (lexerimpl.last_was_eol_ && paren_count == 0 && (options & MasterLexer::INITIAL_WS) != 0) { lexerimpl.last_was_eol_ = false; - lexerimpl.token_ = Token(Token::INITIAL_WS); + lexerimpl.token_ = MasterToken(MasterToken::INITIAL_WS); return (NULL); } } else if (c == '\n') { lexerimpl.last_was_eol_ = true; if (paren_count == 0) { // we don't recognize EOL if we are in () - lexerimpl.token_ = Token(Token::END_OF_LINE); + lexerimpl.token_ = MasterToken(MasterToken::END_OF_LINE); return (NULL); } } else if (c == '\r') { @@ -374,7 +372,7 @@ State::start(MasterLexer& lexer, MasterLexer::Options options) { } else if (c == ')') { lexerimpl.last_was_eol_ = false; if (paren_count == 0) { - lexerimpl.token_ = Token(Token::UNBALANCED_PAREN); + lexerimpl.token_ = MasterToken(MasterToken::UNBALANCED_PAREN); return (NULL); } --paren_count; @@ -406,7 +404,7 @@ String::handle(MasterLexer& lexer) const { if (getLexerImpl(lexer)->isTokenEnd(c, escaped)) { getLexerImpl(lexer)->source_->ungetChar(); getLexerImpl(lexer)->token_ = - MasterLexer::Token(&data.at(0), data.size()); + MasterToken(&data.at(0), data.size()); return; } escaped = (c == '\\' && !escaped); @@ -416,7 +414,7 @@ String::handle(MasterLexer& lexer) const { void QString::handle(MasterLexer& lexer) const { - MasterLexer::Token& token = getLexerImpl(lexer)->token_; + MasterToken& token = getLexerImpl(lexer)->token_; std::vector& data = getLexerImpl(lexer)->data_; data.clear(); @@ -424,7 +422,7 @@ QString::handle(MasterLexer& lexer) const { while (true) { const int c = getLexerImpl(lexer)->source_->getChar(); if (c == InputSource::END_OF_STREAM) { - token = Token(Token::UNEXPECTED_END); + token = MasterToken(MasterToken::UNEXPECTED_END); return; } else if (c == '"') { if (escaped) { @@ -433,12 +431,12 @@ QString::handle(MasterLexer& lexer) const { escaped = false; data.back() = '"'; } else { - token = MasterLexer::Token(&data.at(0), data.size(), true); + token = MasterToken(&data.at(0), data.size(), true); return; } } else if (c == '\n' && !escaped) { getLexerImpl(lexer)->source_->ungetChar(); - token = Token(Token::UNBALANCED_QUOTES); + token = MasterToken(MasterToken::UNBALANCED_QUOTES); return; } else { escaped = (c == '\\' && !escaped); @@ -449,7 +447,7 @@ QString::handle(MasterLexer& lexer) const { void Number::handle(MasterLexer& lexer) const { - MasterLexer::Token& token = getLexerImpl(lexer)->token_; + MasterToken& token = getLexerImpl(lexer)->token_; // It may yet turn out to be a string, so we first // collect all the data @@ -473,11 +471,10 @@ Number::handle(MasterLexer& lexer) const { } catch (const boost::bad_lexical_cast&) { // Since we already know we have only digits, // range should be the only possible problem. - token = Token(Token::NUMBER_OUT_OF_RANGE); + token = Token(MasterToken::NUMBER_OUT_OF_RANGE); } } else { - token = MasterLexer::Token(&data.at(0), - data.size()); + token = MasterToken(&data.at(0), data.size()); } return; } diff --git a/src/lib/dns/master_lexer.h b/src/lib/dns/master_lexer.h index 4a861fc704..13205be14f 100644 --- a/src/lib/dns/master_lexer.h +++ b/src/lib/dns/master_lexer.h @@ -28,225 +28,6 @@ namespace master_lexer_internal { class State; } -/// \brief Tokenizer for parsing DNS master files. -/// -/// The \c MasterLexer class provides tokenize interfaces for parsing DNS -/// master files. It understands some special rules of master files as -/// defined in RFC 1035, such as comments, character escaping, and multi-line -/// data, and provides the user application with the actual data in a -/// more convenient form such as a std::string object. -/// -/// In order to support the $INCLUDE notation, this class is designed to be -/// able to operate on multiple files or input streams in the nested way. -/// The \c pushSource() and \c popSource() methods correspond to the push -/// and pop operations. -/// -/// While this class is public, it is less likely to be used by normal -/// applications; it's mainly expected to be used within this library, -/// specifically by the \c MasterLoader class and \c Rdata implementation -/// classes. -/// -/// \note The error handling policy of this class is slightly different from -/// that of other classes of this library. We generally throw an exception -/// for an invalid input, whether it's more likely to be a program error or -/// a "user error", which means an invalid input that comes from outside of -/// the library. But, this class returns an error code for some certain -/// types of user errors instead of throwing an exception. Such cases include -/// a syntax error identified by the lexer or a misspelled file name that -/// causes a system error at the time of open. This is based on the assumption -/// that the main user of this class is a parser of master files, where -/// we want to give an option to ignore some non fatal errors and continue -/// the parsing. This will be useful if it just performs overall error -/// checks on a master file. When the (immediate) caller needs to do explicit -/// error handling, exceptions are not that a useful tool for error reporting -/// because we cannot separate the normal and error cases anyway, which would -/// be one major advantage when we use exceptions. And, exceptions are -/// generally more expensive, either when it happens or just by being able -/// to handle with \c try and \c catch (depending on the underlying -/// implementation of the exception handling). For these reasons, some of -/// this class does not throw for an error that would be reported as an -/// exception in other classes. -class MasterLexer { - friend class master_lexer_internal::State; -public: - /// \brief Exception thrown when we fail to read from the input - /// stream or file. - struct ReadError : public Unexpected { - ReadError(const char* file, size_t line, const char* what) : - Unexpected(file, line, what) - {} - }; - - class Token; // we define it separately for better readability - - /// \brief Options for getNextToken. - /// - /// A compound option, indicating multiple options are set, can be - /// specified using the logical OR operator (operator|()). - enum Options { - NONE = 0, ///< No option - INITIAL_WS = 1, ///< recognize begin-of-line spaces after an - ///< end-of-line - QSTRING = 2, ///< recognize quoted string - NUMBER = 4 ///< recognize numeric text as integer - }; - - /// \brief The constructor. - /// - /// \throw std::bad_alloc Internal resource allocation fails (rare case). - MasterLexer(); - - /// \brief The destructor. - /// - /// It internally closes any remaining input sources. - ~MasterLexer(); - - /// \brief Open a file and make it the current input source of MasterLexer. - /// - /// The opened file can be explicitly closed by the \c popSource() method; - /// if \c popSource() is not called within the lifetime of the - /// \c MasterLexer, it will be closed in the destructor. - /// - /// In the case possible system errors in opening the file (most likely - /// because of specifying a non-existent or unreadable file), it returns - /// false, and if the optional \c error parameter is non NULL, it will be - /// set to a description of the error (any existing content of the string - /// will be discarded). If opening the file succeeds, the given - /// \c error parameter will be intact. - /// - /// Note that this method has two styles of error reporting: one by - /// returning \c false (and setting \c error optionally) and the other - /// by throwing an exception. See the note for the class description - /// about the distinction. - /// - /// \throw InvalidParameter filename is NULL - /// \param filename A non NULL string specifying a master file - /// \param error If non null, a placeholder to set error description in - /// case of failure. - /// - /// \return true if pushing the file succeeds; false otherwise. - bool pushSource(const char* filename, std::string* error = NULL); - - /// \brief Make the given stream the current input source of MasterLexer. - /// - /// The caller still holds the ownership of the passed stream; it's the - /// caller's responsibility to keep it valid as long as it's used in - /// \c MasterLexer or to release any resource for the stream after that. - /// The caller can explicitly tell \c MasterLexer to stop using the - /// stream by calling the \c popSource() method. - /// - /// \param input An input stream object that produces textual - /// representation of DNS RRs. - void pushSource(std::istream& input); - - /// \brief Stop using the most recently opened input source (file or - /// stream). - /// - /// If it's a file, the previously opened file will be closed internally. - /// If it's a stream, \c MasterLexer will simply stop using - /// the stream; the caller can assume it will be never used in - /// \c MasterLexer thereafter. - /// - /// This method must not be called when there is no source pushed for - /// \c MasterLexer. This method is otherwise exception free. - /// - /// \throw isc::InvalidOperation Called with no pushed source. - void popSource(); - - /// \brief Return the name of the current input source name. - /// - /// If it's a file, it will be the C string given at the corresponding - /// \c pushSource() call, that is, its filename. If it's a stream, it will - /// be formatted as \c "stream-%p" where \c %p is hex representation - /// of the address of the stream object. - /// - /// If there is no opened source at the time of the call, this method - /// returns an empty string. - /// - /// \throw std::bad_alloc Resource allocation failed for string - /// construction (rare case) - /// - /// \return A string representation of the current source (see the - /// description) - std::string getSourceName() const; - - /// \brief Return the input source line number. - /// - /// If there is an opened source, the return value will be a non-0 - /// integer indicating the line number of the current source where - /// the \c MasterLexer is currently working. The expected usage of - /// this value is to print a helpful error message when parsing fails - /// by specifically identifying the position of the error. - /// - /// If there is no opened source at the time of the call, this method - /// returns 0. - /// - /// \throw None - /// - /// \return The current line number of the source (see the description) - size_t getSourceLine() const; - - /// \brief Parse and return another token from the input. - /// - /// It reads a bit of the last opened source and produces another token - /// found in it. - /// - /// This method does not provide the strong exception guarantee. Generally, - /// if it throws, the object should not be used any more and should be - /// discarded. It was decided all the exceptions thrown from here are - /// serious enough that aborting the loading process is the only reasonable - /// recovery anyway, so the strong exception guarantee is not needed. - /// - /// \param options The options can be used to modify the tokenization. - /// The method can be made reporting things which are usually ignored - /// by this parameter. Multiple options can be passed at once by - /// bitwise or (eg. option1 | option 2). See description of available - /// options. - /// \return Next token found in the input. Note that the token refers to - /// some internal data in the lexer. It is valid only until - /// getNextToken or ungetToken is called. Also, the token becomes - /// invalid when the lexer is destroyed. - /// \throw isc::InvalidOperation in case the source is not available. This - /// may mean the pushSource() has not been called yet, or that the - /// current source has been read past the end. - /// \throw ReadError in case there's problem reading from the underlying - /// source (eg. I/O error in the file on the disk). - /// \throw std::bad_alloc in case allocation of some internal resources - /// or the token fail. - const Token& getNextToken(Options options = NONE); - - /// \brief Return the last token back to the lexer. - /// - /// The method undoes the lasts call to getNextToken(). If you call the - /// getNextToken() again with the same options, it'll return the same - /// token. If the options are different, it may return a different token, - /// but it acts as if the previous getNextToken() was never called. - /// - /// It is possible to return only one token back in time (you can't call - /// ungetToken() twice in a row without calling getNextToken() in between - /// successfully). - /// - /// It does not work after change of source (by pushSource or popSource). - /// - /// \throw isc::InvalidOperation If called second time in a row or if - /// getNextToken() was not called since the last change of the source. - void ungetToken(); - -private: - struct MasterLexerImpl; - MasterLexerImpl* impl_; -}; - -/// \brief Operator to combine \c MasterLexer options -/// -/// This is a trivial shortcut so that compound options can be specified -/// in an intuitive way. -inline MasterLexer::Options -operator|(MasterLexer::Options o1, MasterLexer::Options o2) { - return (static_cast( - static_cast(o1) | static_cast(o2))); -} - /// \brief Tokens for \c MasterLexer /// /// This is a simple value-class encapsulating a type of a lexer token and @@ -261,7 +42,7 @@ operator|(MasterLexer::Options o1, MasterLexer::Options o2) { /// (using the default version of copy constructor and assignment operator), /// but it's mainly for internal implementation convenience. Applications will /// simply refer to Token object as a reference via the \c MasterLexer class. -class MasterLexer::Token { +class MasterToken { public: /// \brief Enumeration for token types /// @@ -318,7 +99,7 @@ public: /// \throw InvalidParameter A value type token is specified. /// \param type The type of the token. It must indicate a non-value /// type (not larger than \c NOVALUE_TYPE_MAX). - explicit Token(Type type) : type_(type) { + explicit MasterToken(Type type) : type_(type) { if (type > NOVALUE_TYPE_MAX) { isc_throw(InvalidParameter, "Token per-type constructor " "called with invalid type: " << type); @@ -340,7 +121,7 @@ public: /// \param str_beg The start address of the string /// \param str_len The size of the string in bytes /// \param quoted true if it's a quoted string; false otherwise. - Token(const char* str_beg, size_t str_len, bool quoted = false) : + MasterToken(const char* str_beg, size_t str_len, bool quoted = false) : type_(quoted ? QSTRING : STRING) { val_.str_region_.beg = str_beg; @@ -351,7 +132,7 @@ public: /// /// \brief number An unsigned 32-bit integer corresponding to the token /// value. - explicit Token(uint32_t number) : type_(NUMBER) { + explicit MasterToken(uint32_t number) : type_(NUMBER) { val_.number_ = number; } @@ -359,7 +140,7 @@ public: /// /// \throw InvalidParameter Invalid error code value is specified. /// \brief error_code A pre-defined constant of \c ErrorCode. - explicit Token(ErrorCode error_code) : type_(ERROR) { + explicit MasterToken(ErrorCode error_code) : type_(ERROR) { if (!(error_code < MAX_ERROR_CODE)) { isc_throw(InvalidParameter, "Invalid master lexer error code: " << error_code); @@ -476,6 +257,223 @@ private: } val_; }; +/// \brief Tokenizer for parsing DNS master files. +/// +/// The \c MasterLexer class provides tokenize interfaces for parsing DNS +/// master files. It understands some special rules of master files as +/// defined in RFC 1035, such as comments, character escaping, and multi-line +/// data, and provides the user application with the actual data in a +/// more convenient form such as a std::string object. +/// +/// In order to support the $INCLUDE notation, this class is designed to be +/// able to operate on multiple files or input streams in the nested way. +/// The \c pushSource() and \c popSource() methods correspond to the push +/// and pop operations. +/// +/// While this class is public, it is less likely to be used by normal +/// applications; it's mainly expected to be used within this library, +/// specifically by the \c MasterLoader class and \c Rdata implementation +/// classes. +/// +/// \note The error handling policy of this class is slightly different from +/// that of other classes of this library. We generally throw an exception +/// for an invalid input, whether it's more likely to be a program error or +/// a "user error", which means an invalid input that comes from outside of +/// the library. But, this class returns an error code for some certain +/// types of user errors instead of throwing an exception. Such cases include +/// a syntax error identified by the lexer or a misspelled file name that +/// causes a system error at the time of open. This is based on the assumption +/// that the main user of this class is a parser of master files, where +/// we want to give an option to ignore some non fatal errors and continue +/// the parsing. This will be useful if it just performs overall error +/// checks on a master file. When the (immediate) caller needs to do explicit +/// error handling, exceptions are not that a useful tool for error reporting +/// because we cannot separate the normal and error cases anyway, which would +/// be one major advantage when we use exceptions. And, exceptions are +/// generally more expensive, either when it happens or just by being able +/// to handle with \c try and \c catch (depending on the underlying +/// implementation of the exception handling). For these reasons, some of +/// this class does not throw for an error that would be reported as an +/// exception in other classes. +class MasterLexer { + friend class master_lexer_internal::State; +public: + /// \brief Exception thrown when we fail to read from the input + /// stream or file. + struct ReadError : public Unexpected { + ReadError(const char* file, size_t line, const char* what) : + Unexpected(file, line, what) + {} + }; + + /// \brief Options for getNextToken. + /// + /// A compound option, indicating multiple options are set, can be + /// specified using the logical OR operator (operator|()). + enum Options { + NONE = 0, ///< No option + INITIAL_WS = 1, ///< recognize begin-of-line spaces after an + ///< end-of-line + QSTRING = 2, ///< recognize quoted string + NUMBER = 4 ///< recognize numeric text as integer + }; + + /// \brief The constructor. + /// + /// \throw std::bad_alloc Internal resource allocation fails (rare case). + MasterLexer(); + + /// \brief The destructor. + /// + /// It internally closes any remaining input sources. + ~MasterLexer(); + + /// \brief Open a file and make it the current input source of MasterLexer. + /// + /// The opened file can be explicitly closed by the \c popSource() method; + /// if \c popSource() is not called within the lifetime of the + /// \c MasterLexer, it will be closed in the destructor. + /// + /// In the case possible system errors in opening the file (most likely + /// because of specifying a non-existent or unreadable file), it returns + /// false, and if the optional \c error parameter is non NULL, it will be + /// set to a description of the error (any existing content of the string + /// will be discarded). If opening the file succeeds, the given + /// \c error parameter will be intact. + /// + /// Note that this method has two styles of error reporting: one by + /// returning \c false (and setting \c error optionally) and the other + /// by throwing an exception. See the note for the class description + /// about the distinction. + /// + /// \throw InvalidParameter filename is NULL + /// \param filename A non NULL string specifying a master file + /// \param error If non null, a placeholder to set error description in + /// case of failure. + /// + /// \return true if pushing the file succeeds; false otherwise. + bool pushSource(const char* filename, std::string* error = NULL); + + /// \brief Make the given stream the current input source of MasterLexer. + /// + /// The caller still holds the ownership of the passed stream; it's the + /// caller's responsibility to keep it valid as long as it's used in + /// \c MasterLexer or to release any resource for the stream after that. + /// The caller can explicitly tell \c MasterLexer to stop using the + /// stream by calling the \c popSource() method. + /// + /// \param input An input stream object that produces textual + /// representation of DNS RRs. + void pushSource(std::istream& input); + + /// \brief Stop using the most recently opened input source (file or + /// stream). + /// + /// If it's a file, the previously opened file will be closed internally. + /// If it's a stream, \c MasterLexer will simply stop using + /// the stream; the caller can assume it will be never used in + /// \c MasterLexer thereafter. + /// + /// This method must not be called when there is no source pushed for + /// \c MasterLexer. This method is otherwise exception free. + /// + /// \throw isc::InvalidOperation Called with no pushed source. + void popSource(); + + /// \brief Return the name of the current input source name. + /// + /// If it's a file, it will be the C string given at the corresponding + /// \c pushSource() call, that is, its filename. If it's a stream, it will + /// be formatted as \c "stream-%p" where \c %p is hex representation + /// of the address of the stream object. + /// + /// If there is no opened source at the time of the call, this method + /// returns an empty string. + /// + /// \throw std::bad_alloc Resource allocation failed for string + /// construction (rare case) + /// + /// \return A string representation of the current source (see the + /// description) + std::string getSourceName() const; + + /// \brief Return the input source line number. + /// + /// If there is an opened source, the return value will be a non-0 + /// integer indicating the line number of the current source where + /// the \c MasterLexer is currently working. The expected usage of + /// this value is to print a helpful error message when parsing fails + /// by specifically identifying the position of the error. + /// + /// If there is no opened source at the time of the call, this method + /// returns 0. + /// + /// \throw None + /// + /// \return The current line number of the source (see the description) + size_t getSourceLine() const; + + /// \brief Parse and return another token from the input. + /// + /// It reads a bit of the last opened source and produces another token + /// found in it. + /// + /// This method does not provide the strong exception guarantee. Generally, + /// if it throws, the object should not be used any more and should be + /// discarded. It was decided all the exceptions thrown from here are + /// serious enough that aborting the loading process is the only reasonable + /// recovery anyway, so the strong exception guarantee is not needed. + /// + /// \param options The options can be used to modify the tokenization. + /// The method can be made reporting things which are usually ignored + /// by this parameter. Multiple options can be passed at once by + /// bitwise or (eg. option1 | option 2). See description of available + /// options. + /// \return Next token found in the input. Note that the token refers to + /// some internal data in the lexer. It is valid only until + /// getNextToken or ungetToken is called. Also, the token becomes + /// invalid when the lexer is destroyed. + /// \throw isc::InvalidOperation in case the source is not available. This + /// may mean the pushSource() has not been called yet, or that the + /// current source has been read past the end. + /// \throw ReadError in case there's problem reading from the underlying + /// source (eg. I/O error in the file on the disk). + /// \throw std::bad_alloc in case allocation of some internal resources + /// or the token fail. + const MasterToken& getNextToken(Options options = NONE); + + /// \brief Return the last token back to the lexer. + /// + /// The method undoes the lasts call to getNextToken(). If you call the + /// getNextToken() again with the same options, it'll return the same + /// token. If the options are different, it may return a different token, + /// but it acts as if the previous getNextToken() was never called. + /// + /// It is possible to return only one token back in time (you can't call + /// ungetToken() twice in a row without calling getNextToken() in between + /// successfully). + /// + /// It does not work after change of source (by pushSource or popSource). + /// + /// \throw isc::InvalidOperation If called second time in a row or if + /// getNextToken() was not called since the last change of the source. + void ungetToken(); + +private: + struct MasterLexerImpl; + MasterLexerImpl* impl_; +}; + +/// \brief Operator to combine \c MasterLexer options +/// +/// This is a trivial shortcut so that compound options can be specified +/// in an intuitive way. +inline MasterLexer::Options +operator|(MasterLexer::Options o1, MasterLexer::Options o2) { + return (static_cast( + static_cast(o1) | static_cast(o2))); +} + } // namespace dns } // namespace isc #endif // MASTER_LEXER_H diff --git a/src/lib/dns/master_lexer_state.h b/src/lib/dns/master_lexer_state.h index 9ddd68a598..f296c1c810 100644 --- a/src/lib/dns/master_lexer_state.h +++ b/src/lib/dns/master_lexer_state.h @@ -119,7 +119,7 @@ public: /// purposes. ///@{ bool wasLastEOL(const MasterLexer& lexer) const; - const MasterLexer::Token& getToken(const MasterLexer& lexer) const; + const MasterToken& getToken(const MasterLexer& lexer) const; size_t getParenCount(const MasterLexer& lexer) const; ///@} diff --git a/src/lib/dns/tests/master_lexer_state_unittest.cc b/src/lib/dns/tests/master_lexer_state_unittest.cc index d8a6b669b9..7645ede198 100644 --- a/src/lib/dns/tests/master_lexer_state_unittest.cc +++ b/src/lib/dns/tests/master_lexer_state_unittest.cc @@ -24,7 +24,7 @@ using namespace isc::dns; using namespace master_lexer_internal; namespace { -typedef MasterLexer::Token Token; // shortcut +typedef MasterToken Token; // shortcut class MasterLexerStateTest : public ::testing::Test { protected: @@ -260,7 +260,7 @@ TEST_F(MasterLexerStateTest, crlf) { // Commonly used check for string related test cases, checking if the given // token has expected values. void -stringTokenCheck(const std::string& expected, const MasterLexer::Token& token, +stringTokenCheck(const std::string& expected, const MasterToken& token, bool quoted = false) { EXPECT_EQ(quoted ? Token::QSTRING : Token::STRING, token.getType()); diff --git a/src/lib/dns/tests/master_lexer_token_unittest.cc b/src/lib/dns/tests/master_lexer_token_unittest.cc index 1f022dfc21..0fc7ff9519 100644 --- a/src/lib/dns/tests/master_lexer_token_unittest.cc +++ b/src/lib/dns/tests/master_lexer_token_unittest.cc @@ -31,27 +31,27 @@ const size_t TEST_STRING_LEN = sizeof(TEST_STRING) - 1; class MasterLexerTokenTest : public ::testing::Test { protected: MasterLexerTokenTest() : - token_eof(MasterLexer::Token::END_OF_FILE), + token_eof(MasterToken::END_OF_FILE), token_str(TEST_STRING, TEST_STRING_LEN), token_num(42), - token_err(MasterLexer::Token::UNEXPECTED_END) + token_err(MasterToken::UNEXPECTED_END) {} - const MasterLexer::Token token_eof; // an example of non-value type token - const MasterLexer::Token token_str; - const MasterLexer::Token token_num; - const MasterLexer::Token token_err; + const MasterToken token_eof; // an example of non-value type token + const MasterToken token_str; + const MasterToken token_num; + const MasterToken token_err; }; TEST_F(MasterLexerTokenTest, strings) { // basic construction and getter checks - EXPECT_EQ(MasterLexer::Token::STRING, token_str.getType()); + EXPECT_EQ(MasterToken::STRING, token_str.getType()); EXPECT_EQ(std::string("string token"), token_str.getString()); std::string strval = "dummy"; // this should be replaced token_str.getString(strval); EXPECT_EQ(std::string("string token"), strval); - const MasterLexer::Token::StringRegion str_region = + const MasterToken::StringRegion str_region = token_str.getStringRegion(); EXPECT_EQ(TEST_STRING, str_region.beg); EXPECT_EQ(TEST_STRING_LEN, str_region.len); @@ -62,17 +62,17 @@ TEST_F(MasterLexerTokenTest, strings) { std::string expected_str("string token"); expected_str.push_back('\0'); EXPECT_EQ(expected_str, - MasterLexer::Token(TEST_STRING, TEST_STRING_LEN + 1).getString()); - MasterLexer::Token(TEST_STRING, TEST_STRING_LEN + 1).getString(strval); + MasterToken(TEST_STRING, TEST_STRING_LEN + 1).getString()); + MasterToken(TEST_STRING, TEST_STRING_LEN + 1).getString(strval); EXPECT_EQ(expected_str, strval); // Construct type of qstring - EXPECT_EQ(MasterLexer::Token::QSTRING, - MasterLexer::Token(TEST_STRING, sizeof(TEST_STRING), true). + EXPECT_EQ(MasterToken::QSTRING, + MasterToken(TEST_STRING, sizeof(TEST_STRING), true). getType()); // if we explicitly set 'quoted' to false, it should be normal string - EXPECT_EQ(MasterLexer::Token::STRING, - MasterLexer::Token(TEST_STRING, sizeof(TEST_STRING), false). + EXPECT_EQ(MasterToken::STRING, + MasterToken(TEST_STRING, sizeof(TEST_STRING), false). getType()); // getString/StringRegion() aren't allowed for non string(-variant) types @@ -86,23 +86,23 @@ TEST_F(MasterLexerTokenTest, strings) { TEST_F(MasterLexerTokenTest, numbers) { EXPECT_EQ(42, token_num.getNumber()); - EXPECT_EQ(MasterLexer::Token::NUMBER, token_num.getType()); + EXPECT_EQ(MasterToken::NUMBER, token_num.getType()); // It's copyable and assignable. - MasterLexer::Token token(token_num); + MasterToken token(token_num); EXPECT_EQ(42, token.getNumber()); - EXPECT_EQ(MasterLexer::Token::NUMBER, token.getType()); + EXPECT_EQ(MasterToken::NUMBER, token.getType()); token = token_num; EXPECT_EQ(42, token.getNumber()); - EXPECT_EQ(MasterLexer::Token::NUMBER, token.getType()); + EXPECT_EQ(MasterToken::NUMBER, token.getType()); // it's okay to replace it with a different type of token token = token_eof; - EXPECT_EQ(MasterLexer::Token::END_OF_FILE, token.getType()); + EXPECT_EQ(MasterToken::END_OF_FILE, token.getType()); // Possible max value - token = MasterLexer::Token(0xffffffff); + token = MasterToken(0xffffffff); EXPECT_EQ(4294967295u, token.getNumber()); // getNumber() isn't allowed for non number types @@ -112,41 +112,34 @@ TEST_F(MasterLexerTokenTest, numbers) { TEST_F(MasterLexerTokenTest, novalues) { // Just checking we can construct them and getType() returns correct value. - EXPECT_EQ(MasterLexer::Token::END_OF_FILE, token_eof.getType()); - EXPECT_EQ(MasterLexer::Token::END_OF_LINE, - MasterLexer::Token(MasterLexer::Token::END_OF_LINE).getType()); - EXPECT_EQ(MasterLexer::Token::INITIAL_WS, - MasterLexer::Token(MasterLexer::Token::INITIAL_WS).getType()); + EXPECT_EQ(MasterToken::END_OF_FILE, token_eof.getType()); + EXPECT_EQ(MasterToken::END_OF_LINE, + MasterToken(MasterToken::END_OF_LINE).getType()); + EXPECT_EQ(MasterToken::INITIAL_WS, + MasterToken(MasterToken::INITIAL_WS).getType()); // Special types of tokens cannot have value-based types - EXPECT_THROW(MasterLexer::Token t(MasterLexer::Token::STRING), - isc::InvalidParameter); - EXPECT_THROW(MasterLexer::Token t(MasterLexer::Token::QSTRING), - isc::InvalidParameter); - EXPECT_THROW(MasterLexer::Token t(MasterLexer::Token::NUMBER), - isc::InvalidParameter); - EXPECT_THROW(MasterLexer::Token t(MasterLexer::Token::ERROR), - isc::InvalidParameter); + EXPECT_THROW(MasterToken t(MasterToken::STRING), isc::InvalidParameter); + EXPECT_THROW(MasterToken t(MasterToken::QSTRING), isc::InvalidParameter); + EXPECT_THROW(MasterToken t(MasterToken::NUMBER), isc::InvalidParameter); + EXPECT_THROW(MasterToken t(MasterToken::ERROR), isc::InvalidParameter); } TEST_F(MasterLexerTokenTest, errors) { - EXPECT_EQ(MasterLexer::Token::ERROR, token_err.getType()); - EXPECT_EQ(MasterLexer::Token::UNEXPECTED_END, token_err.getErrorCode()); + EXPECT_EQ(MasterToken::ERROR, token_err.getType()); + EXPECT_EQ(MasterToken::UNEXPECTED_END, token_err.getErrorCode()); EXPECT_EQ("unexpected end of input", token_err.getErrorText()); - EXPECT_EQ("lexer not started", - MasterLexer::Token(MasterLexer::Token::NOT_STARTED). + EXPECT_EQ("lexer not started", MasterToken(MasterToken::NOT_STARTED). getErrorText()); EXPECT_EQ("unbalanced parentheses", - MasterLexer::Token(MasterLexer::Token::UNBALANCED_PAREN). + MasterToken(MasterToken::UNBALANCED_PAREN). getErrorText()); - EXPECT_EQ("unbalanced quotes", - MasterLexer::Token(MasterLexer::Token::UNBALANCED_QUOTES). + EXPECT_EQ("unbalanced quotes", MasterToken(MasterToken::UNBALANCED_QUOTES). getErrorText()); - EXPECT_EQ("no token produced", - MasterLexer::Token(MasterLexer::Token::NO_TOKEN_PRODUCED). + EXPECT_EQ("no token produced", MasterToken(MasterToken::NO_TOKEN_PRODUCED). getErrorText()); EXPECT_EQ("number out of range", - MasterLexer::Token(MasterLexer::Token::NUMBER_OUT_OF_RANGE). + MasterLexer::Token(MasterToken::NUMBER_OUT_OF_RANGE). getErrorText()); // getErrorCode/Text() isn't allowed for non number types @@ -156,14 +149,13 @@ TEST_F(MasterLexerTokenTest, errors) { // Only the pre-defined error code is accepted. Hardcoding '6' (max code // + 1) is intentional; it'd be actually better if we notice it when we // update the enum list (which shouldn't happen too often). - EXPECT_THROW(MasterLexer::Token(MasterLexer::Token::ErrorCode(6)), + EXPECT_THROW(MasterToken(MasterToken::ErrorCode(6)), isc::InvalidParameter); // Check the coexistence of "from number" and "from error-code" // constructors won't cause confusion. - EXPECT_EQ(MasterLexer::Token::NUMBER, - MasterLexer::Token(static_cast( - MasterLexer::Token::NOT_STARTED)). + EXPECT_EQ(MasterToken::NUMBER, + MasterToken(static_cast(MasterToken::NOT_STARTED)). getType()); } } diff --git a/src/lib/dns/tests/master_lexer_unittest.cc b/src/lib/dns/tests/master_lexer_unittest.cc index eca6a73663..7bec68f359 100644 --- a/src/lib/dns/tests/master_lexer_unittest.cc +++ b/src/lib/dns/tests/master_lexer_unittest.cc @@ -141,19 +141,19 @@ TEST_F(MasterLexerTest, getNextToken) { lexer.pushSource(ss); // First, the newline should get out. - EXPECT_EQ(MasterLexer::Token::END_OF_LINE, lexer.getNextToken().getType()); + EXPECT_EQ(MasterToken::END_OF_LINE, lexer.getNextToken().getType()); // Then the whitespace, if we specify the option. - EXPECT_EQ(MasterLexer::Token::INITIAL_WS, + EXPECT_EQ(MasterToken::INITIAL_WS, lexer.getNextToken(MasterLexer::INITIAL_WS).getType()); // The newline - EXPECT_EQ(MasterLexer::Token::END_OF_LINE, lexer.getNextToken().getType()); + EXPECT_EQ(MasterToken::END_OF_LINE, lexer.getNextToken().getType()); // The (quoted) string - EXPECT_EQ(MasterLexer::Token::QSTRING, + EXPECT_EQ(MasterToken::QSTRING, lexer.getNextToken(MasterLexer::QSTRING).getType()); // And the end of line and file - EXPECT_EQ(MasterLexer::Token::END_OF_LINE, lexer.getNextToken().getType()); - EXPECT_EQ(MasterLexer::Token::END_OF_FILE, lexer.getNextToken().getType()); + EXPECT_EQ(MasterToken::END_OF_LINE, lexer.getNextToken().getType()); + EXPECT_EQ(MasterToken::END_OF_FILE, lexer.getNextToken().getType()); } // Test we correctly find end of file. @@ -162,12 +162,12 @@ TEST_F(MasterLexerTest, eof) { lexer.pushSource(ss); // The first one is found to be EOF - EXPECT_EQ(MasterLexer::Token::END_OF_FILE, lexer.getNextToken().getType()); + EXPECT_EQ(MasterToken::END_OF_FILE, lexer.getNextToken().getType()); // And it stays on EOF for any following attempts - EXPECT_EQ(MasterLexer::Token::END_OF_FILE, lexer.getNextToken().getType()); + EXPECT_EQ(MasterToken::END_OF_FILE, lexer.getNextToken().getType()); // And we can step back one token, but that is the EOF too. lexer.ungetToken(); - EXPECT_EQ(MasterLexer::Token::END_OF_FILE, lexer.getNextToken().getType()); + EXPECT_EQ(MasterToken::END_OF_FILE, lexer.getNextToken().getType()); } // Check we properly return error when there's an opened parentheses and no @@ -177,12 +177,12 @@ TEST_F(MasterLexerTest, getUnbalancedParen) { lexer.pushSource(ss); // The string gets out first - EXPECT_EQ(MasterLexer::Token::STRING, lexer.getNextToken().getType()); + EXPECT_EQ(MasterToken::STRING, lexer.getNextToken().getType()); // Then an unbalanced parenthesis - EXPECT_EQ(MasterLexer::Token::UNBALANCED_PAREN, + EXPECT_EQ(MasterToken::UNBALANCED_PAREN, lexer.getNextToken().getErrorCode()); // And then EOF - EXPECT_EQ(MasterLexer::Token::END_OF_FILE, lexer.getNextToken().getType()); + EXPECT_EQ(MasterToken::END_OF_FILE, lexer.getNextToken().getType()); } // Check we properly return error when there's an opened quoted string and no @@ -192,10 +192,10 @@ TEST_F(MasterLexerTest, getUnbalancedString) { lexer.pushSource(ss); // Then an unbalanced qstring (reported as an unexpected end) - EXPECT_EQ(MasterLexer::Token::UNEXPECTED_END, + EXPECT_EQ(MasterToken::UNEXPECTED_END, lexer.getNextToken(MasterLexer::QSTRING).getErrorCode()); // And then EOF - EXPECT_EQ(MasterLexer::Token::END_OF_FILE, lexer.getNextToken().getType()); + EXPECT_EQ(MasterToken::END_OF_FILE, lexer.getNextToken().getType()); } // Test ungetting tokens works @@ -204,28 +204,28 @@ TEST_F(MasterLexerTest, ungetToken) { lexer.pushSource(ss); // Try getting the newline - EXPECT_EQ(MasterLexer::Token::END_OF_LINE, lexer.getNextToken().getType()); + EXPECT_EQ(MasterToken::END_OF_LINE, lexer.getNextToken().getType()); // Return it and get again lexer.ungetToken(); - EXPECT_EQ(MasterLexer::Token::END_OF_LINE, lexer.getNextToken().getType()); + EXPECT_EQ(MasterToken::END_OF_LINE, lexer.getNextToken().getType()); // Get the string and return it back - EXPECT_EQ(MasterLexer::Token::QSTRING, + EXPECT_EQ(MasterToken::QSTRING, lexer.getNextToken(MasterLexer::QSTRING).getType()); lexer.ungetToken(); // But if we change the options, it honors them - EXPECT_EQ(MasterLexer::Token::INITIAL_WS, + EXPECT_EQ(MasterToken::INITIAL_WS, lexer.getNextToken(MasterLexer::QSTRING | MasterLexer::INITIAL_WS).getType()); // Get to the "more" string - EXPECT_EQ(MasterLexer::Token::QSTRING, + EXPECT_EQ(MasterToken::QSTRING, lexer.getNextToken(MasterLexer::QSTRING).getType()); - EXPECT_EQ(MasterLexer::Token::STRING, + EXPECT_EQ(MasterToken::STRING, lexer.getNextToken(MasterLexer::QSTRING).getType()); // Return it back. It should get inside the parentheses. // Upon next attempt to get it again, the newline inside the parentheses // should be still ignored. lexer.ungetToken(); - EXPECT_EQ(MasterLexer::Token::STRING, + EXPECT_EQ(MasterToken::STRING, lexer.getNextToken(MasterLexer::QSTRING).getType()); } @@ -235,16 +235,16 @@ TEST_F(MasterLexerTest, ungetRealOptions) { ss << "\n \n"; lexer.pushSource(ss); // Skip the first newline - EXPECT_EQ(MasterLexer::Token::END_OF_LINE, lexer.getNextToken().getType()); + EXPECT_EQ(MasterToken::END_OF_LINE, lexer.getNextToken().getType()); // If we call it the usual way, it skips up to the newline and returns // it - EXPECT_EQ(MasterLexer::Token::END_OF_LINE, lexer.getNextToken().getType()); + EXPECT_EQ(MasterToken::END_OF_LINE, lexer.getNextToken().getType()); // Now we return it. If we call it again, but with different options, // we get the initial whitespace. lexer.ungetToken(); - EXPECT_EQ(MasterLexer::Token::INITIAL_WS, + EXPECT_EQ(MasterToken::INITIAL_WS, lexer.getNextToken(MasterLexer::INITIAL_WS).getType()); } @@ -253,7 +253,7 @@ TEST_F(MasterLexerTest, ungetTwice) { ss << "\n"; lexer.pushSource(ss); - EXPECT_EQ(MasterLexer::Token::END_OF_LINE, lexer.getNextToken().getType()); + EXPECT_EQ(MasterToken::END_OF_LINE, lexer.getNextToken().getType()); // Unget the token. It can be done once lexer.ungetToken(); // But not twice @@ -271,14 +271,14 @@ TEST_F(MasterLexerTest, ungetBeforeGet) { TEST_F(MasterLexerTest, ungetAfterSwitch) { ss << "\n\n"; lexer.pushSource(ss); - EXPECT_EQ(MasterLexer::Token::END_OF_LINE, lexer.getNextToken().getType()); + EXPECT_EQ(MasterToken::END_OF_LINE, lexer.getNextToken().getType()); // Switch the source std::stringstream ss2; ss2 << "\n\n"; lexer.pushSource(ss2); EXPECT_THROW(lexer.ungetToken(), isc::InvalidOperation); // We can get from the new source - EXPECT_EQ(MasterLexer::Token::END_OF_LINE, lexer.getNextToken().getType()); + EXPECT_EQ(MasterToken::END_OF_LINE, lexer.getNextToken().getType()); // And when we drop the current source, we can't unget again lexer.popSource(); EXPECT_THROW(lexer.ungetToken(), isc::InvalidOperation);