struct MasterLexer::MasterLexerImpl {
- MasterLexerImpl() : source_(NULL), token_(Token::NOT_STARTED),
+ MasterLexerImpl() : source_(NULL), token_(MasterToken::NOT_STARTED),
paren_count_(0), last_was_eol_(false),
has_previous_(false),
previous_paren_count_(0),
std::vector<InputSourcePtr> sources_;
InputSource* source_; // current source (NULL if sources_ is empty)
- Token token_; // currently recognized token (set by a state)
+ MasterToken token_; // currently recognized token (set by a state)
std::vector<char> data_; // placeholder for string data
// These are used in states, and defined here only as a placeholder.
return (impl_->sources_.back()->getCurrentLine());
}
-const MasterLexer::Token&
+const MasterToken&
MasterLexer::getNextToken(Options options) {
if (impl_->source_ == NULL) {
isc_throw(isc::InvalidOperation, "No source to read tokens from");
impl_->has_previous_ = true;
// Reset the token now. This is to check a token was actually produced.
// This is debugging aid.
- impl_->token_ = Token(Token::NO_TOKEN_PRODUCED);
+ impl_->token_ = MasterToken(MasterToken::NO_TOKEN_PRODUCED);
// And get the token
// This actually handles EOF internally too.
}
// Make sure a token was produced. Since this Can Not Happen, we assert
// here instead of throwing.
- assert(impl_->token_.getType() != Token::ERROR ||
- impl_->token_.getErrorCode() != Token::NO_TOKEN_PRODUCED);
+ assert(impl_->token_.getType() != MasterToken::ERROR ||
+ impl_->token_.getErrorCode() != MasterToken::NO_TOKEN_PRODUCED);
return (impl_->token_);
}
} // end unnamed namespace
std::string
-MasterLexer::Token::getErrorText() const {
+MasterToken::getErrorText() const {
if (type_ != ERROR) {
isc_throw(InvalidOperation,
- "Token::getErrorText() for non error type");
+ "MasterToken::getErrorText() for non error type");
}
// The class integrity ensures the following:
// Note that these need to be defined here so that they can refer to
// the details of MasterLexerImpl.
-typedef MasterLexer::Token Token; // convenience shortcut
-
bool
State::wasLastEOL(const MasterLexer& lexer) const {
return (lexer.impl_->last_was_eol_);
}
-const MasterLexer::Token&
+const MasterToken&
State::getToken(const MasterLexer& lexer) const {
return (lexer.impl_->token_);
}
if (c != '\n') {
getLexerImpl(lexer)->source_->ungetChar();
}
- getLexerImpl(lexer)->token_ = Token(Token::END_OF_LINE);
+ getLexerImpl(lexer)->token_ = MasterToken(MasterToken::END_OF_LINE);
getLexerImpl(lexer)->last_was_eol_ = true;
}
};
if (c == InputSource::END_OF_STREAM) {
lexerimpl.last_was_eol_ = false;
if (paren_count != 0) {
- lexerimpl.token_ = Token(Token::UNBALANCED_PAREN);
+ lexerimpl.token_ = MasterToken(MasterToken::UNBALANCED_PAREN);
paren_count = 0; // reset to 0; this helps in lenient mode.
return (NULL);
}
- lexerimpl.token_ = Token(Token::END_OF_FILE);
+ lexerimpl.token_ = MasterToken(MasterToken::END_OF_FILE);
return (NULL);
} else if (c == ' ' || c == '\t') {
// If requested and we are not in (), recognize the initial space.
if (lexerimpl.last_was_eol_ && paren_count == 0 &&
(options & MasterLexer::INITIAL_WS) != 0) {
lexerimpl.last_was_eol_ = false;
- lexerimpl.token_ = Token(Token::INITIAL_WS);
+ lexerimpl.token_ = MasterToken(MasterToken::INITIAL_WS);
return (NULL);
}
} else if (c == '\n') {
lexerimpl.last_was_eol_ = true;
if (paren_count == 0) { // we don't recognize EOL if we are in ()
- lexerimpl.token_ = Token(Token::END_OF_LINE);
+ lexerimpl.token_ = MasterToken(MasterToken::END_OF_LINE);
return (NULL);
}
} else if (c == '\r') {
} else if (c == ')') {
lexerimpl.last_was_eol_ = false;
if (paren_count == 0) {
- lexerimpl.token_ = Token(Token::UNBALANCED_PAREN);
+ lexerimpl.token_ = MasterToken(MasterToken::UNBALANCED_PAREN);
return (NULL);
}
--paren_count;
if (getLexerImpl(lexer)->isTokenEnd(c, escaped)) {
getLexerImpl(lexer)->source_->ungetChar();
getLexerImpl(lexer)->token_ =
- MasterLexer::Token(&data.at(0), data.size());
+ MasterToken(&data.at(0), data.size());
return;
}
escaped = (c == '\\' && !escaped);
void
QString::handle(MasterLexer& lexer) const {
- MasterLexer::Token& token = getLexerImpl(lexer)->token_;
+ MasterToken& token = getLexerImpl(lexer)->token_;
std::vector<char>& data = getLexerImpl(lexer)->data_;
data.clear();
while (true) {
const int c = getLexerImpl(lexer)->source_->getChar();
if (c == InputSource::END_OF_STREAM) {
- token = Token(Token::UNEXPECTED_END);
+ token = MasterToken(MasterToken::UNEXPECTED_END);
return;
} else if (c == '"') {
if (escaped) {
escaped = false;
data.back() = '"';
} else {
- token = MasterLexer::Token(&data.at(0), data.size(), true);
+ token = MasterToken(&data.at(0), data.size(), true);
return;
}
} else if (c == '\n' && !escaped) {
getLexerImpl(lexer)->source_->ungetChar();
- token = Token(Token::UNBALANCED_QUOTES);
+ token = MasterToken(MasterToken::UNBALANCED_QUOTES);
return;
} else {
escaped = (c == '\\' && !escaped);
void
Number::handle(MasterLexer& lexer) const {
- MasterLexer::Token& token = getLexerImpl(lexer)->token_;
+ MasterToken& token = getLexerImpl(lexer)->token_;
// It may yet turn out to be a string, so we first
// collect all the data
} catch (const boost::bad_lexical_cast&) {
// Since we already know we have only digits,
// range should be the only possible problem.
- token = Token(Token::NUMBER_OUT_OF_RANGE);
+ token = Token(MasterToken::NUMBER_OUT_OF_RANGE);
}
} else {
- token = MasterLexer::Token(&data.at(0),
- data.size());
+ token = MasterToken(&data.at(0), data.size());
}
return;
}
class State;
}
-/// \brief Tokenizer for parsing DNS master files.
-///
-/// The \c MasterLexer class provides tokenize interfaces for parsing DNS
-/// master files. It understands some special rules of master files as
-/// defined in RFC 1035, such as comments, character escaping, and multi-line
-/// data, and provides the user application with the actual data in a
-/// more convenient form such as a std::string object.
-///
-/// In order to support the $INCLUDE notation, this class is designed to be
-/// able to operate on multiple files or input streams in the nested way.
-/// The \c pushSource() and \c popSource() methods correspond to the push
-/// and pop operations.
-///
-/// While this class is public, it is less likely to be used by normal
-/// applications; it's mainly expected to be used within this library,
-/// specifically by the \c MasterLoader class and \c Rdata implementation
-/// classes.
-///
-/// \note The error handling policy of this class is slightly different from
-/// that of other classes of this library. We generally throw an exception
-/// for an invalid input, whether it's more likely to be a program error or
-/// a "user error", which means an invalid input that comes from outside of
-/// the library. But, this class returns an error code for some certain
-/// types of user errors instead of throwing an exception. Such cases include
-/// a syntax error identified by the lexer or a misspelled file name that
-/// causes a system error at the time of open. This is based on the assumption
-/// that the main user of this class is a parser of master files, where
-/// we want to give an option to ignore some non fatal errors and continue
-/// the parsing. This will be useful if it just performs overall error
-/// checks on a master file. When the (immediate) caller needs to do explicit
-/// error handling, exceptions are not that a useful tool for error reporting
-/// because we cannot separate the normal and error cases anyway, which would
-/// be one major advantage when we use exceptions. And, exceptions are
-/// generally more expensive, either when it happens or just by being able
-/// to handle with \c try and \c catch (depending on the underlying
-/// implementation of the exception handling). For these reasons, some of
-/// this class does not throw for an error that would be reported as an
-/// exception in other classes.
-class MasterLexer {
- friend class master_lexer_internal::State;
-public:
- /// \brief Exception thrown when we fail to read from the input
- /// stream or file.
- struct ReadError : public Unexpected {
- ReadError(const char* file, size_t line, const char* what) :
- Unexpected(file, line, what)
- {}
- };
-
- class Token; // we define it separately for better readability
-
- /// \brief Options for getNextToken.
- ///
- /// A compound option, indicating multiple options are set, can be
- /// specified using the logical OR operator (operator|()).
- enum Options {
- NONE = 0, ///< No option
- INITIAL_WS = 1, ///< recognize begin-of-line spaces after an
- ///< end-of-line
- QSTRING = 2, ///< recognize quoted string
- NUMBER = 4 ///< recognize numeric text as integer
- };
-
- /// \brief The constructor.
- ///
- /// \throw std::bad_alloc Internal resource allocation fails (rare case).
- MasterLexer();
-
- /// \brief The destructor.
- ///
- /// It internally closes any remaining input sources.
- ~MasterLexer();
-
- /// \brief Open a file and make it the current input source of MasterLexer.
- ///
- /// The opened file can be explicitly closed by the \c popSource() method;
- /// if \c popSource() is not called within the lifetime of the
- /// \c MasterLexer, it will be closed in the destructor.
- ///
- /// In the case possible system errors in opening the file (most likely
- /// because of specifying a non-existent or unreadable file), it returns
- /// false, and if the optional \c error parameter is non NULL, it will be
- /// set to a description of the error (any existing content of the string
- /// will be discarded). If opening the file succeeds, the given
- /// \c error parameter will be intact.
- ///
- /// Note that this method has two styles of error reporting: one by
- /// returning \c false (and setting \c error optionally) and the other
- /// by throwing an exception. See the note for the class description
- /// about the distinction.
- ///
- /// \throw InvalidParameter filename is NULL
- /// \param filename A non NULL string specifying a master file
- /// \param error If non null, a placeholder to set error description in
- /// case of failure.
- ///
- /// \return true if pushing the file succeeds; false otherwise.
- bool pushSource(const char* filename, std::string* error = NULL);
-
- /// \brief Make the given stream the current input source of MasterLexer.
- ///
- /// The caller still holds the ownership of the passed stream; it's the
- /// caller's responsibility to keep it valid as long as it's used in
- /// \c MasterLexer or to release any resource for the stream after that.
- /// The caller can explicitly tell \c MasterLexer to stop using the
- /// stream by calling the \c popSource() method.
- ///
- /// \param input An input stream object that produces textual
- /// representation of DNS RRs.
- void pushSource(std::istream& input);
-
- /// \brief Stop using the most recently opened input source (file or
- /// stream).
- ///
- /// If it's a file, the previously opened file will be closed internally.
- /// If it's a stream, \c MasterLexer will simply stop using
- /// the stream; the caller can assume it will be never used in
- /// \c MasterLexer thereafter.
- ///
- /// This method must not be called when there is no source pushed for
- /// \c MasterLexer. This method is otherwise exception free.
- ///
- /// \throw isc::InvalidOperation Called with no pushed source.
- void popSource();
-
- /// \brief Return the name of the current input source name.
- ///
- /// If it's a file, it will be the C string given at the corresponding
- /// \c pushSource() call, that is, its filename. If it's a stream, it will
- /// be formatted as \c "stream-%p" where \c %p is hex representation
- /// of the address of the stream object.
- ///
- /// If there is no opened source at the time of the call, this method
- /// returns an empty string.
- ///
- /// \throw std::bad_alloc Resource allocation failed for string
- /// construction (rare case)
- ///
- /// \return A string representation of the current source (see the
- /// description)
- std::string getSourceName() const;
-
- /// \brief Return the input source line number.
- ///
- /// If there is an opened source, the return value will be a non-0
- /// integer indicating the line number of the current source where
- /// the \c MasterLexer is currently working. The expected usage of
- /// this value is to print a helpful error message when parsing fails
- /// by specifically identifying the position of the error.
- ///
- /// If there is no opened source at the time of the call, this method
- /// returns 0.
- ///
- /// \throw None
- ///
- /// \return The current line number of the source (see the description)
- size_t getSourceLine() const;
-
- /// \brief Parse and return another token from the input.
- ///
- /// It reads a bit of the last opened source and produces another token
- /// found in it.
- ///
- /// This method does not provide the strong exception guarantee. Generally,
- /// if it throws, the object should not be used any more and should be
- /// discarded. It was decided all the exceptions thrown from here are
- /// serious enough that aborting the loading process is the only reasonable
- /// recovery anyway, so the strong exception guarantee is not needed.
- ///
- /// \param options The options can be used to modify the tokenization.
- /// The method can be made reporting things which are usually ignored
- /// by this parameter. Multiple options can be passed at once by
- /// bitwise or (eg. option1 | option 2). See description of available
- /// options.
- /// \return Next token found in the input. Note that the token refers to
- /// some internal data in the lexer. It is valid only until
- /// getNextToken or ungetToken is called. Also, the token becomes
- /// invalid when the lexer is destroyed.
- /// \throw isc::InvalidOperation in case the source is not available. This
- /// may mean the pushSource() has not been called yet, or that the
- /// current source has been read past the end.
- /// \throw ReadError in case there's problem reading from the underlying
- /// source (eg. I/O error in the file on the disk).
- /// \throw std::bad_alloc in case allocation of some internal resources
- /// or the token fail.
- const Token& getNextToken(Options options = NONE);
-
- /// \brief Return the last token back to the lexer.
- ///
- /// The method undoes the lasts call to getNextToken(). If you call the
- /// getNextToken() again with the same options, it'll return the same
- /// token. If the options are different, it may return a different token,
- /// but it acts as if the previous getNextToken() was never called.
- ///
- /// It is possible to return only one token back in time (you can't call
- /// ungetToken() twice in a row without calling getNextToken() in between
- /// successfully).
- ///
- /// It does not work after change of source (by pushSource or popSource).
- ///
- /// \throw isc::InvalidOperation If called second time in a row or if
- /// getNextToken() was not called since the last change of the source.
- void ungetToken();
-
-private:
- struct MasterLexerImpl;
- MasterLexerImpl* impl_;
-};
-
-/// \brief Operator to combine \c MasterLexer options
-///
-/// This is a trivial shortcut so that compound options can be specified
-/// in an intuitive way.
-inline MasterLexer::Options
-operator|(MasterLexer::Options o1, MasterLexer::Options o2) {
- return (static_cast<MasterLexer::Options>(
- static_cast<unsigned>(o1) | static_cast<unsigned>(o2)));
-}
-
/// \brief Tokens for \c MasterLexer
///
/// This is a simple value-class encapsulating a type of a lexer token and
/// (using the default version of copy constructor and assignment operator),
/// but it's mainly for internal implementation convenience. Applications will
/// simply refer to Token object as a reference via the \c MasterLexer class.
-class MasterLexer::Token {
+class MasterToken {
public:
/// \brief Enumeration for token types
///
/// \throw InvalidParameter A value type token is specified.
/// \param type The type of the token. It must indicate a non-value
/// type (not larger than \c NOVALUE_TYPE_MAX).
- explicit Token(Type type) : type_(type) {
+ explicit MasterToken(Type type) : type_(type) {
if (type > NOVALUE_TYPE_MAX) {
isc_throw(InvalidParameter, "Token per-type constructor "
"called with invalid type: " << type);
/// \param str_beg The start address of the string
/// \param str_len The size of the string in bytes
/// \param quoted true if it's a quoted string; false otherwise.
- Token(const char* str_beg, size_t str_len, bool quoted = false) :
+ MasterToken(const char* str_beg, size_t str_len, bool quoted = false) :
type_(quoted ? QSTRING : STRING)
{
val_.str_region_.beg = str_beg;
///
/// \brief number An unsigned 32-bit integer corresponding to the token
/// value.
- explicit Token(uint32_t number) : type_(NUMBER) {
+ explicit MasterToken(uint32_t number) : type_(NUMBER) {
val_.number_ = number;
}
///
/// \throw InvalidParameter Invalid error code value is specified.
/// \brief error_code A pre-defined constant of \c ErrorCode.
- explicit Token(ErrorCode error_code) : type_(ERROR) {
+ explicit MasterToken(ErrorCode error_code) : type_(ERROR) {
if (!(error_code < MAX_ERROR_CODE)) {
isc_throw(InvalidParameter, "Invalid master lexer error code: "
<< error_code);
} val_;
};
+/// \brief Tokenizer for parsing DNS master files.
+///
+/// The \c MasterLexer class provides tokenize interfaces for parsing DNS
+/// master files. It understands some special rules of master files as
+/// defined in RFC 1035, such as comments, character escaping, and multi-line
+/// data, and provides the user application with the actual data in a
+/// more convenient form such as a std::string object.
+///
+/// In order to support the $INCLUDE notation, this class is designed to be
+/// able to operate on multiple files or input streams in the nested way.
+/// The \c pushSource() and \c popSource() methods correspond to the push
+/// and pop operations.
+///
+/// While this class is public, it is less likely to be used by normal
+/// applications; it's mainly expected to be used within this library,
+/// specifically by the \c MasterLoader class and \c Rdata implementation
+/// classes.
+///
+/// \note The error handling policy of this class is slightly different from
+/// that of other classes of this library. We generally throw an exception
+/// for an invalid input, whether it's more likely to be a program error or
+/// a "user error", which means an invalid input that comes from outside of
+/// the library. But, this class returns an error code for some certain
+/// types of user errors instead of throwing an exception. Such cases include
+/// a syntax error identified by the lexer or a misspelled file name that
+/// causes a system error at the time of open. This is based on the assumption
+/// that the main user of this class is a parser of master files, where
+/// we want to give an option to ignore some non fatal errors and continue
+/// the parsing. This will be useful if it just performs overall error
+/// checks on a master file. When the (immediate) caller needs to do explicit
+/// error handling, exceptions are not that a useful tool for error reporting
+/// because we cannot separate the normal and error cases anyway, which would
+/// be one major advantage when we use exceptions. And, exceptions are
+/// generally more expensive, either when it happens or just by being able
+/// to handle with \c try and \c catch (depending on the underlying
+/// implementation of the exception handling). For these reasons, some of
+/// this class does not throw for an error that would be reported as an
+/// exception in other classes.
+class MasterLexer {
+ friend class master_lexer_internal::State;
+public:
+ /// \brief Exception thrown when we fail to read from the input
+ /// stream or file.
+ struct ReadError : public Unexpected {
+ ReadError(const char* file, size_t line, const char* what) :
+ Unexpected(file, line, what)
+ {}
+ };
+
+ /// \brief Options for getNextToken.
+ ///
+ /// A compound option, indicating multiple options are set, can be
+ /// specified using the logical OR operator (operator|()).
+ enum Options {
+ NONE = 0, ///< No option
+ INITIAL_WS = 1, ///< recognize begin-of-line spaces after an
+ ///< end-of-line
+ QSTRING = 2, ///< recognize quoted string
+ NUMBER = 4 ///< recognize numeric text as integer
+ };
+
+ /// \brief The constructor.
+ ///
+ /// \throw std::bad_alloc Internal resource allocation fails (rare case).
+ MasterLexer();
+
+ /// \brief The destructor.
+ ///
+ /// It internally closes any remaining input sources.
+ ~MasterLexer();
+
+ /// \brief Open a file and make it the current input source of MasterLexer.
+ ///
+ /// The opened file can be explicitly closed by the \c popSource() method;
+ /// if \c popSource() is not called within the lifetime of the
+ /// \c MasterLexer, it will be closed in the destructor.
+ ///
+ /// In the case possible system errors in opening the file (most likely
+ /// because of specifying a non-existent or unreadable file), it returns
+ /// false, and if the optional \c error parameter is non NULL, it will be
+ /// set to a description of the error (any existing content of the string
+ /// will be discarded). If opening the file succeeds, the given
+ /// \c error parameter will be intact.
+ ///
+ /// Note that this method has two styles of error reporting: one by
+ /// returning \c false (and setting \c error optionally) and the other
+ /// by throwing an exception. See the note for the class description
+ /// about the distinction.
+ ///
+ /// \throw InvalidParameter filename is NULL
+ /// \param filename A non NULL string specifying a master file
+ /// \param error If non null, a placeholder to set error description in
+ /// case of failure.
+ ///
+ /// \return true if pushing the file succeeds; false otherwise.
+ bool pushSource(const char* filename, std::string* error = NULL);
+
+ /// \brief Make the given stream the current input source of MasterLexer.
+ ///
+ /// The caller still holds the ownership of the passed stream; it's the
+ /// caller's responsibility to keep it valid as long as it's used in
+ /// \c MasterLexer or to release any resource for the stream after that.
+ /// The caller can explicitly tell \c MasterLexer to stop using the
+ /// stream by calling the \c popSource() method.
+ ///
+ /// \param input An input stream object that produces textual
+ /// representation of DNS RRs.
+ void pushSource(std::istream& input);
+
+ /// \brief Stop using the most recently opened input source (file or
+ /// stream).
+ ///
+ /// If it's a file, the previously opened file will be closed internally.
+ /// If it's a stream, \c MasterLexer will simply stop using
+ /// the stream; the caller can assume it will be never used in
+ /// \c MasterLexer thereafter.
+ ///
+ /// This method must not be called when there is no source pushed for
+ /// \c MasterLexer. This method is otherwise exception free.
+ ///
+ /// \throw isc::InvalidOperation Called with no pushed source.
+ void popSource();
+
+ /// \brief Return the name of the current input source name.
+ ///
+ /// If it's a file, it will be the C string given at the corresponding
+ /// \c pushSource() call, that is, its filename. If it's a stream, it will
+ /// be formatted as \c "stream-%p" where \c %p is hex representation
+ /// of the address of the stream object.
+ ///
+ /// If there is no opened source at the time of the call, this method
+ /// returns an empty string.
+ ///
+ /// \throw std::bad_alloc Resource allocation failed for string
+ /// construction (rare case)
+ ///
+ /// \return A string representation of the current source (see the
+ /// description)
+ std::string getSourceName() const;
+
+ /// \brief Return the input source line number.
+ ///
+ /// If there is an opened source, the return value will be a non-0
+ /// integer indicating the line number of the current source where
+ /// the \c MasterLexer is currently working. The expected usage of
+ /// this value is to print a helpful error message when parsing fails
+ /// by specifically identifying the position of the error.
+ ///
+ /// If there is no opened source at the time of the call, this method
+ /// returns 0.
+ ///
+ /// \throw None
+ ///
+ /// \return The current line number of the source (see the description)
+ size_t getSourceLine() const;
+
+ /// \brief Parse and return another token from the input.
+ ///
+ /// It reads a bit of the last opened source and produces another token
+ /// found in it.
+ ///
+ /// This method does not provide the strong exception guarantee. Generally,
+ /// if it throws, the object should not be used any more and should be
+ /// discarded. It was decided all the exceptions thrown from here are
+ /// serious enough that aborting the loading process is the only reasonable
+ /// recovery anyway, so the strong exception guarantee is not needed.
+ ///
+ /// \param options The options can be used to modify the tokenization.
+ /// The method can be made reporting things which are usually ignored
+ /// by this parameter. Multiple options can be passed at once by
+ /// bitwise or (eg. option1 | option 2). See description of available
+ /// options.
+ /// \return Next token found in the input. Note that the token refers to
+ /// some internal data in the lexer. It is valid only until
+ /// getNextToken or ungetToken is called. Also, the token becomes
+ /// invalid when the lexer is destroyed.
+ /// \throw isc::InvalidOperation in case the source is not available. This
+ /// may mean the pushSource() has not been called yet, or that the
+ /// current source has been read past the end.
+ /// \throw ReadError in case there's problem reading from the underlying
+ /// source (eg. I/O error in the file on the disk).
+ /// \throw std::bad_alloc in case allocation of some internal resources
+ /// or the token fail.
+ const MasterToken& getNextToken(Options options = NONE);
+
+ /// \brief Return the last token back to the lexer.
+ ///
+ /// The method undoes the lasts call to getNextToken(). If you call the
+ /// getNextToken() again with the same options, it'll return the same
+ /// token. If the options are different, it may return a different token,
+ /// but it acts as if the previous getNextToken() was never called.
+ ///
+ /// It is possible to return only one token back in time (you can't call
+ /// ungetToken() twice in a row without calling getNextToken() in between
+ /// successfully).
+ ///
+ /// It does not work after change of source (by pushSource or popSource).
+ ///
+ /// \throw isc::InvalidOperation If called second time in a row or if
+ /// getNextToken() was not called since the last change of the source.
+ void ungetToken();
+
+private:
+ struct MasterLexerImpl;
+ MasterLexerImpl* impl_;
+};
+
+/// \brief Operator to combine \c MasterLexer options
+///
+/// This is a trivial shortcut so that compound options can be specified
+/// in an intuitive way.
+inline MasterLexer::Options
+operator|(MasterLexer::Options o1, MasterLexer::Options o2) {
+ return (static_cast<MasterLexer::Options>(
+ static_cast<unsigned>(o1) | static_cast<unsigned>(o2)));
+}
+
} // namespace dns
} // namespace isc
#endif // MASTER_LEXER_H
/// purposes.
///@{
bool wasLastEOL(const MasterLexer& lexer) const;
- const MasterLexer::Token& getToken(const MasterLexer& lexer) const;
+ const MasterToken& getToken(const MasterLexer& lexer) const;
size_t getParenCount(const MasterLexer& lexer) const;
///@}
using namespace master_lexer_internal;
namespace {
-typedef MasterLexer::Token Token; // shortcut
+typedef MasterToken Token; // shortcut
class MasterLexerStateTest : public ::testing::Test {
protected:
// Commonly used check for string related test cases, checking if the given
// token has expected values.
void
-stringTokenCheck(const std::string& expected, const MasterLexer::Token& token,
+stringTokenCheck(const std::string& expected, const MasterToken& token,
bool quoted = false)
{
EXPECT_EQ(quoted ? Token::QSTRING : Token::STRING, token.getType());
class MasterLexerTokenTest : public ::testing::Test {
protected:
MasterLexerTokenTest() :
- token_eof(MasterLexer::Token::END_OF_FILE),
+ token_eof(MasterToken::END_OF_FILE),
token_str(TEST_STRING, TEST_STRING_LEN),
token_num(42),
- token_err(MasterLexer::Token::UNEXPECTED_END)
+ token_err(MasterToken::UNEXPECTED_END)
{}
- const MasterLexer::Token token_eof; // an example of non-value type token
- const MasterLexer::Token token_str;
- const MasterLexer::Token token_num;
- const MasterLexer::Token token_err;
+ const MasterToken token_eof; // an example of non-value type token
+ const MasterToken token_str;
+ const MasterToken token_num;
+ const MasterToken token_err;
};
TEST_F(MasterLexerTokenTest, strings) {
// basic construction and getter checks
- EXPECT_EQ(MasterLexer::Token::STRING, token_str.getType());
+ EXPECT_EQ(MasterToken::STRING, token_str.getType());
EXPECT_EQ(std::string("string token"), token_str.getString());
std::string strval = "dummy"; // this should be replaced
token_str.getString(strval);
EXPECT_EQ(std::string("string token"), strval);
- const MasterLexer::Token::StringRegion str_region =
+ const MasterToken::StringRegion str_region =
token_str.getStringRegion();
EXPECT_EQ(TEST_STRING, str_region.beg);
EXPECT_EQ(TEST_STRING_LEN, str_region.len);
std::string expected_str("string token");
expected_str.push_back('\0');
EXPECT_EQ(expected_str,
- MasterLexer::Token(TEST_STRING, TEST_STRING_LEN + 1).getString());
- MasterLexer::Token(TEST_STRING, TEST_STRING_LEN + 1).getString(strval);
+ MasterToken(TEST_STRING, TEST_STRING_LEN + 1).getString());
+ MasterToken(TEST_STRING, TEST_STRING_LEN + 1).getString(strval);
EXPECT_EQ(expected_str, strval);
// Construct type of qstring
- EXPECT_EQ(MasterLexer::Token::QSTRING,
- MasterLexer::Token(TEST_STRING, sizeof(TEST_STRING), true).
+ EXPECT_EQ(MasterToken::QSTRING,
+ MasterToken(TEST_STRING, sizeof(TEST_STRING), true).
getType());
// if we explicitly set 'quoted' to false, it should be normal string
- EXPECT_EQ(MasterLexer::Token::STRING,
- MasterLexer::Token(TEST_STRING, sizeof(TEST_STRING), false).
+ EXPECT_EQ(MasterToken::STRING,
+ MasterToken(TEST_STRING, sizeof(TEST_STRING), false).
getType());
// getString/StringRegion() aren't allowed for non string(-variant) types
TEST_F(MasterLexerTokenTest, numbers) {
EXPECT_EQ(42, token_num.getNumber());
- EXPECT_EQ(MasterLexer::Token::NUMBER, token_num.getType());
+ EXPECT_EQ(MasterToken::NUMBER, token_num.getType());
// It's copyable and assignable.
- MasterLexer::Token token(token_num);
+ MasterToken token(token_num);
EXPECT_EQ(42, token.getNumber());
- EXPECT_EQ(MasterLexer::Token::NUMBER, token.getType());
+ EXPECT_EQ(MasterToken::NUMBER, token.getType());
token = token_num;
EXPECT_EQ(42, token.getNumber());
- EXPECT_EQ(MasterLexer::Token::NUMBER, token.getType());
+ EXPECT_EQ(MasterToken::NUMBER, token.getType());
// it's okay to replace it with a different type of token
token = token_eof;
- EXPECT_EQ(MasterLexer::Token::END_OF_FILE, token.getType());
+ EXPECT_EQ(MasterToken::END_OF_FILE, token.getType());
// Possible max value
- token = MasterLexer::Token(0xffffffff);
+ token = MasterToken(0xffffffff);
EXPECT_EQ(4294967295u, token.getNumber());
// getNumber() isn't allowed for non number types
TEST_F(MasterLexerTokenTest, novalues) {
// Just checking we can construct them and getType() returns correct value.
- EXPECT_EQ(MasterLexer::Token::END_OF_FILE, token_eof.getType());
- EXPECT_EQ(MasterLexer::Token::END_OF_LINE,
- MasterLexer::Token(MasterLexer::Token::END_OF_LINE).getType());
- EXPECT_EQ(MasterLexer::Token::INITIAL_WS,
- MasterLexer::Token(MasterLexer::Token::INITIAL_WS).getType());
+ EXPECT_EQ(MasterToken::END_OF_FILE, token_eof.getType());
+ EXPECT_EQ(MasterToken::END_OF_LINE,
+ MasterToken(MasterToken::END_OF_LINE).getType());
+ EXPECT_EQ(MasterToken::INITIAL_WS,
+ MasterToken(MasterToken::INITIAL_WS).getType());
// Special types of tokens cannot have value-based types
- EXPECT_THROW(MasterLexer::Token t(MasterLexer::Token::STRING),
- isc::InvalidParameter);
- EXPECT_THROW(MasterLexer::Token t(MasterLexer::Token::QSTRING),
- isc::InvalidParameter);
- EXPECT_THROW(MasterLexer::Token t(MasterLexer::Token::NUMBER),
- isc::InvalidParameter);
- EXPECT_THROW(MasterLexer::Token t(MasterLexer::Token::ERROR),
- isc::InvalidParameter);
+ EXPECT_THROW(MasterToken t(MasterToken::STRING), isc::InvalidParameter);
+ EXPECT_THROW(MasterToken t(MasterToken::QSTRING), isc::InvalidParameter);
+ EXPECT_THROW(MasterToken t(MasterToken::NUMBER), isc::InvalidParameter);
+ EXPECT_THROW(MasterToken t(MasterToken::ERROR), isc::InvalidParameter);
}
TEST_F(MasterLexerTokenTest, errors) {
- EXPECT_EQ(MasterLexer::Token::ERROR, token_err.getType());
- EXPECT_EQ(MasterLexer::Token::UNEXPECTED_END, token_err.getErrorCode());
+ EXPECT_EQ(MasterToken::ERROR, token_err.getType());
+ EXPECT_EQ(MasterToken::UNEXPECTED_END, token_err.getErrorCode());
EXPECT_EQ("unexpected end of input", token_err.getErrorText());
- EXPECT_EQ("lexer not started",
- MasterLexer::Token(MasterLexer::Token::NOT_STARTED).
+ EXPECT_EQ("lexer not started", MasterToken(MasterToken::NOT_STARTED).
getErrorText());
EXPECT_EQ("unbalanced parentheses",
- MasterLexer::Token(MasterLexer::Token::UNBALANCED_PAREN).
+ MasterToken(MasterToken::UNBALANCED_PAREN).
getErrorText());
- EXPECT_EQ("unbalanced quotes",
- MasterLexer::Token(MasterLexer::Token::UNBALANCED_QUOTES).
+ EXPECT_EQ("unbalanced quotes", MasterToken(MasterToken::UNBALANCED_QUOTES).
getErrorText());
- EXPECT_EQ("no token produced",
- MasterLexer::Token(MasterLexer::Token::NO_TOKEN_PRODUCED).
+ EXPECT_EQ("no token produced", MasterToken(MasterToken::NO_TOKEN_PRODUCED).
getErrorText());
EXPECT_EQ("number out of range",
- MasterLexer::Token(MasterLexer::Token::NUMBER_OUT_OF_RANGE).
+ MasterLexer::Token(MasterToken::NUMBER_OUT_OF_RANGE).
getErrorText());
// getErrorCode/Text() isn't allowed for non number types
// Only the pre-defined error code is accepted. Hardcoding '6' (max code
// + 1) is intentional; it'd be actually better if we notice it when we
// update the enum list (which shouldn't happen too often).
- EXPECT_THROW(MasterLexer::Token(MasterLexer::Token::ErrorCode(6)),
+ EXPECT_THROW(MasterToken(MasterToken::ErrorCode(6)),
isc::InvalidParameter);
// Check the coexistence of "from number" and "from error-code"
// constructors won't cause confusion.
- EXPECT_EQ(MasterLexer::Token::NUMBER,
- MasterLexer::Token(static_cast<uint32_t>(
- MasterLexer::Token::NOT_STARTED)).
+ EXPECT_EQ(MasterToken::NUMBER,
+ MasterToken(static_cast<uint32_t>(MasterToken::NOT_STARTED)).
getType());
}
}
lexer.pushSource(ss);
// First, the newline should get out.
- EXPECT_EQ(MasterLexer::Token::END_OF_LINE, lexer.getNextToken().getType());
+ EXPECT_EQ(MasterToken::END_OF_LINE, lexer.getNextToken().getType());
// Then the whitespace, if we specify the option.
- EXPECT_EQ(MasterLexer::Token::INITIAL_WS,
+ EXPECT_EQ(MasterToken::INITIAL_WS,
lexer.getNextToken(MasterLexer::INITIAL_WS).getType());
// The newline
- EXPECT_EQ(MasterLexer::Token::END_OF_LINE, lexer.getNextToken().getType());
+ EXPECT_EQ(MasterToken::END_OF_LINE, lexer.getNextToken().getType());
// The (quoted) string
- EXPECT_EQ(MasterLexer::Token::QSTRING,
+ EXPECT_EQ(MasterToken::QSTRING,
lexer.getNextToken(MasterLexer::QSTRING).getType());
// And the end of line and file
- EXPECT_EQ(MasterLexer::Token::END_OF_LINE, lexer.getNextToken().getType());
- EXPECT_EQ(MasterLexer::Token::END_OF_FILE, lexer.getNextToken().getType());
+ EXPECT_EQ(MasterToken::END_OF_LINE, lexer.getNextToken().getType());
+ EXPECT_EQ(MasterToken::END_OF_FILE, lexer.getNextToken().getType());
}
// Test we correctly find end of file.
lexer.pushSource(ss);
// The first one is found to be EOF
- EXPECT_EQ(MasterLexer::Token::END_OF_FILE, lexer.getNextToken().getType());
+ EXPECT_EQ(MasterToken::END_OF_FILE, lexer.getNextToken().getType());
// And it stays on EOF for any following attempts
- EXPECT_EQ(MasterLexer::Token::END_OF_FILE, lexer.getNextToken().getType());
+ EXPECT_EQ(MasterToken::END_OF_FILE, lexer.getNextToken().getType());
// And we can step back one token, but that is the EOF too.
lexer.ungetToken();
- EXPECT_EQ(MasterLexer::Token::END_OF_FILE, lexer.getNextToken().getType());
+ EXPECT_EQ(MasterToken::END_OF_FILE, lexer.getNextToken().getType());
}
// Check we properly return error when there's an opened parentheses and no
lexer.pushSource(ss);
// The string gets out first
- EXPECT_EQ(MasterLexer::Token::STRING, lexer.getNextToken().getType());
+ EXPECT_EQ(MasterToken::STRING, lexer.getNextToken().getType());
// Then an unbalanced parenthesis
- EXPECT_EQ(MasterLexer::Token::UNBALANCED_PAREN,
+ EXPECT_EQ(MasterToken::UNBALANCED_PAREN,
lexer.getNextToken().getErrorCode());
// And then EOF
- EXPECT_EQ(MasterLexer::Token::END_OF_FILE, lexer.getNextToken().getType());
+ EXPECT_EQ(MasterToken::END_OF_FILE, lexer.getNextToken().getType());
}
// Check we properly return error when there's an opened quoted string and no
lexer.pushSource(ss);
// Then an unbalanced qstring (reported as an unexpected end)
- EXPECT_EQ(MasterLexer::Token::UNEXPECTED_END,
+ EXPECT_EQ(MasterToken::UNEXPECTED_END,
lexer.getNextToken(MasterLexer::QSTRING).getErrorCode());
// And then EOF
- EXPECT_EQ(MasterLexer::Token::END_OF_FILE, lexer.getNextToken().getType());
+ EXPECT_EQ(MasterToken::END_OF_FILE, lexer.getNextToken().getType());
}
// Test ungetting tokens works
lexer.pushSource(ss);
// Try getting the newline
- EXPECT_EQ(MasterLexer::Token::END_OF_LINE, lexer.getNextToken().getType());
+ EXPECT_EQ(MasterToken::END_OF_LINE, lexer.getNextToken().getType());
// Return it and get again
lexer.ungetToken();
- EXPECT_EQ(MasterLexer::Token::END_OF_LINE, lexer.getNextToken().getType());
+ EXPECT_EQ(MasterToken::END_OF_LINE, lexer.getNextToken().getType());
// Get the string and return it back
- EXPECT_EQ(MasterLexer::Token::QSTRING,
+ EXPECT_EQ(MasterToken::QSTRING,
lexer.getNextToken(MasterLexer::QSTRING).getType());
lexer.ungetToken();
// But if we change the options, it honors them
- EXPECT_EQ(MasterLexer::Token::INITIAL_WS,
+ EXPECT_EQ(MasterToken::INITIAL_WS,
lexer.getNextToken(MasterLexer::QSTRING |
MasterLexer::INITIAL_WS).getType());
// Get to the "more" string
- EXPECT_EQ(MasterLexer::Token::QSTRING,
+ EXPECT_EQ(MasterToken::QSTRING,
lexer.getNextToken(MasterLexer::QSTRING).getType());
- EXPECT_EQ(MasterLexer::Token::STRING,
+ EXPECT_EQ(MasterToken::STRING,
lexer.getNextToken(MasterLexer::QSTRING).getType());
// Return it back. It should get inside the parentheses.
// Upon next attempt to get it again, the newline inside the parentheses
// should be still ignored.
lexer.ungetToken();
- EXPECT_EQ(MasterLexer::Token::STRING,
+ EXPECT_EQ(MasterToken::STRING,
lexer.getNextToken(MasterLexer::QSTRING).getType());
}
ss << "\n \n";
lexer.pushSource(ss);
// Skip the first newline
- EXPECT_EQ(MasterLexer::Token::END_OF_LINE, lexer.getNextToken().getType());
+ EXPECT_EQ(MasterToken::END_OF_LINE, lexer.getNextToken().getType());
// If we call it the usual way, it skips up to the newline and returns
// it
- EXPECT_EQ(MasterLexer::Token::END_OF_LINE, lexer.getNextToken().getType());
+ EXPECT_EQ(MasterToken::END_OF_LINE, lexer.getNextToken().getType());
// Now we return it. If we call it again, but with different options,
// we get the initial whitespace.
lexer.ungetToken();
- EXPECT_EQ(MasterLexer::Token::INITIAL_WS,
+ EXPECT_EQ(MasterToken::INITIAL_WS,
lexer.getNextToken(MasterLexer::INITIAL_WS).getType());
}
ss << "\n";
lexer.pushSource(ss);
- EXPECT_EQ(MasterLexer::Token::END_OF_LINE, lexer.getNextToken().getType());
+ EXPECT_EQ(MasterToken::END_OF_LINE, lexer.getNextToken().getType());
// Unget the token. It can be done once
lexer.ungetToken();
// But not twice
TEST_F(MasterLexerTest, ungetAfterSwitch) {
ss << "\n\n";
lexer.pushSource(ss);
- EXPECT_EQ(MasterLexer::Token::END_OF_LINE, lexer.getNextToken().getType());
+ EXPECT_EQ(MasterToken::END_OF_LINE, lexer.getNextToken().getType());
// Switch the source
std::stringstream ss2;
ss2 << "\n\n";
lexer.pushSource(ss2);
EXPECT_THROW(lexer.ungetToken(), isc::InvalidOperation);
// We can get from the new source
- EXPECT_EQ(MasterLexer::Token::END_OF_LINE, lexer.getNextToken().getType());
+ EXPECT_EQ(MasterToken::END_OF_LINE, lexer.getNextToken().getType());
// And when we drop the current source, we can't unget again
lexer.popSource();
EXPECT_THROW(lexer.ungetToken(), isc::InvalidOperation);
projects / thirdparty / kea.git / commitdiff
