Skip to content

GitLab

Commit 425e18f7 authored by Michal 'vorner' Vaner's avatar Michal 'vorner' Vaner
Browse files

Merge remote-tracking branch 'origin/trac2506' into base/loader

parents 5bb4db41 d4b9c332
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 606 additions and 335 deletions
src/lib/dns/master_lexer.cc
View file @ 425e18f7
......@@ -36,7 +36,7 @@ using namespace master_lexer_internal;
struct MasterLexer::MasterLexerImpl {
MasterLexerImpl() : source_(NULL), token_(Token::NOT_STARTED),
MasterLexerImpl() : source_(NULL), token_(MasterToken::NOT_STARTED),
paren_count_(0), last_was_eol_(false),
has_previous_(false),
previous_paren_count_(0),
......@@ -82,7 +82,7 @@ struct MasterLexer::MasterLexerImpl {
std::vector<InputSourcePtr> sources_;
InputSource* source_; // current source (NULL if sources_ is empty)
Token token_; // currently recognized token (set by a state)
MasterToken token_; // currently recognized token (set by a state)
std::vector<char> data_; // placeholder for string data
// These are used in states, and defined here only as a placeholder.
......@@ -165,9 +165,8 @@ MasterLexer::getSourceLine() const {
return (impl_->sources_.back()->getCurrentLine());
}
const MasterLexer::Token&
const MasterToken&
MasterLexer::getNextToken(Options options) {
// If the source is not available
if (impl_->source_ == NULL) {
isc_throw(isc::InvalidOperation, "No source to read tokens from");
}
......@@ -178,7 +177,7 @@ MasterLexer::getNextToken(Options options) {
impl_->has_previous_ = true;
// Reset the token now. This is to check a token was actually produced.
// This is debugging aid.
impl_->token_ = Token(Token::NO_TOKEN_PRODUCED);
impl_->token_ = MasterToken(MasterToken::NO_TOKEN_PRODUCED);
// And get the token
// This actually handles EOF internally too.
......@@ -188,8 +187,62 @@ MasterLexer::getNextToken(Options options) {
}
// Make sure a token was produced. Since this Can Not Happen, we assert
// here instead of throwing.
assert(impl_->token_.getType() != Token::ERROR ||
impl_->token_.getErrorCode() != Token::NO_TOKEN_PRODUCED);
assert(impl_->token_.getType() != MasterToken::ERROR ||
impl_->token_.getErrorCode() != MasterToken::NO_TOKEN_PRODUCED);
return (impl_->token_);
}
namespace {
inline MasterLexer::Options
optionsForTokenType(MasterToken::Type expect) {
switch (expect) {
case MasterToken::STRING:
return (MasterLexer::NONE);
case MasterToken::QSTRING:
return (MasterLexer::QSTRING);
case MasterToken::NUMBER:
return (MasterLexer::NUMBER);
default:
isc_throw(InvalidParameter,
"expected type for getNextToken not supported: " << expect);
}
}
}
const MasterToken&
MasterLexer::getNextToken(MasterToken::Type expect, bool eol_ok) {
// Get the next token, specifying an appropriate option corresponding to
// the expected type. The result should be set in impl_->token_.
getNextToken(optionsForTokenType(expect));
if (impl_->token_.getType() == MasterToken::ERROR) {
if (impl_->token_.getErrorCode() == MasterToken::NUMBER_OUT_OF_RANGE) {
ungetToken();
}
throw LexerError(__FILE__, __LINE__, impl_->token_);
}
const bool is_eol_like =
(impl_->token_.getType() == MasterToken::END_OF_LINE ||
impl_->token_.getType() == MasterToken::END_OF_FILE);
if (eol_ok && is_eol_like) {
return (impl_->token_);
}
if (impl_->token_.getType() == MasterToken::STRING &&
expect == MasterToken::QSTRING) {
return (impl_->token_);
}
if (impl_->token_.getType() != expect) {
ungetToken();
if (is_eol_like) {
throw LexerError(__FILE__, __LINE__,
MasterToken(MasterToken::UNEXPECTED_END));
}
assert(expect == MasterToken::NUMBER);
throw LexerError(__FILE__, __LINE__,
MasterToken(MasterToken::BAD_NUMBER));
}
return (impl_->token_);
}
......@@ -212,16 +265,17 @@ const char* const error_text[] = {
"unexpected end of input", // UNEXPECTED_END
"unbalanced quotes", // UNBALANCED_QUOTES
"no token produced", // NO_TOKEN_PRODUCED
"number out of range" // NUMBER_OUT_OF_RANGE
"number out of range", // NUMBER_OUT_OF_RANGE
"not a valid number" // BAD_NUMBER
};
const size_t error_text_max_count = sizeof(error_text) / sizeof(error_text[0]);
} // end unnamed namespace
std::string
MasterLexer::Token::getErrorText() const {
MasterToken::getErrorText() const {
if (type_ != ERROR) {
isc_throw(InvalidOperation,
"Token::getErrorText() for non error type");
"MasterToken::getErrorText() for non error type");
}
// The class integrity ensures the following:
......@@ -234,14 +288,12 @@ namespace master_lexer_internal {
// Note that these need to be defined here so that they can refer to
// the details of MasterLexerImpl.
typedef MasterLexer::Token Token; // convenience shortcut
bool
State::wasLastEOL(const MasterLexer& lexer) const {
return (lexer.impl_->last_was_eol_);
}
const MasterLexer::Token&
const MasterToken&
State::getToken(const MasterLexer& lexer) const {
return (lexer.impl_->token_);
}
......@@ -271,7 +323,7 @@ public:
if (c != '\n') {
getLexerImpl(lexer)->source_->ungetChar();
}
getLexerImpl(lexer)->token_ = Token(Token::END_OF_LINE);
getLexerImpl(lexer)->token_ = MasterToken(MasterToken::END_OF_LINE);
getLexerImpl(lexer)->last_was_eol_ = true;
}
};
......@@ -342,24 +394,24 @@ State::start(MasterLexer& lexer, MasterLexer::Options options) {
if (c == InputSource::END_OF_STREAM) {
lexerimpl.last_was_eol_ = false;
if (paren_count != 0) {
lexerimpl.token_ = Token(Token::UNBALANCED_PAREN);
lexerimpl.token_ = MasterToken(MasterToken::UNBALANCED_PAREN);
paren_count = 0; // reset to 0; this helps in lenient mode.
return (NULL);
}
lexerimpl.token_ = Token(Token::END_OF_FILE);
lexerimpl.token_ = MasterToken(MasterToken::END_OF_FILE);
return (NULL);
} else if (c == ' ' || c == '\t') {
// If requested and we are not in (), recognize the initial space.
if (lexerimpl.last_was_eol_ && paren_count == 0 &&
(options & MasterLexer::INITIAL_WS) != 0) {
lexerimpl.last_was_eol_ = false;
lexerimpl.token_ = Token(Token::INITIAL_WS);
lexerimpl.token_ = MasterToken(MasterToken::INITIAL_WS);
return (NULL);
}
} else if (c == '\n') {
lexerimpl.last_was_eol_ = true;
if (paren_count == 0) { // we don't recognize EOL if we are in ()
lexerimpl.token_ = Token(Token::END_OF_LINE);
lexerimpl.token_ = MasterToken(MasterToken::END_OF_LINE);
return (NULL);
}
} else if (c == '\r') {
......@@ -375,7 +427,7 @@ State::start(MasterLexer& lexer, MasterLexer::Options options) {
} else if (c == ')') {
lexerimpl.last_was_eol_ = false;
if (paren_count == 0) {
lexerimpl.token_ = Token(Token::UNBALANCED_PAREN);
lexerimpl.token_ = MasterToken(MasterToken::UNBALANCED_PAREN);
return (NULL);
}
--paren_count;
......@@ -407,7 +459,7 @@ String::handle(MasterLexer& lexer) const {
if (getLexerImpl(lexer)->isTokenEnd(c, escaped)) {
getLexerImpl(lexer)->source_->ungetChar();
getLexerImpl(lexer)->token_ =
MasterLexer::Token(&data.at(0), data.size());
MasterToken(&data.at(0), data.size());
return;
}
escaped = (c == '\\' && !escaped);
......@@ -417,7 +469,7 @@ String::handle(MasterLexer& lexer) const {
void
QString::handle(MasterLexer& lexer) const {
MasterLexer::Token& token = getLexerImpl(lexer)->token_;
MasterToken& token = getLexerImpl(lexer)->token_;
std::vector<char>& data = getLexerImpl(lexer)->data_;
data.clear();
......@@ -425,7 +477,7 @@ QString::handle(MasterLexer& lexer) const {
while (true) {
const int c = getLexerImpl(lexer)->source_->getChar();
if (c == InputSource::END_OF_STREAM) {
token = Token(Token::UNEXPECTED_END);
token = MasterToken(MasterToken::UNEXPECTED_END);
return;
} else if (c == '"') {
if (escaped) {
......@@ -434,12 +486,12 @@ QString::handle(MasterLexer& lexer) const {
escaped = false;
data.back() = '"';
} else {
token = MasterLexer::Token(&data.at(0), data.size(), true);
token = MasterToken(&data.at(0), data.size(), true);
return;
}
} else if (c == '\n' && !escaped) {
getLexerImpl(lexer)->source_->ungetChar();
token = Token(Token::UNBALANCED_QUOTES);
token = MasterToken(MasterToken::UNBALANCED_QUOTES);
return;
} else {
escaped = (c == '\\' && !escaped);
......@@ -450,7 +502,7 @@ QString::handle(MasterLexer& lexer) const {
void
Number::handle(MasterLexer& lexer) const {
MasterLexer::Token& token = getLexerImpl(lexer)->token_;
MasterToken& token = getLexerImpl(lexer)->token_;
// It may yet turn out to be a string, so we first
// collect all the data
......@@ -470,15 +522,14 @@ Number::handle(MasterLexer& lexer) const {
try {
const uint32_t number32 =
boost::lexical_cast<uint32_t, const char*>(&data[0]);
token = MasterLexer::Token(number32);
token = MasterToken(number32);
} catch (const boost::bad_lexical_cast&) {
// Since we already know we have only digits,
// range should be the only possible problem.
token = Token(Token::NUMBER_OUT_OF_RANGE);
token = MasterToken(MasterToken::NUMBER_OUT_OF_RANGE);
}
} else {
token = MasterLexer::Token(&data.at(0),
data.size());
token = MasterToken(&data.at(0), data.size());
}
return;
}
......
src/lib/dns/master_lexer.h
View file @ 425e18f7
This diff is collapsed.
src/lib/dns/master_lexer_state.h
View file @ 425e18f7
......@@ -43,10 +43,10 @@ namespace master_lexer_internal {
/// state, so it makes more sense to separate the interface for the transition
/// from the initial state.
///
/// When an object of a specific state class completes the session, it
/// normally sets the identified token in the lexer, and returns NULL;
/// if more transition is necessary, it returns a pointer to the next state
/// object.
/// If the whole lexer transition is completed within start(), it sets the
/// identified token and returns NULL; otherwise it returns a pointer to
/// an object of a specific state class that completes the session
/// on the call of handle().
///
/// As is usual in the state design pattern, the \c State class is made
/// a friend class of \c MasterLexer and can refer to its internal details.
......@@ -119,7 +119,7 @@ public:
/// purposes.
///@{
bool wasLastEOL(const MasterLexer& lexer) const;
const MasterLexer::Token& getToken(const MasterLexer& lexer) const;
const MasterToken& getToken(const MasterLexer& lexer) const;
size_t getParenCount(const MasterLexer& lexer) const;
///@}
......
src/lib/dns/tests/master_lexer_state_unittest.cc
View file @ 425e18f7
......@@ -24,7 +24,7 @@ using namespace isc::dns;
using namespace master_lexer_internal;
namespace {
typedef MasterLexer::Token Token; // shortcut
typedef MasterToken Token; // shortcut
class MasterLexerStateTest : public ::testing::Test {
protected:
......@@ -260,7 +260,7 @@ TEST_F(MasterLexerStateTest, crlf) {
// Commonly used check for string related test cases, checking if the given
// token has expected values.
void
stringTokenCheck(const std::string& expected, const MasterLexer::Token& token,
stringTokenCheck(const std::string& expected, const MasterToken& token,
bool quoted = false)
{
EXPECT_EQ(quoted ? Token::QSTRING : Token::STRING, token.getType());
......
src/lib/dns/tests/master_lexer_token_unittest.cc
View file @ 425e18f7
......@@ -31,27 +31,27 @@ const size_t TEST_STRING_LEN = sizeof(TEST_STRING) - 1;
class MasterLexerTokenTest : public ::testing::Test {
protected:
MasterLexerTokenTest() :
token_eof(MasterLexer::Token::END_OF_FILE),
token_eof(MasterToken::END_OF_FILE),
token_str(TEST_STRING, TEST_STRING_LEN),
token_num(42),
token_err(MasterLexer::Token::UNEXPECTED_END)
token_err(MasterToken::UNEXPECTED_END)
{}
const MasterLexer::Token token_eof; // an example of non-value type token
const MasterLexer::Token token_str;
const MasterLexer::Token token_num;
const MasterLexer::Token token_err;
const MasterToken token_eof; // an example of non-value type token
const MasterToken token_str;
const MasterToken token_num;
const MasterToken token_err;
};
TEST_F(MasterLexerTokenTest, strings) {
// basic construction and getter checks
EXPECT_EQ(MasterLexer::Token::STRING, token_str.getType());
EXPECT_EQ(MasterToken::STRING, token_str.getType());
EXPECT_EQ(std::string("string token"), token_str.getString());
std::string strval = "dummy"; // this should be replaced
token_str.getString(strval);
EXPECT_EQ(std::string("string token"), strval);
const MasterLexer::Token::StringRegion str_region =
const MasterToken::StringRegion str_region =
token_str.getStringRegion();
EXPECT_EQ(TEST_STRING, str_region.beg);
EXPECT_EQ(TEST_STRING_LEN, str_region.len);
......@@ -62,17 +62,17 @@ TEST_F(MasterLexerTokenTest, strings) {
std::string expected_str("string token");
expected_str.push_back('\0');
EXPECT_EQ(expected_str,
MasterLexer::Token(TEST_STRING, TEST_STRING_LEN + 1).getString());
MasterLexer::Token(TEST_STRING, TEST_STRING_LEN + 1).getString(strval);
MasterToken(TEST_STRING, TEST_STRING_LEN + 1).getString());
MasterToken(TEST_STRING, TEST_STRING_LEN + 1).getString(strval);
EXPECT_EQ(expected_str, strval);
// Construct type of qstring
EXPECT_EQ(MasterLexer::Token::QSTRING,
MasterLexer::Token(TEST_STRING, sizeof(TEST_STRING), true).
EXPECT_EQ(MasterToken::QSTRING,
MasterToken(TEST_STRING, sizeof(TEST_STRING), true).
getType());
// if we explicitly set 'quoted' to false, it should be normal string
EXPECT_EQ(MasterLexer::Token::STRING,
MasterLexer::Token(TEST_STRING, sizeof(TEST_STRING), false).
EXPECT_EQ(MasterToken::STRING,
MasterToken(TEST_STRING, sizeof(TEST_STRING), false).
getType());
// getString/StringRegion() aren't allowed for non string(-variant) types
......@@ -86,23 +86,23 @@ TEST_F(MasterLexerTokenTest, strings) {
TEST_F(MasterLexerTokenTest, numbers) {
EXPECT_EQ(42, token_num.getNumber());
EXPECT_EQ(MasterLexer::Token::NUMBER, token_num.getType());
EXPECT_EQ(MasterToken::NUMBER, token_num.getType());
// It's copyable and assignable.
MasterLexer::Token token(token_num);
MasterToken token(token_num);
EXPECT_EQ(42, token.getNumber());
EXPECT_EQ(MasterLexer::Token::NUMBER, token.getType());
EXPECT_EQ(MasterToken::NUMBER, token.getType());
token = token_num;
EXPECT_EQ(42, token.getNumber());
EXPECT_EQ(MasterLexer::Token::NUMBER, token.getType());
EXPECT_EQ(MasterToken::NUMBER, token.getType());
// it's okay to replace it with a different type of token
token = token_eof;
EXPECT_EQ(MasterLexer::Token::END_OF_FILE, token.getType());
EXPECT_EQ(MasterToken::END_OF_FILE, token.getType());
// Possible max value
token = MasterLexer::Token(0xffffffff);
token = MasterToken(0xffffffff);
EXPECT_EQ(4294967295u, token.getNumber());
// getNumber() isn't allowed for non number types
......@@ -112,58 +112,52 @@ TEST_F(MasterLexerTokenTest, numbers) {
TEST_F(MasterLexerTokenTest, novalues) {
// Just checking we can construct them and getType() returns correct value.
EXPECT_EQ(MasterLexer::Token::END_OF_FILE, token_eof.getType());
EXPECT_EQ(MasterLexer::Token::END_OF_LINE,
MasterLexer::Token(MasterLexer::Token::END_OF_LINE).getType());
EXPECT_EQ(MasterLexer::Token::INITIAL_WS,
MasterLexer::Token(MasterLexer::Token::INITIAL_WS).getType());
EXPECT_EQ(MasterToken::END_OF_FILE, token_eof.getType());
EXPECT_EQ(MasterToken::END_OF_LINE,
MasterToken(MasterToken::END_OF_LINE).getType());
EXPECT_EQ(MasterToken::INITIAL_WS,
MasterToken(MasterToken::INITIAL_WS).getType());
// Special types of tokens cannot have value-based types
EXPECT_THROW(MasterLexer::Token t(MasterLexer::Token::STRING),
isc::InvalidParameter);
EXPECT_THROW(MasterLexer::Token t(MasterLexer::Token::QSTRING),
isc::InvalidParameter);
EXPECT_THROW(MasterLexer::Token t(MasterLexer::Token::NUMBER),
isc::InvalidParameter);
EXPECT_THROW(MasterLexer::Token t(MasterLexer::Token::ERROR),
isc::InvalidParameter);
EXPECT_THROW(MasterToken t(MasterToken::STRING), isc::InvalidParameter);
EXPECT_THROW(MasterToken t(MasterToken::QSTRING), isc::InvalidParameter);
EXPECT_THROW(MasterToken t(MasterToken::NUMBER), isc::InvalidParameter);
EXPECT_THROW(MasterToken t(MasterToken::ERROR), isc::InvalidParameter);
}
TEST_F(MasterLexerTokenTest, errors) {
EXPECT_EQ(MasterLexer::Token::ERROR, token_err.getType());
EXPECT_EQ(MasterLexer::Token::UNEXPECTED_END, token_err.getErrorCode());
EXPECT_EQ(MasterToken::ERROR, token_err.getType());
EXPECT_EQ(MasterToken::UNEXPECTED_END, token_err.getErrorCode());
EXPECT_EQ("unexpected end of input", token_err.getErrorText());
EXPECT_EQ("lexer not started",
MasterLexer::Token(MasterLexer::Token::NOT_STARTED).
EXPECT_EQ("lexer not started", MasterToken(MasterToken::NOT_STARTED).
getErrorText());
EXPECT_EQ("unbalanced parentheses",
MasterLexer::Token(MasterLexer::Token::UNBALANCED_PAREN).
MasterToken(MasterToken::UNBALANCED_PAREN).
getErrorText());
EXPECT_EQ("unbalanced quotes",
MasterLexer::Token(MasterLexer::Token::UNBALANCED_QUOTES).
EXPECT_EQ("unbalanced quotes", MasterToken(MasterToken::UNBALANCED_QUOTES).
getErrorText());
EXPECT_EQ("no token produced",
MasterLexer::Token(MasterLexer::Token::NO_TOKEN_PRODUCED).
EXPECT_EQ("no token produced", MasterToken(MasterToken::NO_TOKEN_PRODUCED).
getErrorText());
EXPECT_EQ("number out of range",
MasterLexer::Token(MasterLexer::Token::NUMBER_OUT_OF_RANGE).
MasterToken(MasterToken::NUMBER_OUT_OF_RANGE).
getErrorText());
EXPECT_EQ("not a valid number",
MasterToken(MasterToken::BAD_NUMBER).getErrorText());
// getErrorCode/Text() isn't allowed for non number types
EXPECT_THROW(token_num.getErrorCode(), isc::InvalidOperation);
EXPECT_THROW(token_num.getErrorText(), isc::InvalidOperation);
// Only the pre-defined error code is accepted. Hardcoding '6' (max code
// Only the pre-defined error code is accepted. Hardcoding '7' (max code
// + 1) is intentional; it'd be actually better if we notice it when we
// update the enum list (which shouldn't happen too often).
EXPECT_THROW(MasterLexer::Token(MasterLexer::Token::ErrorCode(6)),
EXPECT_THROW(MasterToken(MasterToken::ErrorCode(7)),
isc::InvalidParameter);
// Check the coexistence of "from number" and "from error-code"
// constructors won't cause confusion.
EXPECT_EQ(MasterLexer::Token::NUMBER,
MasterLexer::Token(static_cast<uint32_t>(
MasterLexer::Token::NOT_STARTED)).
EXPECT_EQ(MasterToken::NUMBER,
MasterToken(static_cast<uint32_t>(MasterToken::NOT_STARTED)).
getType());
}
}
src/lib/dns/tests/master_lexer_unittest.cc
View file @ 425e18f7
......@@ -141,19 +141,19 @@ TEST_F(MasterLexerTest, getNextToken) {
lexer.pushSource(ss);
// First, the newline should get out.
EXPECT_EQ(MasterLexer::Token::END_OF_LINE, lexer.getNextToken().getType());
EXPECT_EQ(MasterToken::END_OF_LINE, lexer.getNextToken().getType());
// Then the whitespace, if we specify the option.
EXPECT_EQ(MasterLexer::Token::INITIAL_WS,
EXPECT_EQ(MasterToken::INITIAL_WS,
lexer.getNextToken(MasterLexer::INITIAL_WS).getType());
// The newline
EXPECT_EQ(MasterLexer::Token::END_OF_LINE, lexer.getNextToken().getType());
EXPECT_EQ(MasterToken::END_OF_LINE, lexer.getNextToken().getType());
// The (quoted) string
EXPECT_EQ(MasterLexer::Token::QSTRING,
EXPECT_EQ(MasterToken::QSTRING,
lexer.getNextToken(MasterLexer::QSTRING).getType());
// And the end of line and file
EXPECT_EQ(MasterLexer::Token::END_OF_LINE, lexer.getNextToken().getType());
EXPECT_EQ(MasterLexer::Token::END_OF_FILE, lexer.getNextToken().getType());
EXPECT_EQ(MasterToken::END_OF_LINE, lexer.getNextToken().getType());
EXPECT_EQ(MasterToken::END_OF_FILE, lexer.getNextToken().getType());
}
// Test we correctly find end of file.
......@@ -162,12 +162,12 @@ TEST_F(MasterLexerTest, eof) {
lexer.pushSource(ss);
// The first one is found to be EOF
EXPECT_EQ(MasterLexer::Token::END_OF_FILE, lexer.getNextToken().getType());
EXPECT_EQ(MasterToken::END_OF_FILE, lexer.getNextToken().getType());
// And it stays on EOF for any following attempts
EXPECT_EQ(MasterLexer::Token::END_OF_FILE, lexer.getNextToken().getType());
EXPECT_EQ(MasterToken::END_OF_FILE, lexer.getNextToken().getType());
// And we can step back one token, but that is the EOF too.
lexer.ungetToken();
EXPECT_EQ(MasterLexer::Token::END_OF_FILE, lexer.getNextToken().getType());
EXPECT_EQ(MasterToken::END_OF_FILE, lexer.getNextToken().getType());
}
// Check we properly return error when there's an opened parentheses and no
......@@ -177,12 +177,12 @@ TEST_F(MasterLexerTest, getUnbalancedParen) {
lexer.pushSource(ss);
// The string gets out first
EXPECT_EQ(MasterLexer::Token::STRING, lexer.getNextToken().getType());
EXPECT_EQ(MasterToken::STRING, lexer.getNextToken().getType());
// Then an unbalanced parenthesis
EXPECT_EQ(MasterLexer::Token::UNBALANCED_PAREN,
EXPECT_EQ(MasterToken::UNBALANCED_PAREN,
lexer.getNextToken().getErrorCode());
// And then EOF
EXPECT_EQ(MasterLexer::Token::END_OF_FILE, lexer.getNextToken().getType());
EXPECT_EQ(MasterToken::END_OF_FILE, lexer.getNextToken().getType());
}
// Check we properly return error when there's an opened quoted string and no
......@@ -192,10 +192,10 @@ TEST_F(MasterLexerTest, getUnbalancedString) {
lexer.pushSource(ss);
// Then an unbalanced qstring (reported as an unexpected end)
EXPECT_EQ(MasterLexer::Token::UNEXPECTED_END,
EXPECT_EQ(MasterToken::UNEXPECTED_END,
lexer.getNextToken(MasterLexer::QSTRING).getErrorCode());
// And then EOF
EXPECT_EQ(MasterLexer::Token::END_OF_FILE, lexer.getNextToken().getType());
EXPECT_EQ(MasterToken::END_OF_FILE, lexer.getNextToken().getType());
}
// Test ungetting tokens works
......@@ -204,28 +204,28 @@ TEST_F(MasterLexerTest, ungetToken) {
lexer.pushSource(ss);
// Try getting the newline
EXPECT_EQ(MasterLexer::Token::END_OF_LINE, lexer.getNextToken().getType());
EXPECT_EQ(MasterToken::END_OF_LINE, lexer.getNextToken().getType());
// Return it and get again
lexer.ungetToken();
EXPECT_EQ(MasterLexer::Token::END_OF_LINE, lexer.getNextToken().getType());
EXPECT_EQ(MasterToken::END_OF_LINE, lexer.getNextToken().getType());
// Get the string and return it back
EXPECT_EQ(MasterLexer::Token::QSTRING,
EXPECT_EQ(MasterToken::QSTRING,
lexer.getNextToken(MasterLexer::QSTRING).getType());
lexer.ungetToken();
// But if we change the options, it honors them
EXPECT_EQ(MasterLexer::Token::INITIAL_WS,
EXPECT_EQ(MasterToken::INITIAL_WS,
lexer.getNextToken(MasterLexer::QSTRING |
MasterLexer::INITIAL_WS).getType());
// Get to the "more" string
EXPECT_EQ(MasterLexer::Token::QSTRING,
EXPECT_EQ(MasterToken::QSTRING,
lexer.getNextToken(MasterLexer::QSTRING).getType());
EXPECT_EQ(MasterLexer::Token::STRING,
EXPECT_EQ(MasterToken::STRING,
lexer.getNextToken(MasterLexer::QSTRING).getType());
// Return it back. It should get inside the parentheses.
// Upon next attempt to get it again, the newline inside the parentheses
// should be still ignored.
lexer.ungetToken();
EXPECT_EQ(MasterLexer::Token::STRING,
EXPECT_EQ(MasterToken::STRING,
lexer.getNextToken(MasterLexer::QSTRING).getType());
}
......@@ -235,16 +235,16 @@ TEST_F(MasterLexerTest, ungetRealOptions) {
ss << "\n \n";
lexer.pushSource(ss);
// Skip the first newline
EXPECT_EQ(MasterLexer::Token::END_OF_LINE, lexer.getNextToken().getType());
EXPECT_EQ(MasterToken::END_OF_LINE, lexer.getNextToken().getType());
// If we call it the usual way, it skips up to the newline and returns
// it
EXPECT_EQ(MasterLexer::Token::END_OF_LINE, lexer.getNextToken().getType());
EXPECT_EQ(MasterToken::END_OF_LINE, lexer.getNextToken().getType());
// Now we return it. If we call it again, but with different options,
// we get the initial whitespace.
lexer.ungetToken();
EXPECT_EQ(MasterLexer::Token::INITIAL_WS,
EXPECT_EQ(MasterToken::INITIAL_WS,
lexer.getNextToken(MasterLexer::INITIAL_WS).getType());
}
......@@ -253,7 +253,7 @@ TEST_F(MasterLexerTest, ungetTwice) {
ss << "\n";
lexer.pushSource(ss);
EXPECT_EQ(MasterLexer::Token::END_OF_LINE, lexer.getNextToken().getType());
EXPECT_EQ(MasterToken::END_OF_LINE, lexer.getNextToken().getType());
// Unget the token. It can be done once
lexer.ungetToken();
// But not twice
......@@ -271,17 +271,157 @@ TEST_F(MasterLexerTest, ungetBeforeGet) {
TEST_F(MasterLexerTest, ungetAfterSwitch) {
ss <<