Merge remote-tracking branch 'origin/trac2506' into base/loader

src/lib/dns/master_lexer.cc

@@ -36,7 +36,7 @@ using namespace master_lexer_internal;
 
 
 struct MasterLexer::MasterLexerImpl {
-    MasterLexerImpl() : source_(NULL), token_(Token::NOT_STARTED),
+    MasterLexerImpl() : source_(NULL), token_(MasterToken::NOT_STARTED),
                         paren_count_(0), last_was_eol_(false),
                         has_previous_(false),
                         previous_paren_count_(0),
@@ -82,7 +82,7 @@ struct MasterLexer::MasterLexerImpl {
 
     std::vector<InputSourcePtr> sources_;
     InputSource* source_;       // current source (NULL if sources_ is empty)
-    Token token_;               // currently recognized token (set by a state)
+    MasterToken token_;         // currently recognized token (set by a state)
     std::vector<char> data_;    // placeholder for string data
 
     // These are used in states, and defined here only as a placeholder.
@@ -165,9 +165,8 @@ MasterLexer::getSourceLine() const {
     return (impl_->sources_.back()->getCurrentLine());
 }
 
-const MasterLexer::Token&
+const MasterToken&
 MasterLexer::getNextToken(Options options) {
-    // If the source is not available
     if (impl_->source_ == NULL) {
         isc_throw(isc::InvalidOperation, "No source to read tokens from");
     }
@@ -178,7 +177,7 @@ MasterLexer::getNextToken(Options options) {
     impl_->has_previous_ = true;
     // Reset the token now. This is to check a token was actually produced.
     // This is debugging aid.
-    impl_->token_ = Token(Token::NO_TOKEN_PRODUCED);
+    impl_->token_ = MasterToken(MasterToken::NO_TOKEN_PRODUCED);
     // And get the token
 
     // This actually handles EOF internally too.
@@ -188,8 +187,62 @@ MasterLexer::getNextToken(Options options) {
     }
     // Make sure a token was produced. Since this Can Not Happen, we assert
     // here instead of throwing.
-    assert(impl_->token_.getType() != Token::ERROR ||
-           impl_->token_.getErrorCode() != Token::NO_TOKEN_PRODUCED);
+    assert(impl_->token_.getType() != MasterToken::ERROR ||
+           impl_->token_.getErrorCode() != MasterToken::NO_TOKEN_PRODUCED);
+    return (impl_->token_);
+}
+
+namespace {
+inline MasterLexer::Options
+optionsForTokenType(MasterToken::Type expect) {
+    switch (expect) {
+    case MasterToken::STRING:
+        return (MasterLexer::NONE);
+    case MasterToken::QSTRING:
+        return (MasterLexer::QSTRING);
+    case MasterToken::NUMBER:
+        return (MasterLexer::NUMBER);
+    default:
+        isc_throw(InvalidParameter,
+                  "expected type for getNextToken not supported: " << expect);
+    }
+}
+}
+
+const MasterToken&
+MasterLexer::getNextToken(MasterToken::Type expect, bool eol_ok) {
+    // Get the next token, specifying an appropriate option corresponding to
+    // the expected type.  The result should be set in impl_->token_.
+    getNextToken(optionsForTokenType(expect));
+
+    if (impl_->token_.getType() == MasterToken::ERROR) {
+        if (impl_->token_.getErrorCode() == MasterToken::NUMBER_OUT_OF_RANGE) {
+            ungetToken();
+        }
+        throw LexerError(__FILE__, __LINE__, impl_->token_);
+    }
+
+    const bool is_eol_like =
+        (impl_->token_.getType() == MasterToken::END_OF_LINE ||
+         impl_->token_.getType() == MasterToken::END_OF_FILE);
+    if (eol_ok && is_eol_like) {
+        return (impl_->token_);
+    }
+    if (impl_->token_.getType() == MasterToken::STRING &&
+        expect == MasterToken::QSTRING) {
+        return (impl_->token_);
+    }
+    if (impl_->token_.getType() != expect) {
+        ungetToken();
+        if (is_eol_like) {
+            throw LexerError(__FILE__, __LINE__,
+                             MasterToken(MasterToken::UNEXPECTED_END));
+        }
+        assert(expect == MasterToken::NUMBER);
+        throw LexerError(__FILE__, __LINE__,
+                         MasterToken(MasterToken::BAD_NUMBER));
+    }
+
     return (impl_->token_);
 }
 
@@ -212,16 +265,17 @@ const char* const error_text[] = {
     "unexpected end of input",  // UNEXPECTED_END
     "unbalanced quotes",        // UNBALANCED_QUOTES
     "no token produced",        // NO_TOKEN_PRODUCED
-    "number out of range"       // NUMBER_OUT_OF_RANGE
+    "number out of range",      // NUMBER_OUT_OF_RANGE
+    "not a valid number"        // BAD_NUMBER
 };
 const size_t error_text_max_count = sizeof(error_text) / sizeof(error_text[0]);
 } // end unnamed namespace
 
 std::string
-MasterLexer::Token::getErrorText() const {
+MasterToken::getErrorText() const {
     if (type_ != ERROR) {
         isc_throw(InvalidOperation,
-                  "Token::getErrorText() for non error type");
+                  "MasterToken::getErrorText() for non error type");
     }
 
     // The class integrity ensures the following:
@@ -234,14 +288,12 @@ namespace master_lexer_internal {
 // Note that these need to be defined here so that they can refer to
 // the details of MasterLexerImpl.
 
-typedef MasterLexer::Token Token; // convenience shortcut
-
 bool
 State::wasLastEOL(const MasterLexer& lexer) const {
     return (lexer.impl_->last_was_eol_);
 }
 
-const MasterLexer::Token&
+const MasterToken&
 State::getToken(const MasterLexer& lexer) const {
     return (lexer.impl_->token_);
 }
@@ -271,7 +323,7 @@ public:
         if (c != '\n') {
             getLexerImpl(lexer)->source_->ungetChar();
         }
-        getLexerImpl(lexer)->token_ = Token(Token::END_OF_LINE);
+        getLexerImpl(lexer)->token_ = MasterToken(MasterToken::END_OF_LINE);
         getLexerImpl(lexer)->last_was_eol_ = true;
     }
 };
@@ -342,24 +394,24 @@ State::start(MasterLexer& lexer, MasterLexer::Options options) {
         if (c == InputSource::END_OF_STREAM) {
             lexerimpl.last_was_eol_ = false;
             if (paren_count != 0) {
-                lexerimpl.token_ = Token(Token::UNBALANCED_PAREN);
+                lexerimpl.token_ = MasterToken(MasterToken::UNBALANCED_PAREN);
                 paren_count = 0; // reset to 0; this helps in lenient mode.
                 return (NULL);
             }
-            lexerimpl.token_ = Token(Token::END_OF_FILE);
+            lexerimpl.token_ = MasterToken(MasterToken::END_OF_FILE);
             return (NULL);
         } else if (c == ' ' || c == '\t') {
             // If requested and we are not in (), recognize the initial space.
             if (lexerimpl.last_was_eol_ && paren_count == 0 &&
                 (options & MasterLexer::INITIAL_WS) != 0) {
                 lexerimpl.last_was_eol_ = false;
-                lexerimpl.token_ = Token(Token::INITIAL_WS);
+                lexerimpl.token_ = MasterToken(MasterToken::INITIAL_WS);
                 return (NULL);
             }
         } else if (c == '\n') {
             lexerimpl.last_was_eol_ = true;
             if (paren_count == 0) { // we don't recognize EOL if we are in ()
-                lexerimpl.token_ = Token(Token::END_OF_LINE);
+                lexerimpl.token_ = MasterToken(MasterToken::END_OF_LINE);
                 return (NULL);
             }
         } else if (c == '\r') {
@@ -375,7 +427,7 @@ State::start(MasterLexer& lexer, MasterLexer::Options options) {
         } else if (c == ')') {
             lexerimpl.last_was_eol_ = false;
             if (paren_count == 0) {
-                lexerimpl.token_ = Token(Token::UNBALANCED_PAREN);
+                lexerimpl.token_ = MasterToken(MasterToken::UNBALANCED_PAREN);
                 return (NULL);
             }
             --paren_count;
@@ -407,7 +459,7 @@ String::handle(MasterLexer& lexer) const {
         if (getLexerImpl(lexer)->isTokenEnd(c, escaped)) {
             getLexerImpl(lexer)->source_->ungetChar();
             getLexerImpl(lexer)->token_ =
-                MasterLexer::Token(&data.at(0), data.size());
+                MasterToken(&data.at(0), data.size());
             return;
         }
         escaped = (c == '\\' && !escaped);
@@ -417,7 +469,7 @@ String::handle(MasterLexer& lexer) const {
 
 void
 QString::handle(MasterLexer& lexer) const {
-    MasterLexer::Token& token = getLexerImpl(lexer)->token_;
+    MasterToken& token = getLexerImpl(lexer)->token_;
     std::vector<char>& data = getLexerImpl(lexer)->data_;
     data.clear();
 
@@ -425,7 +477,7 @@ QString::handle(MasterLexer& lexer) const {
     while (true) {
         const int c = getLexerImpl(lexer)->source_->getChar();
         if (c == InputSource::END_OF_STREAM) {
-            token = Token(Token::UNEXPECTED_END);
+            token = MasterToken(MasterToken::UNEXPECTED_END);
             return;
         } else if (c == '"') {
             if (escaped) {
@@ -434,12 +486,12 @@ QString::handle(MasterLexer& lexer) const {
                 escaped = false;
                 data.back() = '"';
             } else {
-                token = MasterLexer::Token(&data.at(0), data.size(), true);
+                token = MasterToken(&data.at(0), data.size(), true);
                 return;
             }
         } else if (c == '\n' && !escaped) {
             getLexerImpl(lexer)->source_->ungetChar();
-            token = Token(Token::UNBALANCED_QUOTES);
+            token = MasterToken(MasterToken::UNBALANCED_QUOTES);
             return;
         } else {
             escaped = (c == '\\' && !escaped);
@@ -450,7 +502,7 @@ QString::handle(MasterLexer& lexer) const {
 
 void
 Number::handle(MasterLexer& lexer) const {
-    MasterLexer::Token& token = getLexerImpl(lexer)->token_;
+    MasterToken& token = getLexerImpl(lexer)->token_;
 
     // It may yet turn out to be a string, so we first
     // collect all the data
@@ -470,15 +522,14 @@ Number::handle(MasterLexer& lexer) const {
                 try {
                     const uint32_t number32 =
                         boost::lexical_cast<uint32_t, const char*>(&data[0]);
-                    token = MasterLexer::Token(number32);
+                    token = MasterToken(number32);
                 } catch (const boost::bad_lexical_cast&) {
                     // Since we already know we have only digits,
                     // range should be the only possible problem.
-                    token = Token(Token::NUMBER_OUT_OF_RANGE);
+                    token = MasterToken(MasterToken::NUMBER_OUT_OF_RANGE);
                 }
             } else {
-                token = MasterLexer::Token(&data.at(0),
-                                           data.size());
+                token = MasterToken(&data.at(0), data.size());
             }
             return;
         }

src/lib/dns/master_lexer_state.h

@@ -43,10 +43,10 @@ namespace master_lexer_internal {
 /// state, so it makes more sense to separate the interface for the transition
 /// from the initial state.
 ///
-/// When an object of a specific state class completes the session, it
-/// normally sets the identified token in the lexer, and returns NULL;
-/// if more transition is necessary, it returns a pointer to the next state
-/// object.
+/// If the whole lexer transition is completed within start(), it sets the
+/// identified token and returns NULL; otherwise it returns a pointer to
+/// an object of a specific state class that completes the session
+/// on the call of handle().
 ///
 /// As is usual in the state design pattern, the \c State class is made
 /// a friend class of \c MasterLexer and can refer to its internal details.
@@ -119,7 +119,7 @@ public:
     /// purposes.
     ///@{
     bool wasLastEOL(const MasterLexer& lexer) const;
-    const MasterLexer::Token& getToken(const MasterLexer& lexer) const;
+    const MasterToken& getToken(const MasterLexer& lexer) const;
     size_t getParenCount(const MasterLexer& lexer) const;
     ///@}
 

src/lib/dns/tests/master_lexer_state_unittest.cc

@@ -24,7 +24,7 @@ using namespace isc::dns;
 using namespace master_lexer_internal;
 
 namespace {
-typedef MasterLexer::Token Token; // shortcut
+typedef MasterToken Token; // shortcut
 
 class MasterLexerStateTest : public ::testing::Test {
 protected:
@@ -260,7 +260,7 @@ TEST_F(MasterLexerStateTest, crlf) {
 // Commonly used check for string related test cases, checking if the given
 // token has expected values.
 void
-stringTokenCheck(const std::string& expected, const MasterLexer::Token& token,
+stringTokenCheck(const std::string& expected, const MasterToken& token,
                  bool quoted = false)
 {
     EXPECT_EQ(quoted ? Token::QSTRING : Token::STRING, token.getType());

src/lib/dns/tests/master_lexer_token_unittest.cc

@@ -31,27 +31,27 @@ const size_t TEST_STRING_LEN = sizeof(TEST_STRING) - 1;
 class MasterLexerTokenTest : public ::testing::Test {
 protected:
     MasterLexerTokenTest() :
-        token_eof(MasterLexer::Token::END_OF_FILE),
+        token_eof(MasterToken::END_OF_FILE),
         token_str(TEST_STRING, TEST_STRING_LEN),
         token_num(42),
-        token_err(MasterLexer::Token::UNEXPECTED_END)
+        token_err(MasterToken::UNEXPECTED_END)
     {}
 
-    const MasterLexer::Token token_eof; // an example of non-value type token
-    const MasterLexer::Token token_str;
-    const MasterLexer::Token token_num;
-    const MasterLexer::Token token_err;
+    const MasterToken token_eof; // an example of non-value type token
+    const MasterToken token_str;
+    const MasterToken token_num;
+    const MasterToken token_err;
 };
 
 
 TEST_F(MasterLexerTokenTest, strings) {
     // basic construction and getter checks
-    EXPECT_EQ(MasterLexer::Token::STRING, token_str.getType());
+    EXPECT_EQ(MasterToken::STRING, token_str.getType());
     EXPECT_EQ(std::string("string token"), token_str.getString());
     std::string strval = "dummy"; // this should be replaced
     token_str.getString(strval);
     EXPECT_EQ(std::string("string token"), strval);
-    const MasterLexer::Token::StringRegion str_region =
+    const MasterToken::StringRegion str_region =
         token_str.getStringRegion();
     EXPECT_EQ(TEST_STRING, str_region.beg);
     EXPECT_EQ(TEST_STRING_LEN, str_region.len);
@@ -62,17 +62,17 @@ TEST_F(MasterLexerTokenTest, strings) {
     std::string expected_str("string token");
     expected_str.push_back('\0');
     EXPECT_EQ(expected_str,
-              MasterLexer::Token(TEST_STRING, TEST_STRING_LEN + 1).getString());
-    MasterLexer::Token(TEST_STRING, TEST_STRING_LEN + 1).getString(strval);
+              MasterToken(TEST_STRING, TEST_STRING_LEN + 1).getString());
+    MasterToken(TEST_STRING, TEST_STRING_LEN + 1).getString(strval);
     EXPECT_EQ(expected_str, strval);
 
     // Construct type of qstring
-    EXPECT_EQ(MasterLexer::Token::QSTRING,
-              MasterLexer::Token(TEST_STRING, sizeof(TEST_STRING), true).
+    EXPECT_EQ(MasterToken::QSTRING,
+              MasterToken(TEST_STRING, sizeof(TEST_STRING), true).
               getType());
     // if we explicitly set 'quoted' to false, it should be normal string
-    EXPECT_EQ(MasterLexer::Token::STRING,
-              MasterLexer::Token(TEST_STRING, sizeof(TEST_STRING), false).
+    EXPECT_EQ(MasterToken::STRING,
+              MasterToken(TEST_STRING, sizeof(TEST_STRING), false).
               getType());
 
     // getString/StringRegion() aren't allowed for non string(-variant) types
@@ -86,23 +86,23 @@ TEST_F(MasterLexerTokenTest, strings) {
 
 TEST_F(MasterLexerTokenTest, numbers) {
     EXPECT_EQ(42, token_num.getNumber());
-    EXPECT_EQ(MasterLexer::Token::NUMBER, token_num.getType());
+    EXPECT_EQ(MasterToken::NUMBER, token_num.getType());
 
     // It's copyable and assignable.
-    MasterLexer::Token token(token_num);
+    MasterToken token(token_num);
     EXPECT_EQ(42, token.getNumber());
-    EXPECT_EQ(MasterLexer::Token::NUMBER, token.getType());
+    EXPECT_EQ(MasterToken::NUMBER, token.getType());
 
     token = token_num;
     EXPECT_EQ(42, token.getNumber());
-    EXPECT_EQ(MasterLexer::Token::NUMBER, token.getType());
+    EXPECT_EQ(MasterToken::NUMBER, token.getType());
 
     // it's okay to replace it with a different type of token
     token = token_eof;
-    EXPECT_EQ(MasterLexer::Token::END_OF_FILE, token.getType());
+    EXPECT_EQ(MasterToken::END_OF_FILE, token.getType());
 
     // Possible max value
-    token = MasterLexer::Token(0xffffffff);
+    token = MasterToken(0xffffffff);
     EXPECT_EQ(4294967295u, token.getNumber());
 
     // getNumber() isn't allowed for non number types
@@ -112,58 +112,52 @@ TEST_F(MasterLexerTokenTest, numbers) {
 
 TEST_F(MasterLexerTokenTest, novalues) {
     // Just checking we can construct them and getType() returns correct value.
-    EXPECT_EQ(MasterLexer::Token::END_OF_FILE, token_eof.getType());
-    EXPECT_EQ(MasterLexer::Token::END_OF_LINE,
-              MasterLexer::Token(MasterLexer::Token::END_OF_LINE).getType());
-    EXPECT_EQ(MasterLexer::Token::INITIAL_WS,
-              MasterLexer::Token(MasterLexer::Token::INITIAL_WS).getType());
+    EXPECT_EQ(MasterToken::END_OF_FILE, token_eof.getType());
+    EXPECT_EQ(MasterToken::END_OF_LINE,
+              MasterToken(MasterToken::END_OF_LINE).getType());
+    EXPECT_EQ(MasterToken::INITIAL_WS,
+              MasterToken(MasterToken::INITIAL_WS).getType());
 
     // Special types of tokens cannot have value-based types
-    EXPECT_THROW(MasterLexer::Token t(MasterLexer::Token::STRING),
-                 isc::InvalidParameter);
-    EXPECT_THROW(MasterLexer::Token t(MasterLexer::Token::QSTRING),
-                 isc::InvalidParameter);
-    EXPECT_THROW(MasterLexer::Token t(MasterLexer::Token::NUMBER),
-                 isc::InvalidParameter);
-    EXPECT_THROW(MasterLexer::Token t(MasterLexer::Token::ERROR),
-                 isc::InvalidParameter);
+    EXPECT_THROW(MasterToken t(MasterToken::STRING), isc::InvalidParameter);
+    EXPECT_THROW(MasterToken t(MasterToken::QSTRING), isc::InvalidParameter);
+    EXPECT_THROW(MasterToken t(MasterToken::NUMBER), isc::InvalidParameter);
+    EXPECT_THROW(MasterToken t(MasterToken::ERROR), isc::InvalidParameter);
 }
 
 TEST_F(MasterLexerTokenTest, errors) {
-    EXPECT_EQ(MasterLexer::Token::ERROR, token_err.getType());
-    EXPECT_EQ(MasterLexer::Token::UNEXPECTED_END, token_err.getErrorCode());
+    EXPECT_EQ(MasterToken::ERROR, token_err.getType());
+    EXPECT_EQ(MasterToken::UNEXPECTED_END, token_err.getErrorCode());
     EXPECT_EQ("unexpected end of input", token_err.getErrorText());
-    EXPECT_EQ("lexer not started",
-              MasterLexer::Token(MasterLexer::Token::NOT_STARTED).
+    EXPECT_EQ("lexer not started", MasterToken(MasterToken::NOT_STARTED).
               getErrorText());
     EXPECT_EQ("unbalanced parentheses",
-              MasterLexer::Token(MasterLexer::Token::UNBALANCED_PAREN).
+              MasterToken(MasterToken::UNBALANCED_PAREN).
               getErrorText());
-    EXPECT_EQ("unbalanced quotes",
-              MasterLexer::Token(MasterLexer::Token::UNBALANCED_QUOTES).
+    EXPECT_EQ("unbalanced quotes", MasterToken(MasterToken::UNBALANCED_QUOTES).
               getErrorText());
-    EXPECT_EQ("no token produced",
-              MasterLexer::Token(MasterLexer::Token::NO_TOKEN_PRODUCED).
+    EXPECT_EQ("no token produced", MasterToken(MasterToken::NO_TOKEN_PRODUCED).
               getErrorText());
     EXPECT_EQ("number out of range",
-              MasterLexer::Token(MasterLexer::Token::NUMBER_OUT_OF_RANGE).
+              MasterToken(MasterToken::NUMBER_OUT_OF_RANGE).
               getErrorText());
+    EXPECT_EQ("not a valid number",
+              MasterToken(MasterToken::BAD_NUMBER).getErrorText());
 
     // getErrorCode/Text() isn't allowed for non number types
     EXPECT_THROW(token_num.getErrorCode(), isc::InvalidOperation);
     EXPECT_THROW(token_num.getErrorText(), isc::InvalidOperation);
 
-    // Only the pre-defined error code is accepted.  Hardcoding '6' (max code
+    // Only the pre-defined error code is accepted.  Hardcoding '7' (max code
     // + 1) is intentional; it'd be actually better if we notice it when we
     // update the enum list (which shouldn't happen too often).
-    EXPECT_THROW(MasterLexer::Token(MasterLexer::Token::ErrorCode(6)),
+    EXPECT_THROW(MasterToken(MasterToken::ErrorCode(7)),
                  isc::InvalidParameter);
 
     // Check the coexistence of "from number" and "from error-code"
     // constructors won't cause confusion.
-    EXPECT_EQ(MasterLexer::Token::NUMBER,
-              MasterLexer::Token(static_cast<uint32_t>(
-                                     MasterLexer::Token::NOT_STARTED)).
+    EXPECT_EQ(MasterToken::NUMBER,
+              MasterToken(static_cast<uint32_t>(MasterToken::NOT_STARTED)).
               getType());
 }
 }

src/lib/dns/tests/master_lexer_unittest.cc

@@ -141,19 +141,19 @@ TEST_F(MasterLexerTest, getNextToken) {
     lexer.pushSource(ss);
 
     // First, the newline should get out.
-    EXPECT_EQ(MasterLexer::Token::END_OF_LINE, lexer.getNextToken().getType());
+    EXPECT_EQ(MasterToken::END_OF_LINE, lexer.getNextToken().getType());
     // Then the whitespace, if we specify the option.
-    EXPECT_EQ(MasterLexer::Token::INITIAL_WS,
+    EXPECT_EQ(MasterToken::INITIAL_WS,
               lexer.getNextToken(MasterLexer::INITIAL_WS).getType());
     // The newline
-    EXPECT_EQ(MasterLexer::Token::END_OF_LINE, lexer.getNextToken().getType());
+    EXPECT_EQ(MasterToken::END_OF_LINE, lexer.getNextToken().getType());
     // The (quoted) string
-    EXPECT_EQ(MasterLexer::Token::QSTRING,
+    EXPECT_EQ(MasterToken::QSTRING,
               lexer.getNextToken(MasterLexer::QSTRING).getType());
 
     // And the end of line and file
-    EXPECT_EQ(MasterLexer::Token::END_OF_LINE, lexer.getNextToken().getType());
-    EXPECT_EQ(MasterLexer::Token::END_OF_FILE, lexer.getNextToken().getType());
+    EXPECT_EQ(MasterToken::END_OF_LINE, lexer.getNextToken().getType());
+    EXPECT_EQ(MasterToken::END_OF_FILE, lexer.getNextToken().getType());
 }
 
 // Test we correctly find end of file.
@@ -162,12 +162,12 @@ TEST_F(MasterLexerTest, eof) {
     lexer.pushSource(ss);
 
     // The first one is found to be EOF
-    EXPECT_EQ(MasterLexer::Token::END_OF_FILE, lexer.getNextToken().getType());
+    EXPECT_EQ(MasterToken::END_OF_FILE, lexer.getNextToken().getType());
     // And it stays on EOF for any following attempts
-    EXPECT_EQ(MasterLexer::Token::END_OF_FILE, lexer.getNextToken().getType());
+    EXPECT_EQ(MasterToken::END_OF_FILE, lexer.getNextToken().getType());
     // And we can step back one token, but that is the EOF too.
     lexer.ungetToken();
-    EXPECT_EQ(MasterLexer::Token::END_OF_FILE, lexer.getNextToken().getType());
+    EXPECT_EQ(MasterToken::END_OF_FILE, lexer.getNextToken().getType());
 }
 
 // Check we properly return error when there's an opened parentheses and no
@@ -177,12 +177,12 @@ TEST_F(MasterLexerTest, getUnbalancedParen) {
     lexer.pushSource(ss);
 
     // The string gets out first
-    EXPECT_EQ(MasterLexer::Token::STRING, lexer.getNextToken().getType());
+    EXPECT_EQ(MasterToken::STRING, lexer.getNextToken().getType());
     // Then an unbalanced parenthesis
-    EXPECT_EQ(MasterLexer::Token::UNBALANCED_PAREN,
+    EXPECT_EQ(MasterToken::UNBALANCED_PAREN,
               lexer.getNextToken().getErrorCode());
     // And then EOF
-    EXPECT_EQ(MasterLexer::Token::END_OF_FILE, lexer.getNextToken().getType());
+    EXPECT_EQ(MasterToken::END_OF_FILE, lexer.getNextToken().getType());
 }
 
 // Check we properly return error when there's an opened quoted string and no
@@ -192,10 +192,10 @@ TEST_F(MasterLexerTest, getUnbalancedString) {
     lexer.pushSource(ss);
 
     // Then an unbalanced qstring (reported as an unexpected end)
-    EXPECT_EQ(MasterLexer::Token::UNEXPECTED_END,
+    EXPECT_EQ(MasterToken::UNEXPECTED_END,
               lexer.getNextToken(MasterLexer::QSTRING).getErrorCode());
     // And then EOF
-    EXPECT_EQ(MasterLexer::Token::END_OF_FILE, lexer.getNextToken().getType());
+    EXPECT_EQ(MasterToken::END_OF_FILE, lexer.getNextToken().getType());
 }
 
 // Test ungetting tokens works
@@ -204,28 +204,28 @@ TEST_F(MasterLexerTest, ungetToken) {
     lexer.pushSource(ss);
 
     // Try getting the newline
-    EXPECT_EQ(MasterLexer::Token::END_OF_LINE, lexer.getNextToken().getType());
+    EXPECT_EQ(MasterToken::END_OF_LINE, lexer.getNextToken().getType());
     // Return it and get again
     lexer.ungetToken();
-    EXPECT_EQ(MasterLexer::Token::END_OF_LINE, lexer.getNextToken().getType());
+    EXPECT_EQ(MasterToken::END_OF_LINE, lexer.getNextToken().getType());
     // Get the string and return it back
-    EXPECT_EQ(MasterLexer::Token::QSTRING,
+    EXPECT_EQ(MasterToken::QSTRING,
               lexer.getNextToken(MasterLexer::QSTRING).getType());
     lexer.ungetToken();
     // But if we change the options, it honors them
-    EXPECT_EQ(MasterLexer::Token::INITIAL_WS,
+    EXPECT_EQ(MasterToken::INITIAL_WS,
               lexer.getNextToken(MasterLexer::QSTRING |
                                  MasterLexer::INITIAL_WS).getType());
     // Get to the "more" string
-    EXPECT_EQ(MasterLexer::Token::QSTRING,
+    EXPECT_EQ(MasterToken::QSTRING,
               lexer.getNextToken(MasterLexer::QSTRING).getType());
-    EXPECT_EQ(MasterLexer::Token::STRING,
+    EXPECT_EQ(MasterToken::STRING,
               lexer.getNextToken(MasterLexer::QSTRING).getType());
     // Return it back. It should get inside the parentheses.
     // Upon next attempt to get it again, the newline inside the parentheses
     // should be still ignored.
     lexer.ungetToken();
-    EXPECT_EQ(MasterLexer::Token::STRING,
+    EXPECT_EQ(MasterToken::STRING,
               lexer.getNextToken(MasterLexer::QSTRING).getType());
 }
 
@@ -235,16 +235,16 @@ TEST_F(MasterLexerTest, ungetRealOptions) {
     ss << "\n    \n";
     lexer.pushSource(ss);
     // Skip the first newline
-    EXPECT_EQ(MasterLexer::Token::END_OF_LINE, lexer.getNextToken().getType());
+    EXPECT_EQ(MasterToken::END_OF_LINE, lexer.getNextToken().getType());
 
     // If we call it the usual way, it skips up to the newline and returns
     // it
-    EXPECT_EQ(MasterLexer::Token::END_OF_LINE, lexer.getNextToken().getType());
+    EXPECT_EQ(MasterToken::END_OF_LINE, lexer.getNextToken().getType());
 
     // Now we return it. If we call it again, but with different options,
     // we get the initial whitespace.
     lexer.ungetToken();
-    EXPECT_EQ(MasterLexer::Token::INITIAL_WS,
+    EXPECT_EQ(MasterToken::INITIAL_WS,
               lexer.getNextToken(MasterLexer::INITIAL_WS).getType());
 }
 
@@ -253,7 +253,7 @@ TEST_F(MasterLexerTest, ungetTwice) {
     ss << "\n";
     lexer.pushSource(ss);
 
-    EXPECT_EQ(MasterLexer::Token::END_OF_LINE, lexer.getNextToken().getType());
+    EXPECT_EQ(MasterToken::END_OF_LINE, lexer.getNextToken().getType());
     // Unget the token. It can be done once
     lexer.ungetToken();