Commit b89f0e0e authored by Jelte Jansen's avatar Jelte Jansen
Browse files

[master] Merge branch 'trac2374'

parents 28d885b4 a5683591
......@@ -212,7 +212,7 @@ const char* const error_text[] = {
"unexpected end of input", // UNEXPECTED_END
"unbalanced quotes", // UNBALANCED_QUOTES
"no token produced", // NO_TOKEN_PRODUCED
"number out of range" // NUMBER_RANGE
"number out of range" // NUMBER_OUT_OF_RANGE
};
const size_t error_text_max_count = sizeof(error_text) / sizeof(error_text[0]);
} // end unnamed namespace
......@@ -379,7 +379,7 @@ State::start(MasterLexer& lexer, MasterLexer::Options options) {
return (NULL);
}
--paren_count;
} else if (isdigit(c)) {
} else if ((options & MasterLexer::NUMBER) != 0 &&isdigit(c)) {
lexerimpl.last_was_eol_ = false;
// this character will be handled in the number state
lexerimpl.source_->ungetChar();
......@@ -469,12 +469,12 @@ Number::handle(MasterLexer& lexer) const {
data.push_back('\0');
try {
const uint32_t number32 =
boost::lexical_cast<uint32_t, const char*>(&data.at(0));
boost::lexical_cast<uint32_t, const char*>(&data[0]);
token = MasterLexer::Token(number32);
} catch (const boost::bad_lexical_cast&) {
// Since we already know we have only digits,
// range should be the only possible problem.
token = Token(Token::NUMBER_RANGE);
token = Token(Token::NUMBER_OUT_OF_RANGE);
}
} else {
token = MasterLexer::Token(&data.at(0),
......
......@@ -292,7 +292,7 @@ public:
UNBALANCED_QUOTES, ///< Unbalanced quotations detected
NO_TOKEN_PRODUCED, ///< No token was produced. This means programmer
/// error and should never get out of the lexer.
NUMBER_RANGE, ///< Number was out of range
NUMBER_OUT_OF_RANGE, ///< Number was out of range
MAX_ERROR_CODE ///< Max integer corresponding to valid error codes.
/// (excluding this one). Mainly for internal use.
};
......
......@@ -467,45 +467,48 @@ TEST_F(MasterLexerStateTest, basicNumbers) {
// the tokenizer
lexer.pushSource(ss);
EXPECT_EQ(&s_number, State::start(lexer, common_options));
// Ask the lexer to recognize numbers as well
const MasterLexer::Options options = common_options | MasterLexer::NUMBER;
EXPECT_EQ(&s_number, State::start(lexer, options));
s_number.handle(lexer);
EXPECT_EQ(0, s_number.getToken(lexer).getNumber());
EXPECT_EQ(&s_number, State::start(lexer, common_options));
EXPECT_EQ(&s_number, State::start(lexer, options));
s_number.handle(lexer);
EXPECT_EQ(1, s_number.getToken(lexer).getNumber());
EXPECT_EQ(&s_number, State::start(lexer, common_options));
EXPECT_EQ(&s_number, State::start(lexer, options));
s_number.handle(lexer);
EXPECT_EQ(12345, s_number.getToken(lexer).getNumber());
EXPECT_EQ(&s_number, State::start(lexer, common_options));
EXPECT_EQ(&s_number, State::start(lexer, options));
s_number.handle(lexer);
EXPECT_EQ(4294967295u, s_number.getToken(lexer).getNumber());
EXPECT_EQ(&s_number, State::start(lexer, common_options));
EXPECT_EQ(&s_number, State::start(lexer, options));
s_number.handle(lexer);
EXPECT_EQ(Token::NUMBER_RANGE,
EXPECT_EQ(Token::NUMBER_OUT_OF_RANGE,
s_number.getToken(lexer).getErrorCode());
EXPECT_EQ(&s_number, State::start(lexer, common_options));
EXPECT_EQ(&s_number, State::start(lexer, options));
s_number.handle(lexer);
EXPECT_EQ(Token::NUMBER_RANGE,
EXPECT_EQ(Token::NUMBER_OUT_OF_RANGE,
s_number.getToken(lexer).getErrorCode());
EXPECT_EQ(&s_number, State::start(lexer, common_options));
EXPECT_EQ(&s_number, State::start(lexer, options));
s_number.handle(lexer);
EXPECT_EQ(5, s_number.getToken(lexer).getNumber());
EXPECT_EQ(&s_number, State::start(lexer, common_options));
EXPECT_EQ(&s_number, State::start(lexer, options));
s_number.handle(lexer);
EXPECT_EQ(42, s_number.getToken(lexer).getNumber());
EXPECT_EQ(s_null, State::start(lexer, common_options));
EXPECT_EQ(s_null, State::start(lexer, options));
EXPECT_TRUE(s_crlf.wasLastEOL(lexer));
EXPECT_EQ(Token::END_OF_LINE, s_crlf.getToken(lexer).getType());
EXPECT_EQ(&s_number, State::start(lexer, common_options));
EXPECT_EQ(&s_number, State::start(lexer, options));
s_number.handle(lexer);
EXPECT_EQ(37, s_number.getToken(lexer).getNumber());
......@@ -517,6 +520,8 @@ TEST_F(MasterLexerStateTest, basicNumbers) {
// Test tokens that look like (or start out as) numbers,
// but turn out to be strings. Tests include escaped characters.
TEST_F(MasterLexerStateTest, stringNumbers) {
ss << "123 "; // Should be read as a string if the
// NUMBER option is not given
ss << "-1 "; // Negative numbers are interpreted
// as strings (unsigned integers only)
ss << "123abc456 "; // 'Numbers' containing non-digits should
......@@ -532,45 +537,54 @@ TEST_F(MasterLexerStateTest, stringNumbers) {
lexer.pushSource(ss);
// Note that common_options does not include MasterLexer::NUMBER,
// so the token should be recognized as a string
EXPECT_EQ(&s_string, State::start(lexer, common_options));
s_string.handle(lexer);
stringTokenCheck("123", s_string.getToken(lexer), false);
// Ask the lexer to recognize numbers as well
const MasterLexer::Options options = common_options | MasterLexer::NUMBER;
EXPECT_EQ(&s_string, State::start(lexer, options));
s_string.handle(lexer);
stringTokenCheck("-1", s_string.getToken(lexer), false);
// Starts out as a number, but ends up being a string
EXPECT_EQ(&s_number, State::start(lexer, common_options));
EXPECT_EQ(&s_number, State::start(lexer, options));
s_number.handle(lexer);
stringTokenCheck("123abc456", s_number.getToken(lexer), false);
EXPECT_EQ(&s_number, State::start(lexer, common_options));
EXPECT_EQ(&s_number, State::start(lexer, options));
s_number.handle(lexer);
stringTokenCheck("123\\456", s_number.getToken(lexer), false);
EXPECT_EQ(&s_number, State::start(lexer, common_options));
EXPECT_EQ(&s_number, State::start(lexer, options));
s_number.handle(lexer); // recognize str, see ' ' at end
stringTokenCheck("3scaped\\ space", s_number.getToken(lexer));
EXPECT_EQ(&s_number, State::start(lexer, common_options));
EXPECT_EQ(&s_number, State::start(lexer, options));
s_number.handle(lexer); // recognize str, see ' ' at end
stringTokenCheck("3scaped\\\ttab", s_number.getToken(lexer));
EXPECT_EQ(&s_number, State::start(lexer, common_options));
EXPECT_EQ(&s_number, State::start(lexer, options));
s_number.handle(lexer); // recognize str, see ' ' at end
stringTokenCheck("3scaped\\(paren", s_number.getToken(lexer));
EXPECT_EQ(&s_number, State::start(lexer, common_options));
EXPECT_EQ(&s_number, State::start(lexer, options));
s_number.handle(lexer); // recognize str, see ' ' at end
stringTokenCheck("3scaped\\)close", s_number.getToken(lexer));
EXPECT_EQ(&s_number, State::start(lexer, common_options));
EXPECT_EQ(&s_number, State::start(lexer, options));
s_number.handle(lexer); // recognize str, see ' ' at end
stringTokenCheck("3scaped\\;comment", s_number.getToken(lexer));
EXPECT_EQ(&s_number, State::start(lexer, common_options));
EXPECT_EQ(&s_number, State::start(lexer, options));
s_number.handle(lexer); // recognize str, see ' ' in mid
stringTokenCheck("3scaped\\\\", s_number.getToken(lexer));
// Confirm the word that follows the escaped '\' is correctly recognized.
EXPECT_EQ(&s_number, State::start(lexer, common_options));
EXPECT_EQ(&s_number, State::start(lexer, options));
s_number.handle(lexer); // recognize str, see ' ' at end
stringTokenCheck("8ackslash", s_number.getToken(lexer));
......
......@@ -146,7 +146,7 @@ TEST_F(MasterLexerTokenTest, errors) {
MasterLexer::Token(MasterLexer::Token::NO_TOKEN_PRODUCED).
getErrorText());
EXPECT_EQ("number out of range",
MasterLexer::Token(MasterLexer::Token::NUMBER_RANGE).
MasterLexer::Token(MasterLexer::Token::NUMBER_OUT_OF_RANGE).
getErrorText());
// getErrorCode/Text() isn't allowed for non number types
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment