Commit e206bfd6 authored by Jelte Jansen's avatar Jelte Jansen
Browse files

[2374] Only recognize numbers if MasterLexer::NUMBER is set

parent 8b10f38a
......@@ -379,7 +379,7 @@ State::start(MasterLexer& lexer, MasterLexer::Options options) {
return (NULL);
}
--paren_count;
} else if (isdigit(c)) {
} else if (isdigit(c) && (options & MasterLexer::NUMBER) != 0) {
lexerimpl.last_was_eol_ = false;
// this character will be handled in the number state
lexerimpl.source_->ungetChar();
......
......@@ -467,45 +467,48 @@ TEST_F(MasterLexerStateTest, basicNumbers) {
// the tokenizer
lexer.pushSource(ss);
EXPECT_EQ(&s_number, State::start(lexer, common_options));
// Ask the lexer to recognize numbers as well
const MasterLexer::Options options = common_options | MasterLexer::NUMBER;
EXPECT_EQ(&s_number, State::start(lexer, options));
s_number.handle(lexer);
EXPECT_EQ(0, s_number.getToken(lexer).getNumber());
EXPECT_EQ(&s_number, State::start(lexer, common_options));
EXPECT_EQ(&s_number, State::start(lexer, options));
s_number.handle(lexer);
EXPECT_EQ(1, s_number.getToken(lexer).getNumber());
EXPECT_EQ(&s_number, State::start(lexer, common_options));
EXPECT_EQ(&s_number, State::start(lexer, options));
s_number.handle(lexer);
EXPECT_EQ(12345, s_number.getToken(lexer).getNumber());
EXPECT_EQ(&s_number, State::start(lexer, common_options));
EXPECT_EQ(&s_number, State::start(lexer, options));
s_number.handle(lexer);
EXPECT_EQ(4294967295u, s_number.getToken(lexer).getNumber());
EXPECT_EQ(&s_number, State::start(lexer, common_options));
EXPECT_EQ(&s_number, State::start(lexer, options));
s_number.handle(lexer);
EXPECT_EQ(Token::NUMBER_RANGE,
s_number.getToken(lexer).getErrorCode());
EXPECT_EQ(&s_number, State::start(lexer, common_options));
EXPECT_EQ(&s_number, State::start(lexer, options));
s_number.handle(lexer);
EXPECT_EQ(Token::NUMBER_RANGE,
s_number.getToken(lexer).getErrorCode());
EXPECT_EQ(&s_number, State::start(lexer, common_options));
EXPECT_EQ(&s_number, State::start(lexer, options));
s_number.handle(lexer);
EXPECT_EQ(5, s_number.getToken(lexer).getNumber());
EXPECT_EQ(&s_number, State::start(lexer, common_options));
EXPECT_EQ(&s_number, State::start(lexer, options));
s_number.handle(lexer);
EXPECT_EQ(42, s_number.getToken(lexer).getNumber());
EXPECT_EQ(s_null, State::start(lexer, common_options));
EXPECT_EQ(s_null, State::start(lexer, options));
EXPECT_TRUE(s_crlf.wasLastEOL(lexer));
EXPECT_EQ(Token::END_OF_LINE, s_crlf.getToken(lexer).getType());
EXPECT_EQ(&s_number, State::start(lexer, common_options));
EXPECT_EQ(&s_number, State::start(lexer, options));
s_number.handle(lexer);
EXPECT_EQ(37, s_number.getToken(lexer).getNumber());
......@@ -517,6 +520,8 @@ TEST_F(MasterLexerStateTest, basicNumbers) {
// Test tokens that look like (or start out as) numbers,
// but turn out to be strings. Tests include escaped characters.
TEST_F(MasterLexerStateTest, stringNumbers) {
ss << "123 "; // Should be read as a string if the
// NUMBER option is not given
ss << "-1 "; // Negative numbers are interpreted
// as strings (unsigned integers only)
ss << "123abc456 "; // 'Numbers' containing non-digits should
......@@ -532,45 +537,54 @@ TEST_F(MasterLexerStateTest, stringNumbers) {
lexer.pushSource(ss);
// Note that common_options does not include MasterLexer::NUMBER,
// so the token should be recognized as a string
EXPECT_EQ(&s_string, State::start(lexer, common_options));
s_string.handle(lexer);
stringTokenCheck("123", s_string.getToken(lexer), false);
// Ask the lexer to recognize numbers as well
const MasterLexer::Options options = common_options | MasterLexer::NUMBER;
EXPECT_EQ(&s_string, State::start(lexer, options));
s_string.handle(lexer);
stringTokenCheck("-1", s_string.getToken(lexer), false);
// Starts out as a number, but ends up being a string
EXPECT_EQ(&s_number, State::start(lexer, common_options));
EXPECT_EQ(&s_number, State::start(lexer, options));
s_number.handle(lexer);
stringTokenCheck("123abc456", s_number.getToken(lexer), false);
EXPECT_EQ(&s_number, State::start(lexer, common_options));
EXPECT_EQ(&s_number, State::start(lexer, options));
s_number.handle(lexer);
stringTokenCheck("123\\456", s_number.getToken(lexer), false);
EXPECT_EQ(&s_number, State::start(lexer, common_options));
EXPECT_EQ(&s_number, State::start(lexer, options));
s_number.handle(lexer); // recognize str, see ' ' at end
stringTokenCheck("3scaped\\ space", s_number.getToken(lexer));
EXPECT_EQ(&s_number, State::start(lexer, common_options));
EXPECT_EQ(&s_number, State::start(lexer, options));
s_number.handle(lexer); // recognize str, see ' ' at end
stringTokenCheck("3scaped\\\ttab", s_number.getToken(lexer));
EXPECT_EQ(&s_number, State::start(lexer, common_options));
EXPECT_EQ(&s_number, State::start(lexer, options));
s_number.handle(lexer); // recognize str, see ' ' at end
stringTokenCheck("3scaped\\(paren", s_number.getToken(lexer));
EXPECT_EQ(&s_number, State::start(lexer, common_options));
EXPECT_EQ(&s_number, State::start(lexer, options));
s_number.handle(lexer); // recognize str, see ' ' at end
stringTokenCheck("3scaped\\)close", s_number.getToken(lexer));
EXPECT_EQ(&s_number, State::start(lexer, common_options));
EXPECT_EQ(&s_number, State::start(lexer, options));
s_number.handle(lexer); // recognize str, see ' ' at end
stringTokenCheck("3scaped\\;comment", s_number.getToken(lexer));
EXPECT_EQ(&s_number, State::start(lexer, common_options));
EXPECT_EQ(&s_number, State::start(lexer, options));
s_number.handle(lexer); // recognize str, see ' ' in mid
stringTokenCheck("3scaped\\\\", s_number.getToken(lexer));
// Confirm the word that follows the escaped '\' is correctly recognized.
EXPECT_EQ(&s_number, State::start(lexer, common_options));
EXPECT_EQ(&s_number, State::start(lexer, options));
s_number.handle(lexer); // recognize str, see ' ' at end
stringTokenCheck("8ackslash", s_number.getToken(lexer));
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment