Commit d25a434d authored by JINMEI Tatuya's avatar JINMEI Tatuya
Browse files

[2373] some more comment updates

parent 1ed3ca78
......@@ -64,9 +64,13 @@ struct MasterLexer::MasterLexerImpl {
}
bool isTokenEnd(int c, bool escaped) {
// Special case of EOF (end of stream); this is not in the bitmaps
if (c == InputSource::END_OF_STREAM) {
return (true);
}
// In this implementation we only ensure the behavior for unsigned
// range of characters, so we restrict the range of the values up to
// 0x7f = 127
return (escaped ? esc_separators_.test(c & 0x7f) :
separators_.test(c & 0x7f));
}
......@@ -84,7 +88,7 @@ struct MasterLexer::MasterLexerImpl {
// Bitmaps that gives whether a given (positive) character should be
// considered a separator of a string/number token. The esc_ version
// is a subset of the other, excluding characters that can be ignored
// if escaped by a backslash.
// if escaped by a backslash. See isTokenEnd() for the bitmap size.
std::bitset<128> separators_;
std::bitset<128> esc_separators_;
};
......
......@@ -255,6 +255,8 @@ TEST_F(MasterLexerStateTest, crlf) {
EXPECT_EQ(Token::END_OF_FILE, s_crlf.getToken(lexer).getType());
}
// Commonly used check for string related test cases, checking if the given
// token has expected values.
void
stringTokenCheck(const std::string& expected, const MasterLexer::Token& token,
bool quoted = false)
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment