master_lexer_state_unittest.cc 24.2 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
// Copyright (C) 2012  Internet Systems Consortium, Inc. ("ISC")
//
// Permission to use, copy, modify, and/or distribute this software for any
// purpose with or without fee is hereby granted, provided that the above
// copyright notice and this permission notice appear in all copies.
//
// THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH
// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
// AND FITNESS.  IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT,
// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
// OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
// PERFORMANCE OF THIS SOFTWARE.

#include <dns/master_lexer.h>
16
#include <dns/master_lexer_inputsource.h>
17
18
19
20
21
22
23
24
25
26
27
28
29
30
#include <dns/master_lexer_state.h>

#include <gtest/gtest.h>

#include <sstream>

using namespace isc::dns;
using namespace master_lexer_internal;

namespace {
typedef MasterLexer::Token Token; // shortcut

class MasterLexerStateTest : public ::testing::Test {
protected:
31
    MasterLexerStateTest() : common_options(MasterLexer::INITIAL_WS),
32
                             s_null(NULL),
JINMEI Tatuya's avatar
JINMEI Tatuya committed
33
                             s_crlf(State::getInstance(State::CRLF)),
34
                             s_string(State::getInstance(State::String)),
35
                             s_qstring(State::getInstance(State::QString)),
Jelte Jansen's avatar
Jelte Jansen committed
36
                             s_number(State::getInstance(State::Number)),
37
                             options(MasterLexer::NONE),
38
                             orig_options(options)
39
    {}
40

41
    // Specify INITIAL_WS as common initial options.
42
43
    const MasterLexer::Options common_options;
    MasterLexer lexer;
44
    const State* const s_null;
45
    const State& s_crlf;
46
    const State& s_string;
47
    const State& s_qstring;
Jelte Jansen's avatar
Jelte Jansen committed
48
    const State& s_number;
49
    std::stringstream ss;
50
    MasterLexer::Options options, orig_options;
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
};

// Common check for the end-of-file condition.
// Token is set to END_OF_FILE, and the lexer was NOT last eol state.
// Passed state can be any valid one; they are stateless, just providing the
// interface for inspection.
void
eofCheck(const State& state, MasterLexer& lexer) {
    EXPECT_EQ(Token::END_OF_FILE, state.getToken(lexer).getType());
    EXPECT_FALSE(state.wasLastEOL(lexer));
}

TEST_F(MasterLexerStateTest, startAndEnd) {
    // A simple case: the input is empty, so we begin with start and
    // are immediately done.
66
    lexer.pushSource(ss);
67
68
    EXPECT_EQ(s_null, State::start(lexer, common_options));
    eofCheck(s_crlf, lexer);
69
70
71
72
}

TEST_F(MasterLexerStateTest, startToEOL) {
    ss << "\n";
73
74
    lexer.pushSource(ss);

75
76
77
    EXPECT_EQ(s_null, State::start(lexer, common_options));
    EXPECT_TRUE(s_crlf.wasLastEOL(lexer));
    EXPECT_EQ(Token::END_OF_LINE, s_crlf.getToken(lexer).getType());
78
79

    // The next lexer session will reach EOF.  Same eof check should pass.
80
81
    EXPECT_EQ(s_null, State::start(lexer, common_options));
    eofCheck(s_crlf, lexer);
82
}
83

JINMEI Tatuya's avatar
JINMEI Tatuya committed
84
TEST_F(MasterLexerStateTest, space) {
85
86
87
88
    // repeat '\t\n' twice (see below), then space after EOL
    ss << " \t\n\t\n ";
    lexer.pushSource(ss);

JINMEI Tatuya's avatar
JINMEI Tatuya committed
89
90
91
92
93
    // by default space characters and tabs will be ignored.  We check this
    // twice; at the second iteration, it's a white space at the beginning
    // of line, but since we don't specify INITIAL_WS option, it's treated as
    // normal space and ignored.
    for (size_t i = 0; i < 2; ++i) {
94
95
96
        EXPECT_EQ(s_null, State::start(lexer, MasterLexer::NONE));
        EXPECT_TRUE(s_crlf.wasLastEOL(lexer));
        EXPECT_EQ(Token::END_OF_LINE, s_crlf.getToken(lexer).getType());
JINMEI Tatuya's avatar
JINMEI Tatuya committed
97
98
99
100
    }

    // Now we specify the INITIAL_WS option.  It will be recognized and the
    // corresponding token will be returned.
101
102
103
    EXPECT_EQ(s_null, State::start(lexer, MasterLexer::INITIAL_WS));
    EXPECT_FALSE(s_crlf.wasLastEOL(lexer));
    EXPECT_EQ(Token::INITIAL_WS, s_crlf.getToken(lexer).getType());
JINMEI Tatuya's avatar
JINMEI Tatuya committed
104
105
}

106
TEST_F(MasterLexerStateTest, parentheses) {
107
    ss << "\n(\na\n )\n "; // 1st \n is to check if 'was EOL' is set to false
108
    lexer.pushSource(ss);
109

110
    EXPECT_EQ(s_null, State::start(lexer, common_options)); // handle \n
111

112
    // Now handle '('.  It skips \n and recognize 'a' as string
113
114
115
116
    EXPECT_EQ(0, s_crlf.getParenCount(lexer)); // check pre condition
    EXPECT_EQ(&s_string, State::start(lexer, common_options));
    EXPECT_EQ(1, s_crlf.getParenCount(lexer)); // check post condition
    EXPECT_FALSE(s_crlf.wasLastEOL(lexer));
117

118
    // skip 'a'
119
    s_string.handle(lexer);
120
121
122
123

    // Then handle ')'.  '\n' before ')' isn't recognized because
    // it's canceled due to the '('.  Likewise, the space after the '\n'
    // shouldn't be recognized but should be just ignored.
124
125
    EXPECT_EQ(s_null, State::start(lexer, common_options));
    EXPECT_EQ(0, s_crlf.getParenCount(lexer));
126
127
128

    // Now, temporarily disabled options are restored: Both EOL and the
    // initial WS are recognized
129
130
131
    EXPECT_EQ(Token::END_OF_LINE, s_crlf.getToken(lexer).getType());
    EXPECT_EQ(s_null, State::start(lexer, common_options));
    EXPECT_EQ(Token::INITIAL_WS, s_crlf.getToken(lexer).getType());
132
133
}

134
135
TEST_F(MasterLexerStateTest, nestedParentheses) {
    // This is an unusual, but allowed (in this implementation) case.
136
    ss << "(a(b)\n c)\n ";
137
138
    lexer.pushSource(ss);

139
140
141
142
143
    EXPECT_EQ(&s_string, State::start(lexer, common_options)); // consume '('
    s_string.handle(lexer);                      // consume 'a'
    EXPECT_EQ(&s_string, State::start(lexer, common_options)); // consume '('
    s_string.handle(lexer);                     // consume 'b'
    EXPECT_EQ(2, s_crlf.getParenCount(lexer)); // now the count is 2
144
145

    // Close the inner most parentheses.  count will be decreased, but option
146
147
    // shouldn't be restored yet, so the intermediate EOL or initial WS won't
    // be recognized.
148
149
150
    EXPECT_EQ(&s_string, State::start(lexer, common_options)); // consume ')'
    s_string.handle(lexer);                      // consume 'c'
    EXPECT_EQ(1, s_crlf.getParenCount(lexer));
151
152
153

    // Close the outermost parentheses.  count will be reset to 0, and original
    // options are restored.
154
    EXPECT_EQ(s_null, State::start(lexer, common_options));
155
156
157

    // Now, temporarily disabled options are restored: Both EOL and the
    // initial WS are recognized
158
159
160
    EXPECT_EQ(Token::END_OF_LINE, s_crlf.getToken(lexer).getType());
    EXPECT_EQ(s_null, State::start(lexer, common_options));
    EXPECT_EQ(Token::INITIAL_WS, s_crlf.getToken(lexer).getType());
161
162
}

163
164
165
166
TEST_F(MasterLexerStateTest, unbalancedParentheses) {
    // Only closing paren is provided.  We prepend a \n to check if it's
    // correctly canceled after detecting the error.
    ss << "\n)";
167
168
    ss << "(a";
    lexer.pushSource(ss);
169

170
171
    EXPECT_EQ(s_null, State::start(lexer, common_options)); // consume '\n'
    EXPECT_TRUE(s_crlf.wasLastEOL(lexer)); // this \n was remembered
172
173
174

    // Now checking ')'.  The result should be error, count shouldn't be
    // changed.  "last EOL" should be canceled.
175
176
177
178
179
180
    EXPECT_EQ(0, s_crlf.getParenCount(lexer));
    EXPECT_EQ(s_null, State::start(lexer, common_options));
    EXPECT_EQ(0, s_crlf.getParenCount(lexer));
    ASSERT_EQ(Token::ERROR, s_crlf.getToken(lexer).getType());
    EXPECT_EQ(Token::UNBALANCED_PAREN, s_crlf.getToken(lexer).getErrorCode());
    EXPECT_FALSE(s_crlf.wasLastEOL(lexer));
181

182
    // Reach EOF with a dangling open parenthesis.
183
184
185
186
187
188
189
    EXPECT_EQ(&s_string, State::start(lexer, common_options)); // consume '('
    s_string.handle(lexer);                      // consume 'a'
    EXPECT_EQ(1, s_crlf.getParenCount(lexer));
    EXPECT_EQ(s_null, State::start(lexer, common_options));    // reach EOF
    ASSERT_EQ(Token::ERROR, s_crlf.getToken(lexer).getType());
    EXPECT_EQ(Token::UNBALANCED_PAREN, s_crlf.getToken(lexer).getErrorCode());
    EXPECT_EQ(0, s_crlf.getParenCount(lexer)); // should be reset to 0
190
191
192
193
194
195
196
}

TEST_F(MasterLexerStateTest, startToComment) {
    // Begin with 'start', skip space, then encounter a comment.  Skip
    // the rest of the line, and recognize the new line.  Note that the
    // second ';' is simply ignored.
    ss << "  ;a;\n";
197
198
199
200
    ss << ";a;";           // Likewise, but the comment ends with EOF.
    lexer.pushSource(ss);

    // Comment ending with EOL
201
202
    EXPECT_EQ(s_null, State::start(lexer, common_options));
    EXPECT_EQ(Token::END_OF_LINE, s_crlf.getToken(lexer).getType());
203

204
    // Comment ending with EOF
205
206
    EXPECT_EQ(s_null, State::start(lexer, common_options));
    EXPECT_EQ(Token::END_OF_FILE, s_crlf.getToken(lexer).getType());
207
208
}

209
210
211
212
213
TEST_F(MasterLexerStateTest, commentAfterParen) {
    // comment after an opening parenthesis.  The code that is tested by
    // other tests should also ensure that it works correctly, but we
    // check it explicitly.
    ss << "( ;this is a comment\na)\n";
214
215
    lexer.pushSource(ss);

216
    // consume '(', skip comments, consume 'a', then consume ')'
217
218
219
220
    EXPECT_EQ(&s_string, State::start(lexer, common_options));
    s_string.handle(lexer);
    EXPECT_EQ(s_null, State::start(lexer, common_options));
    EXPECT_EQ(Token::END_OF_LINE, s_crlf.getToken(lexer).getType());
221
222
}

JINMEI Tatuya's avatar
JINMEI Tatuya committed
223
TEST_F(MasterLexerStateTest, crlf) {
224
225
226
227
228
229
230
    ss << "\r\n";               // case 1
    ss << "\r ";                // case 2
    ss << "\r;comment\na";      // case 3
    ss << "\r";                 // case 4
    lexer.pushSource(ss);

    // 1. A sequence of \r, \n is recognized as a single 'end-of-line'
231
    EXPECT_EQ(&s_crlf, State::start(lexer, common_options)); // recognize '\r'
232
    s_crlf.handle(lexer);   // recognize '\n'
233
234
    EXPECT_EQ(Token::END_OF_LINE, s_crlf.getToken(lexer).getType());
    EXPECT_TRUE(s_crlf.wasLastEOL(lexer));
JINMEI Tatuya's avatar
JINMEI Tatuya committed
235

236
237
    // 2. Single '\r' (not followed by \n) is recognized as a single
    // 'end-of-line'.  then there will be "initial WS"
238
    EXPECT_EQ(&s_crlf, State::start(lexer, common_options)); // recognize '\r'
239
    // see ' ', "unget" it
240
    s_crlf.handle(lexer);
241
242
    EXPECT_EQ(s_null, State::start(lexer, common_options)); // recognize ' '
    EXPECT_EQ(Token::INITIAL_WS, s_crlf.getToken(lexer).getType());
JINMEI Tatuya's avatar
JINMEI Tatuya committed
243

244
    // 3. comment between \r and \n
245
    EXPECT_EQ(&s_crlf, State::start(lexer, common_options)); // recognize '\r'
246
    // skip comments, recognize '\n'
247
    s_crlf.handle(lexer);
248
249
    EXPECT_EQ(Token::END_OF_LINE, s_crlf.getToken(lexer).getType());
    EXPECT_EQ(&s_string, State::start(lexer, common_options));
250
    s_string.handle(lexer); // skip 'a'
JINMEI Tatuya's avatar
JINMEI Tatuya committed
251

252
    // 4. \r then EOF
253
    EXPECT_EQ(&s_crlf, State::start(lexer, common_options)); // recognize '\r'
254
    // see EOF, then "unget" it
255
    s_crlf.handle(lexer);
256
257
    EXPECT_EQ(s_null, State::start(lexer, common_options));  // recognize EOF
    EXPECT_EQ(Token::END_OF_FILE, s_crlf.getToken(lexer).getType());
JINMEI Tatuya's avatar
JINMEI Tatuya committed
258
259
}

260
261
// Commonly used check for string related test cases, checking if the given
// token has expected values.
262
void
263
264
stringTokenCheck(const std::string& expected, const MasterLexer::Token& token,
                 bool quoted = false)
265
{
266
    EXPECT_EQ(quoted ? Token::QSTRING : Token::STRING, token.getType());
267
268
269
270
271
272
273
    EXPECT_EQ(expected, token.getString());
    const std::string actual(token.getStringRegion().beg,
                             token.getStringRegion().beg +
                             token.getStringRegion().len);
    EXPECT_EQ(expected, actual);
}

274
TEST_F(MasterLexerStateTest, string) {
275
276
277
278
279
280
281
282
    // Check with simple strings followed by separate characters
    ss << "followed-by-EOL\n";
    ss << "followed-by-CR\r";
    ss << "followed-by-space ";
    ss << "followed-by-tab\t";
    ss << "followed-by-comment;this is comment and ignored\n";
    ss << "followed-by-paren(closing)";
    ss << "followed-by-EOF";
283
284
285
    lexer.pushSource(ss);

    EXPECT_EQ(&s_string, State::start(lexer, common_options));
286
    s_string.handle(lexer); // recognize str, see \n
287
    EXPECT_FALSE(s_string.wasLastEOL(lexer));
288
289
290
291
    stringTokenCheck("followed-by-EOL", s_string.getToken(lexer));
    EXPECT_EQ(s_null, State::start(lexer, common_options)); // skip \n

    EXPECT_EQ(&s_string, State::start(lexer, common_options));
292
    s_string.handle(lexer); // recognize str, see \r
293
294
    stringTokenCheck("followed-by-CR", s_string.getToken(lexer));
    EXPECT_EQ(&s_crlf, State::start(lexer, common_options)); // handle \r...
295
    s_crlf.handle(lexer); // ...and skip it
296
297

    EXPECT_EQ(&s_string, State::start(lexer, common_options));
298
    s_string.handle(lexer); // recognize str, see ' '
299
300
301
302
    stringTokenCheck("followed-by-space", s_string.getToken(lexer));

    // skip ' ', then recognize the next string
    EXPECT_EQ(&s_string, State::start(lexer, common_options));
303
    s_string.handle(lexer); // recognize str, see \t
304
305
306
307
    stringTokenCheck("followed-by-tab", s_string.getToken(lexer));

    // skip \t, then recognize the next string
    EXPECT_EQ(&s_string, State::start(lexer, common_options));
308
    s_string.handle(lexer); // recognize str, see comment
309
310
311
312
    stringTokenCheck("followed-by-comment", s_string.getToken(lexer));
    EXPECT_EQ(s_null, State::start(lexer, common_options)); // skip \n after it

    EXPECT_EQ(&s_string, State::start(lexer, common_options));
313
    s_string.handle(lexer); // recognize str, see '('
314
315
    stringTokenCheck("followed-by-paren", s_string.getToken(lexer));
    EXPECT_EQ(&s_string, State::start(lexer, common_options)); // str in ()
316
    s_string.handle(lexer); // recognize the str, see ')'
317
318
319
    stringTokenCheck("closing", s_string.getToken(lexer));

    EXPECT_EQ(&s_string, State::start(lexer, common_options));
320
    s_string.handle(lexer); // recognize str, see EOF
321
    stringTokenCheck("followed-by-EOF", s_string.getToken(lexer));
322
323
}

324
325
326
327
328
329
330
331
332
333
334
335
TEST_F(MasterLexerStateTest, stringEscape) {
    // some of the separate characters should be considered part of the
    // string if escaped.
    ss << "escaped\\ space ";
    ss << "escaped\\\ttab ";
    ss << "escaped\\(paren ";
    ss << "escaped\\)close ";
    ss << "escaped\\;comment ";
    ss << "escaped\\\\ backslash "; // second '\' shouldn't escape ' '
    lexer.pushSource(ss);

    EXPECT_EQ(&s_string, State::start(lexer, common_options));
336
    s_string.handle(lexer); // recognize str, see ' ' at end
337
338
339
    stringTokenCheck("escaped\\ space", s_string.getToken(lexer));

    EXPECT_EQ(&s_string, State::start(lexer, common_options));
340
    s_string.handle(lexer); // recognize str, see ' ' at end
341
342
343
    stringTokenCheck("escaped\\\ttab", s_string.getToken(lexer));

    EXPECT_EQ(&s_string, State::start(lexer, common_options));
344
    s_string.handle(lexer); // recognize str, see ' ' at end
345
346
347
    stringTokenCheck("escaped\\(paren", s_string.getToken(lexer));

    EXPECT_EQ(&s_string, State::start(lexer, common_options));
348
    s_string.handle(lexer); // recognize str, see ' ' at end
349
350
351
    stringTokenCheck("escaped\\)close", s_string.getToken(lexer));

    EXPECT_EQ(&s_string, State::start(lexer, common_options));
352
    s_string.handle(lexer); // recognize str, see ' ' at end
353
354
355
    stringTokenCheck("escaped\\;comment", s_string.getToken(lexer));

    EXPECT_EQ(&s_string, State::start(lexer, common_options));
356
    s_string.handle(lexer); // recognize str, see ' ' in mid
357
    stringTokenCheck("escaped\\\\", s_string.getToken(lexer));
358
359
360

    // Confirm the word that follows the escaped '\' is correctly recognized.
    EXPECT_EQ(&s_string, State::start(lexer, common_options));
361
    s_string.handle(lexer); // recognize str, see ' ' at end
362
    stringTokenCheck("backslash", s_string.getToken(lexer));
363
364
}

365
366
TEST_F(MasterLexerStateTest, quotedString) {
    ss << "\"ignore-quotes\"\n";
367
368
369
370
371
372
    ss << "\"quoted string\" "; // space is part of the qstring
    // also check other separator characters. note that \r doesn't cause
    // UNBALANCED_QUOTES.  Not sure if it's intentional, but that's how the
    // BIND 9 version works, so we follow it (it should be too minor to matter
    // in practice anyway)
    ss << "\"quoted()\t\rstring\" ";
373
374
375
376
377
378
379
380
    ss << "\"escape\\ in quote\" ";
    ss << "\"escaped\\\"\" ";
    ss << "\"escaped backslash\\\\\" ";
    ss << "\"no;comment\"";
    lexer.pushSource(ss);

    // by default, '"' doesn't have any special meaning and part of string
    EXPECT_EQ(&s_string, State::start(lexer, common_options));
381
    s_string.handle(lexer); // recognize str, see \n
382
383
384
385
386
387
388
389
390
    stringTokenCheck("\"ignore-quotes\"", s_string.getToken(lexer));
    EXPECT_EQ(s_null, State::start(lexer, common_options)); // skip \n after it
    EXPECT_TRUE(s_string.wasLastEOL(lexer));

    // If QSTRING is specified in option, '"' is regarded as a beginning of
    // a quoted string.
    const MasterLexer::Options options = common_options | MasterLexer::QSTRING;
    EXPECT_EQ(&s_qstring, State::start(lexer, options));
    EXPECT_FALSE(s_string.wasLastEOL(lexer)); // EOL is canceled due to '"'
391
    s_qstring.handle(lexer);
392
393
    stringTokenCheck("quoted string", s_string.getToken(lexer), true);

394
395
    // Also checks other separator characters within a qstring
    EXPECT_EQ(&s_qstring, State::start(lexer, options));
396
    s_qstring.handle(lexer);
397
398
    stringTokenCheck("quoted()\t\rstring", s_string.getToken(lexer), true);

399
400
401
    // escape character mostly doesn't have any effect in the qstring
    // processing
    EXPECT_EQ(&s_qstring, State::start(lexer, options));
402
    s_qstring.handle(lexer);
403
404
405
406
407
    stringTokenCheck("escape\\ in quote", s_string.getToken(lexer), true);

    // The only exception is the quotation mark itself.  Note that the escape
    // only works on the quotation mark immediately after it.
    EXPECT_EQ(&s_qstring, State::start(lexer, options));
408
    s_qstring.handle(lexer);
409
410
411
412
413
    stringTokenCheck("escaped\"", s_string.getToken(lexer), true);

    // quoted '\' then '"'.  Unlike the previous case '"' shouldn't be
    // escaped.
    EXPECT_EQ(&s_qstring, State::start(lexer, options));
414
    s_qstring.handle(lexer);
415
416
417
418
    stringTokenCheck("escaped backslash\\\\", s_string.getToken(lexer), true);

    // ';' has no meaning in a quoted string (not indicating a comment)
    EXPECT_EQ(&s_qstring, State::start(lexer, options));
419
    s_qstring.handle(lexer);
420
421
422
423
424
425
426
427
428
429
430
431
    stringTokenCheck("no;comment", s_string.getToken(lexer), true);
}

TEST_F(MasterLexerStateTest, brokenQuotedString) {
    ss << "\"unbalanced-quote\n";
    ss << "\"quoted\\\n\" ";
    ss << "\"unclosed quote and EOF";
    lexer.pushSource(ss);

    // EOL is encountered without closing the quote
    const MasterLexer::Options options = common_options | MasterLexer::QSTRING;
    EXPECT_EQ(&s_qstring, State::start(lexer, options));
432
    s_qstring.handle(lexer);
433
434
435
436
437
438
439
440
441
    ASSERT_EQ(Token::ERROR, s_qstring.getToken(lexer).getType());
    EXPECT_EQ(Token::UNBALANCED_QUOTES,
              s_qstring.getToken(lexer).getErrorCode());
    // We can resume after the error from the '\n'
    EXPECT_EQ(s_null, State::start(lexer, options));
    EXPECT_EQ(Token::END_OF_LINE, s_crlf.getToken(lexer).getType());

    // \n is okay in a quoted string if escaped
    EXPECT_EQ(&s_qstring, State::start(lexer, options));
442
    s_qstring.handle(lexer);
443
444
445
446
    stringTokenCheck("quoted\\\n", s_string.getToken(lexer), true);

    // EOF is encountered without closing the quote
    EXPECT_EQ(&s_qstring, State::start(lexer, options));
447
    s_qstring.handle(lexer);
448
449
450
451
452
453
454
    ASSERT_EQ(Token::ERROR, s_qstring.getToken(lexer).getType());
    EXPECT_EQ(Token::UNEXPECTED_END, s_qstring.getToken(lexer).getErrorCode());
    // If we continue we'll simply see the EOF
    EXPECT_EQ(s_null, State::start(lexer, options));
    EXPECT_EQ(Token::END_OF_FILE, s_crlf.getToken(lexer).getType());
}

455
TEST_F(MasterLexerStateTest, basicNumbers) {
Jelte Jansen's avatar
Jelte Jansen committed
456
457
458
459
    ss << "0 ";
    ss << "1 ";
    ss << "12345 ";
    ss << "4294967295 "; // 2^32-1
460
461
462
    ss << "4294967296 "; // Out of range
    ss << "340282366920938463463374607431768211456 ";
                         // Very much out of range (2^128)
Jelte Jansen's avatar
Jelte Jansen committed
463
    ss << "005 ";        // Leading zeroes are ignored
464
465
466
467
    ss << "42;asdf\n";   // Number with comment
    ss << "37";          // Simple number again, here to make
                         // sure none of the above messed up
                         // the tokenizer
Jelte Jansen's avatar
Jelte Jansen committed
468
469
470
    lexer.pushSource(ss);

    EXPECT_EQ(&s_number, State::start(lexer, common_options));
471
    s_number.handle(lexer);
Jelte Jansen's avatar
Jelte Jansen committed
472
473
474
    EXPECT_EQ(0, s_number.getToken(lexer).getNumber());

    EXPECT_EQ(&s_number, State::start(lexer, common_options));
475
    s_number.handle(lexer);
Jelte Jansen's avatar
Jelte Jansen committed
476
477
478
    EXPECT_EQ(1, s_number.getToken(lexer).getNumber());

    EXPECT_EQ(&s_number, State::start(lexer, common_options));
479
    s_number.handle(lexer);
Jelte Jansen's avatar
Jelte Jansen committed
480
481
482
    EXPECT_EQ(12345, s_number.getToken(lexer).getNumber());

    EXPECT_EQ(&s_number, State::start(lexer, common_options));
483
    s_number.handle(lexer);
484
    EXPECT_EQ(4294967295u, s_number.getToken(lexer).getNumber());
Jelte Jansen's avatar
Jelte Jansen committed
485
486

    EXPECT_EQ(&s_number, State::start(lexer, common_options));
487
    s_number.handle(lexer);
488
489
    EXPECT_EQ(Token::NUMBER_RANGE,
              s_number.getToken(lexer).getErrorCode());
Jelte Jansen's avatar
Jelte Jansen committed
490
491

    EXPECT_EQ(&s_number, State::start(lexer, common_options));
492
    s_number.handle(lexer);
493
494
    EXPECT_EQ(Token::NUMBER_RANGE,
              s_number.getToken(lexer).getErrorCode());
Jelte Jansen's avatar
Jelte Jansen committed
495
496

    EXPECT_EQ(&s_number, State::start(lexer, common_options));
497
    s_number.handle(lexer);
Jelte Jansen's avatar
Jelte Jansen committed
498
499
    EXPECT_EQ(5, s_number.getToken(lexer).getNumber());

500
    EXPECT_EQ(&s_number, State::start(lexer, common_options));
501
    s_number.handle(lexer);
502
503
504
505
506
507
508
    EXPECT_EQ(42, s_number.getToken(lexer).getNumber());

    EXPECT_EQ(s_null, State::start(lexer, common_options));
    EXPECT_TRUE(s_crlf.wasLastEOL(lexer));
    EXPECT_EQ(Token::END_OF_LINE, s_crlf.getToken(lexer).getType());

    EXPECT_EQ(&s_number, State::start(lexer, common_options));
509
    s_number.handle(lexer);
510
511
512
513
514
    EXPECT_EQ(37, s_number.getToken(lexer).getNumber());

    // If we continue we'll simply see the EOF
    EXPECT_EQ(s_null, State::start(lexer, options));
    EXPECT_EQ(Token::END_OF_FILE, s_crlf.getToken(lexer).getType());
515
}
Jelte Jansen's avatar
Jelte Jansen committed
516

517
518
519
520
521
522
523
// Test tokens that look like (or start out as) numbers,
// but turn out to be strings. Tests include escaped characters.
TEST_F(MasterLexerStateTest, stringNumbers) {
    ss << "-1 ";         // Negative numbers are interpreted
                         // as strings (unsigned integers only)
    ss << "123abc456 ";  // 'Numbers' containing non-digits should
                         // be interpreted as strings
524
525
    ss << "123\\456 ";   // Numbers containing escaped digits are
                         // interpreted as strings
526
527
528
529
530
531
532
533
534
535
    ss << "3scaped\\ space ";
    ss << "3scaped\\\ttab ";
    ss << "3scaped\\(paren ";
    ss << "3scaped\\)close ";
    ss << "3scaped\\;comment ";
    ss << "3scaped\\\\ 8ackslash "; // second '\' shouldn't escape ' '

    lexer.pushSource(ss);

    EXPECT_EQ(&s_string, State::start(lexer, common_options));
536
    s_string.handle(lexer);
537
538
539
540
    stringTokenCheck("-1", s_string.getToken(lexer), false);

    // Starts out as a number, but ends up being a string
    EXPECT_EQ(&s_number, State::start(lexer, common_options));
541
    s_number.handle(lexer);
542
543
    stringTokenCheck("123abc456", s_number.getToken(lexer), false);

544
    EXPECT_EQ(&s_number, State::start(lexer, common_options));
545
    s_number.handle(lexer);
546
547
    stringTokenCheck("123\\456", s_number.getToken(lexer), false);

548
    EXPECT_EQ(&s_number, State::start(lexer, common_options));
549
    s_number.handle(lexer); // recognize str, see ' ' at end
550
551
552
    stringTokenCheck("3scaped\\ space", s_number.getToken(lexer));

    EXPECT_EQ(&s_number, State::start(lexer, common_options));
553
    s_number.handle(lexer); // recognize str, see ' ' at end
554
555
556
    stringTokenCheck("3scaped\\\ttab", s_number.getToken(lexer));

    EXPECT_EQ(&s_number, State::start(lexer, common_options));
557
    s_number.handle(lexer); // recognize str, see ' ' at end
558
559
560
    stringTokenCheck("3scaped\\(paren", s_number.getToken(lexer));

    EXPECT_EQ(&s_number, State::start(lexer, common_options));
561
    s_number.handle(lexer); // recognize str, see ' ' at end
562
563
564
    stringTokenCheck("3scaped\\)close", s_number.getToken(lexer));

    EXPECT_EQ(&s_number, State::start(lexer, common_options));
565
    s_number.handle(lexer); // recognize str, see ' ' at end
566
567
568
    stringTokenCheck("3scaped\\;comment", s_number.getToken(lexer));

    EXPECT_EQ(&s_number, State::start(lexer, common_options));
569
    s_number.handle(lexer); // recognize str, see ' ' in mid
570
571
572
573
    stringTokenCheck("3scaped\\\\", s_number.getToken(lexer));

    // Confirm the word that follows the escaped '\' is correctly recognized.
    EXPECT_EQ(&s_number, State::start(lexer, common_options));
574
    s_number.handle(lexer); // recognize str, see ' ' at end
575
576
577
578
579
580
581
    stringTokenCheck("8ackslash", s_number.getToken(lexer));

    // If we continue we'll simply see the EOF
    EXPECT_EQ(s_null, State::start(lexer, options));
    EXPECT_EQ(Token::END_OF_FILE, s_crlf.getToken(lexer).getType());
}

582
583
} // end anonymous namespace