master_lexer_unittest.cc 15.1 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
// Copyright (C) 2012  Internet Systems Consortium, Inc. ("ISC")
//
// Permission to use, copy, modify, and/or distribute this software for any
// purpose with or without fee is hereby granted, provided that the above
// copyright notice and this permission notice appear in all copies.
//
// THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH
// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
// AND FITNESS.  IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT,
// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
// OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
// PERFORMANCE OF THIS SOFTWARE.

15
16
#include <exceptions/exceptions.h>

17
#include <dns/master_lexer.h>
18
#include <dns/master_lexer_state.h>
19
20
21
22

#include <gtest/gtest.h>

#include <boost/lexical_cast.hpp>
23
#include <boost/function.hpp>
24
#include <boost/scoped_ptr.hpp>
25
#include <boost/bind.hpp>
26
27
28
29
30
31
32
33

#include <string>
#include <sstream>

using namespace isc::dns;
using std::string;
using std::stringstream;
using boost::lexical_cast;
34
35
using boost::scoped_ptr;
using master_lexer_internal::State;
36
37
38

namespace {

39
40
41
42
43
44
45
46
47
48
49
50
51
// This acts like the normal MasterLexer. It, however, allows to mock the start()
// method to return some given state instead of the auto-detected ones.
class TestedMasterLexer : public MasterLexer {
public:
    TestedMasterLexer() :
        fake_start_(NULL)
    {}
    // During the next call to start(), return the given state instead of the
    // auto-detected one.
    void pushFakeStart(const State* state) {
        fake_start_ = state;
    }
protected:
52
    virtual const State* start(Options options) {
53
54
55
56
57
58
59
60
61
        if (fake_start_ != NULL) {
            // There's a fake start, so remove it (not to be used next time)
            // and return it.
            const State* result = fake_start_;
            fake_start_ = NULL;
            return (result);
        } else {
            // No fake start ready. So we act the usual way, by delegating it to
            // the parent class.
62
            return (MasterLexer::start(options));
63
64
65
66
67
68
        }
    }
private:
    const State* fake_start_;
};

69
70
class MasterLexerTest : public ::testing::Test {
protected:
71
72
73
    MasterLexerTest() :
        expected_stream_name("stream-" + lexical_cast<string>(&ss))
    {}
74

75
    TestedMasterLexer lexer;
76
    stringstream ss;
77
    const string expected_stream_name;
78
79
};

80
81
82
// Commonly used check case where the input sources stack is empty.
void
checkEmptySource(const MasterLexer& lexer) {
83
84
85
86
    EXPECT_TRUE(lexer.getSourceName().empty());
    EXPECT_EQ(0, lexer.getSourceLine());
}

87
88
89
90
91
TEST_F(MasterLexerTest, preOpen) {
    // Initially sources stack is empty.
    checkEmptySource(lexer);
}

92
TEST_F(MasterLexerTest, pushStream) {
93
    lexer.pushSource(ss);
94
    EXPECT_EQ(expected_stream_name, lexer.getSourceName());
95
96
97
98
99

    // From the point of view of this test, we only have to check (though
    // indirectly) getSourceLine calls InputSource::getCurrentLine.  It should
    // return 1 initially.
    EXPECT_EQ(1, lexer.getSourceLine());
100

101
    // By popping it the stack will be empty again.
102
    lexer.popSource();
103
    checkEmptySource(lexer);
104
105
}

106
TEST_F(MasterLexerTest, pushFile) {
107
108
    // We use zone file (-like) data, but in this test that actually doesn't
    // matter.
109
    EXPECT_TRUE(lexer.pushSource(TEST_DATA_SRCDIR "/masterload.txt"));
110
111
    EXPECT_EQ(TEST_DATA_SRCDIR "/masterload.txt", lexer.getSourceName());
    EXPECT_EQ(1, lexer.getSourceLine());
112

113
    lexer.popSource();
114
    checkEmptySource(lexer);
115
116
117
118
119
120
121

    // If we give a non NULL string pointer, its content will be intact
    // if pushSource succeeds.
    std::string error_txt = "dummy";
    EXPECT_TRUE(lexer.pushSource(TEST_DATA_SRCDIR "/masterload.txt",
                                 &error_txt));
    EXPECT_EQ("dummy", error_txt);
122
123
}

124
TEST_F(MasterLexerTest, pushBadFileName) {
125
    EXPECT_THROW(lexer.pushSource(NULL), isc::InvalidParameter);
JINMEI Tatuya's avatar
JINMEI Tatuya committed
126
127
}

128
129
130
131
132
133
134
135
136
137
138
139
140
141
TEST_F(MasterLexerTest, pushFileFail) {
    // The file to be pushed doesn't exist.  pushSource() fails and
    // some non empty error string should be set.
    std::string error_txt;
    EXPECT_TRUE(error_txt.empty());
    EXPECT_FALSE(lexer.pushSource("no-such-file", &error_txt));
    EXPECT_FALSE(error_txt.empty());

    // It's safe to pass NULL error_txt (either explicitly or implicitly as
    // the default)
    EXPECT_FALSE(lexer.pushSource("no-such-file", NULL));
    EXPECT_FALSE(lexer.pushSource("no-such-file"));
}

142
TEST_F(MasterLexerTest, nestedPush) {
143
    lexer.pushSource(ss);
144
145
    EXPECT_EQ(expected_stream_name, lexer.getSourceName());

146
    // We can push another source without popping the previous one.
147
    lexer.pushSource(TEST_DATA_SRCDIR "/masterload.txt");
148
149
    EXPECT_EQ(TEST_DATA_SRCDIR "/masterload.txt", lexer.getSourceName());

150
    // popSource() works on the "topmost" (last-pushed) source
151
    lexer.popSource();
152
153
    EXPECT_EQ(expected_stream_name, lexer.getSourceName());

154
    lexer.popSource();
155
156
157
    EXPECT_TRUE(lexer.getSourceName().empty());
}

158
TEST_F(MasterLexerTest, invalidPop) {
159
160
    // popSource() cannot be called if the sources stack is empty.
    EXPECT_THROW(lexer.popSource(), isc::InvalidOperation);
161
162
}

163
164
165
166
167
168
169
170
171
172
173
174
175
176
// Test it is not possible to get token when no source is available.
TEST_F(MasterLexerTest, noSource) {
    EXPECT_THROW(lexer.getNextToken(), isc::InvalidOperation);
}

// Getting a token directly from the start() method.
TEST_F(MasterLexerTest, tokenFromStart) {
    // A class that sets the token directly in start() and returns no
    // state. This is equivalent to the State::start() doing so.
    class StartLexer : public MasterLexer {
    public:
        StartLexer() :
            token_(MasterLexer::Token::END_OF_LINE)
        {}
177
        virtual const State* start(Options) {
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
            // We don't have access directly inside the implementation.
            // We get the fake state, run it to install the token.
            // Then we just delete it ourself and return NULL.
            State* state(State::getFakeState(NULL, 0, &token_));
            state->handle(*this);
            delete state;
            return (NULL);
        }
    private:
        MasterLexer::Token token_;
    } lexer;
    lexer.pushSource(ss);

    // The token gets out.
    MasterLexer::Token generated(lexer.getNextToken());
    EXPECT_EQ(MasterLexer::Token::END_OF_LINE, generated.getType());
}

// Getting a token with a single iteration through the states.
TEST_F(MasterLexerTest, simpleGetToken) {
    // Prepare the fake state.
    MasterLexer::Token token(MasterLexer::Token::END_OF_LINE);
    scoped_ptr<State> state(State::getFakeState(NULL, 3, &token));
    lexer.pushFakeStart(state.get());
    // Push some source inside.
    ss << "12345";
    lexer.pushSource(ss);

    // Get the token.
    MasterLexer::Token generated(lexer.getNextToken());
    // It is the same token (well, on a different address)
    // We can't compare directly, so compare types.
    EXPECT_EQ(token.getType(), generated.getType());
    // 3 characters were read from the source.
    // We test by extracting the rest and comparing.
    int rest;
    ss >> rest;
215
    EXPECT_EQ(45, rest);
216
217
218
219
220
221
222
223
224
225
226
227
}

// A token that takes multiple states.
//
// The first state sets the token as well as the second. The second one should
// survive and be returned.
TEST_F(MasterLexerTest, chainGetToken) {
    // Build the states
    MasterLexer::Token t1(MasterLexer::Token::END_OF_LINE);
    MasterLexer::Token t2(MasterLexer::Token::INITIAL_WS);
    scoped_ptr<State> s2(State::getFakeState(NULL, 1, &t2));
    scoped_ptr<State> s1(State::getFakeState(s2.get(), 2, &t1));
228
    lexer.pushFakeStart(s1.get());
229
230
231
232
233
234
235
236
237
238
239
240
241
    // Put something into the source
    ss << "12345";
    lexer.pushSource(ss);

    // Get the token.
    MasterLexer::Token generated(lexer.getNextToken());
    // It is the same token as the second one (well, on a different address)
    // We can't compare directly, so compare types.
    EXPECT_EQ(t2.getType(), generated.getType());
    // 3 characters were read from the source.
    // We test by extracting the rest and comparing.
    int rest;
    ss >> rest;
242
    EXPECT_EQ(45, rest);
243
244
245
246
247
248
249
250
}

// Test getting a token without overriding the start() method (well, it
// is overriden, but no fake state is set, so it refers to the real one).
//
// This also tests the real start() passes the options, otherwise we wouldn't
// get the initial whitespace.
TEST_F(MasterLexerTest, realStart) {
251
    ss << "\n   \n";
252
253
    lexer.pushSource(ss);

254
255
256
257
258
    // First, the newline should get out.
    EXPECT_EQ(MasterLexer::Token::END_OF_LINE, lexer.getNextToken().getType());
    // Then the whitespace, if we specify the option.
    EXPECT_EQ(MasterLexer::Token::INITIAL_WS,
              lexer.getNextToken(MasterLexer::INITIAL_WS).getType());
259
260
}

261
262
263
264
265
266
267
268
269
270
271
272
// Test we correctly find end of file. Then, upon more attempts to produce
// tokens past the end, it throws.
TEST_F(MasterLexerTest, eof) {
    // Let the ss empty.
    lexer.pushSource(ss);

    // The first one is found to be EOF
    EXPECT_EQ(MasterLexer::Token::END_OF_FILE, lexer.getNextToken().getType());
    // And it is not allowed to use this one any more.
    EXPECT_THROW(lexer.getNextToken(), isc::InvalidOperation);
}

273
274
275
276
277
278
279
280
281
282
283
284
285
// Check we properly return error when there's an opened parentheses and no
// closing one
TEST_F(MasterLexerTest, getUnbalanced) {
    ss << "(\"string\"";
    lexer.pushSource(ss);

    // The string gets out first
    EXPECT_EQ(MasterLexer::Token::STRING, lexer.getNextToken().getType());
    // Then an unbalanced parethsis
    EXPECT_EQ(MasterLexer::Token::UNBALANCED_PAREN,
              lexer.getNextToken().getErrorCode());
}

286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
void
checkInput(const std::string& expected, const std::string& received) {
    EXPECT_EQ(expected, received);
}

// Check ungetting a token, which should get to the previous state. We do
// so with changing the state a little bit.
TEST_F(MasterLexerTest, ungetSimple) {
    ss << "12345";
    lexer.pushSource(ss);

    const bool true_value = true, false_value = false;
    // Make sure we change the state to non-default, so we return to previous
    // not default state.
    MasterLexer::Token t0(MasterLexer::Token::INITIAL_WS);
    scoped_ptr<State> s0(State::getFakeState(NULL, 1, &t0, 1, &true_value));
    lexer.pushFakeStart(s0.get());
    EXPECT_EQ(MasterLexer::Token::INITIAL_WS, lexer.getNextToken().getType());

    // Prepare the token to get and return
    const std::string expected = "234";
    MasterLexer::Token token(MasterLexer::Token::END_OF_LINE);
    // Change the internal state with it too. So we can check it is retured.
    scoped_ptr<State> state(State::getFakeState(NULL, 3, &token, 1,
                                                &false_value,
                                                boost::bind(&checkInput,
                                                            expected, _1)));
    lexer.pushFakeStart(state.get());

    // Check the internal state before getting the token
    // We access the lexer through any state, so use the one we have.
    EXPECT_EQ(1, state->getParenCount(lexer));
    EXPECT_TRUE(state->wasLastEOL(lexer));

    // Now get the token and check the state changed
    EXPECT_EQ(MasterLexer::Token::END_OF_LINE, lexer.getNextToken().getType());
    EXPECT_EQ(2, state->getParenCount(lexer));
    EXPECT_FALSE(state->wasLastEOL(lexer));

    // Return the token back. Check the state is as it was before.
    lexer.ungetToken();
    EXPECT_EQ(1, state->getParenCount(lexer));
    EXPECT_TRUE(state->wasLastEOL(lexer));
    // By calling getToken again, we verify even the source got back to
    // original. We must push it as a fake start again so it is picked.
    lexer.pushFakeStart(state.get());
    EXPECT_EQ(MasterLexer::Token::END_OF_LINE, lexer.getNextToken().getType());
    EXPECT_EQ(2, state->getParenCount(lexer));
    EXPECT_FALSE(state->wasLastEOL(lexer));
}

// Check ungetting token without overriding the start method. We also
// check it works well with changing options between the calls.
TEST_F(MasterLexerTest, ungetRealOptions) {
340
    ss << "\n    \n";
341
    lexer.pushSource(ss);
342
343
    // Skip the first newline
    EXPECT_EQ(MasterLexer::Token::END_OF_LINE, lexer.getNextToken().getType());
344
345
346
347
348
349
350

    // If we call it the usual way, it skips up to the newline and returns
    // it
    EXPECT_EQ(MasterLexer::Token::END_OF_LINE, lexer.getNextToken().getType());

    // Now we return it. If we call it again, but with different options,
    // we get the initial whitespace.
351
    lexer.ungetToken();
352
353
354
355
    EXPECT_EQ(MasterLexer::Token::INITIAL_WS,
              lexer.getNextToken(MasterLexer::INITIAL_WS).getType());
}

356
357
358
359
360
361
362
363
364
365
366
367
// Test only one token can be ungotten
TEST_F(MasterLexerTest, ungetTwice) {
    ss << "\n";
    lexer.pushSource(ss);

    EXPECT_EQ(MasterLexer::Token::END_OF_LINE, lexer.getNextToken().getType());
    // Unget the token. It can be done once
    lexer.ungetToken();
    // But not twice
    EXPECT_THROW(lexer.ungetToken(), isc::InvalidOperation);
}

368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
// Test we can't unget a token before we get one
TEST_F(MasterLexerTest, ungetBeforeGet) {
    lexer.pushSource(ss); // Just to eliminate the missing source problem
    EXPECT_THROW(lexer.ungetToken(), isc::InvalidOperation);
}

// Test we can't unget a token after a source switch, even when we got
// something before.
TEST_F(MasterLexerTest, ungetAfterSwitch) {
    ss << "\n\n";
    lexer.pushSource(ss);
    EXPECT_EQ(MasterLexer::Token::END_OF_LINE, lexer.getNextToken().getType());
    // Switch the source
    std::stringstream ss2;
    ss2 << "\n\n";
    lexer.pushSource(ss2);
    EXPECT_THROW(lexer.ungetToken(), isc::InvalidOperation);
    // We can get from the new source
    EXPECT_EQ(MasterLexer::Token::END_OF_LINE, lexer.getNextToken().getType());
    // And when we drop the current source, we can't unget again
    lexer.popSource();
    EXPECT_THROW(lexer.ungetToken(), isc::InvalidOperation);
}

392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
class TestException {};

void
doThrow(const std::string&) {
    throw TestException();
}

// Check the getNextToken provides at least the weak exception guarantee.
TEST_F(MasterLexerTest, getTokenExceptions) {
    ss << "\n12345";
    lexer.pushSource(ss);

    // Prepare a chain that changes the internal state, reads something.
    // The next item in the chain will throw an exception (we explicitly
    // throw something not known to it, so we know it can handle anything).
    // Then the thing should get to the previous state and getting the
    // token the usual way without mock should work.
    const bool true_value = true;
    boost::scoped_ptr<State> s2(State::getFakeState(NULL, 3, NULL, 0, NULL,
                                                    &doThrow));
    boost::scoped_ptr<State> s1(State::getFakeState(s2.get(), 3, NULL, 1,
                                                    &true_value));
    lexer.pushFakeStart(s1.get());

    // Getting the token with the fake start should throw. But then, the
    // current state should be untouched.
    EXPECT_THROW(lexer.getNextToken(), TestException);
    EXPECT_EQ(0, s1->getParenCount(lexer));
    EXPECT_FALSE(s1->wasLastEOL(lexer));

    // It gets back to the original state, so getting the newline works.
    EXPECT_EQ(MasterLexer::Token::END_OF_LINE, lexer.getNextToken().getType());
}

426
}