parser_unittest.cc 20.3 KB
Newer Older
1
// Copyright (C) 2016-2017 Internet Systems Consortium, Inc. ("ISC")
2 3 4 5 6 7 8 9
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

#include <gtest/gtest.h>
#include <cc/data.h>
#include <dhcp4/parser_context.h>
10
#include <testutils/io_utils.h>
11 12 13 14

using namespace isc::data;
using namespace std;

15 16 17
namespace isc {
namespace dhcp {
namespace test {
18

19 20 21 22 23 24
/// @brief compares two JSON trees
///
/// If differences are discovered, gtest failure is reported (using EXPECT_EQ)
///
/// @param a first to be compared
/// @param b second to be compared
25
void compareJSON(ConstElementPtr a, ConstElementPtr b) {
26 27 28 29 30
    ASSERT_TRUE(a);
    ASSERT_TRUE(b);
    EXPECT_EQ(a->str(), b->str());
}

31 32 33 34 35 36 37 38 39 40 41 42 43
/// @brief Tests if the input string can be parsed with specific parser
///
/// The input text will be passed to bison parser of specified type.
/// Then the same input text is passed to legacy JSON parser and outputs
/// from both parsers are compared. The legacy comparison can be disabled,
/// if the feature tested is not supported by the old parser (e.g.
/// new comment styles)
///
/// @param txt text to be compared
/// @param parser_type bison parser type to be instantiated
/// @param compare whether to compare the output with legacy JSON parser
void testParser(const std::string& txt, Parser4Context::ParserType parser_type,
                bool compare = true) {
44 45 46 47
    ConstElementPtr test_json;

    ASSERT_NO_THROW({
            try {
48 49
                Parser4Context ctx;
                test_json = ctx.parseString(txt, parser_type);
50 51 52 53 54 55 56
            } catch (const std::exception &e) {
                cout << "EXCEPTION: " << e.what() << endl;
                throw;
            }

    });

57 58 59 60
    if (!compare) {
        return;
    };

61
    // Now compare if both representations are the same.
62 63
    ElementPtr reference_json;
    ASSERT_NO_THROW(reference_json = Element::fromJSON(txt, true));
64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123
    compareJSON(reference_json, test_json);
}

TEST(ParserTest, mapInMap) {
    string txt = "{ \"xyzzy\": { \"foo\": 123, \"baz\": 456 } }";
    testParser(txt, Parser4Context::PARSER_JSON);
}

TEST(ParserTest, listInList) {
    string txt = "[ [ \"Britain\", \"Wales\", \"Scotland\" ], "
                 "[ \"Pomorze\", \"Wielkopolska\", \"Tatry\"] ]";
    testParser(txt, Parser4Context::PARSER_JSON);
}

TEST(ParserTest, nestedMaps) {
    string txt = "{ \"europe\": { \"UK\": { \"London\": { \"street\": \"221B Baker\" }}}}";
    testParser(txt, Parser4Context::PARSER_JSON);
}

TEST(ParserTest, nestedLists) {
    string txt = "[ \"half\", [ \"quarter\", [ \"eighth\", [ \"sixteenth\" ]]]]";
    testParser(txt, Parser4Context::PARSER_JSON);
}

TEST(ParserTest, listsInMaps) {
    string txt = "{ \"constellations\": { \"orion\": [ \"rigel\", \"betelguese\" ], "
                    "\"cygnus\": [ \"deneb\", \"albireo\"] } }";
    testParser(txt, Parser4Context::PARSER_JSON);
}

TEST(ParserTest, mapsInLists) {
    string txt = "[ { \"body\": \"earth\", \"gravity\": 1.0 },"
                 " { \"body\": \"mars\", \"gravity\": 0.376 } ]";
    testParser(txt, Parser4Context::PARSER_JSON);
}

TEST(ParserTest, types) {
    string txt = "{ \"string\": \"foo\","
                   "\"integer\": 42,"
                   "\"boolean\": true,"
                   "\"map\": { \"foo\": \"bar\" },"
                   "\"list\": [ 1, 2, 3 ],"
                   "\"null\": null }";
    testParser(txt, Parser4Context::PARSER_JSON);
}

TEST(ParserTest, keywordJSON) {
    string txt = "{ \"name\": \"user\","
                   "\"type\": \"password\","
                   "\"user\": \"name\","
                   "\"password\": \"type\" }";
    testParser(txt, Parser4Context::PARSER_JSON);
}

TEST(ParserTest, keywordDhcp4) {
     string txt = "{ \"Dhcp4\": { \"interfaces-config\": {"
                  " \"interfaces\": [ \"type\", \"htype\" ] },\n"
                  "\"rebind-timer\": 2000, \n"
                  "\"renew-timer\": 1000, \n"
                  "\"subnet4\": [ { "
Francis Dupont's avatar
Francis Dupont committed
124 125 126
                  "  \"pools\": [ { \"pool\": \"192.0.2.1 - 192.0.2.100\" } ],"
                  "  \"subnet\": \"192.0.2.0/24\", "
                  "  \"interface\": \"test\" } ],\n"
127
                   "\"valid-lifetime\": 4000 } }";
128
     testParser(txt, Parser4Context::PARSER_DHCP4);
129 130
}

131 132
// Tests if bash (#) comments are supported. That's the only comment type that
// was supported by the old parser.
133
TEST(ParserTest, bashComments) {
Francis Dupont's avatar
Francis Dupont committed
134
    string txt= "{ \"Dhcp4\": { \"interfaces-config\": {"
135 136 137 138 139 140 141
                "  \"interfaces\": [ \"*\" ]"
                "},\n"
                "# this is a comment\n"
                "\"rebind-timer\": 2000, \n"
                "# lots of comments here\n"
                "# and here\n"
                "\"renew-timer\": 1000, \n"
Francis Dupont's avatar
Francis Dupont committed
142 143 144
                "\"subnet4\": [ { "
                "    \"pools\": [ { \"pool\": \"192.0.2.1 - 192.0.2.100\" } ],"
                "    \"subnet\": \"192.0.2.0/24\", "
145 146 147
                "    \"interface\": \"eth0\""
                " } ],"
                "\"valid-lifetime\": 4000 } }";
148
    testParser(txt, Parser4Context::PARSER_DHCP4, false);
149 150
}

151 152
// Tests if C++ (//) comments can start anywhere, not just in the first line.
TEST(ParserTest, cppComments) {
Francis Dupont's avatar
Francis Dupont committed
153
    string txt= "{ \"Dhcp4\": { \"interfaces-config\": {"
154 155 156 157
                "  \"interfaces\": [ \"*\" ]"
                "},\n"
                "\"rebind-timer\": 2000, // everything after // is ignored\n"
                "\"renew-timer\": 1000, // this will be ignored, too\n"
Francis Dupont's avatar
Francis Dupont committed
158 159 160
                "\"subnet4\": [ { "
                "    \"pools\": [ { \"pool\": \"192.0.2.1 - 192.0.2.100\" } ],"
                "    \"subnet\": \"192.0.2.0/24\", "
161 162 163
                "    \"interface\": \"eth0\""
                " } ],"
                "\"valid-lifetime\": 4000 } }";
164
    testParser(txt, Parser4Context::PARSER_DHCP4, false);
165 166
}

167
// Tests if bash (#) comments can start anywhere, not just in the first line.
168
TEST(ParserTest, bashCommentsInline) {
Francis Dupont's avatar
Francis Dupont committed
169
    string txt= "{ \"Dhcp4\": { \"interfaces-config\": {"
170 171 172 173
                "  \"interfaces\": [ \"*\" ]"
                "},\n"
                "\"rebind-timer\": 2000, # everything after # is ignored\n"
                "\"renew-timer\": 1000, # this will be ignored, too\n"
Francis Dupont's avatar
Francis Dupont committed
174 175 176
                "\"subnet4\": [ { "
                "    \"pools\": [ { \"pool\": \"192.0.2.1 - 192.0.2.100\" } ],"
                "    \"subnet\": \"192.0.2.0/24\", "
177 178 179
                "    \"interface\": \"eth0\""
                " } ],"
                "\"valid-lifetime\": 4000 } }";
180
    testParser(txt, Parser4Context::PARSER_DHCP4, false);
181 182
}

183
// Tests if multi-line C style comments are handled correctly.
184
TEST(ParserTest, multilineComments) {
Francis Dupont's avatar
Francis Dupont committed
185
    string txt= "{ \"Dhcp4\": { \"interfaces-config\": {"
186 187
                "  \"interfaces\": [ \"*\" ]"
                "},\n"
Francis Dupont's avatar
Francis Dupont committed
188
                "   /* this is a C style comment\n"
189 190 191
                "that\n can \n span \n multiple \n lines */ \n"
                "\"rebind-timer\": 2000,\n"
                "\"renew-timer\": 1000, \n"
Francis Dupont's avatar
Francis Dupont committed
192 193 194
                "\"subnet4\": [ { "
                "    \"pools\": [ { \"pool\": \"192.0.2.1 - 192.0.2.100\" } ],"
                "    \"subnet\": \"192.0.2.0/24\", "
195 196 197
                "    \"interface\": \"eth0\""
                " } ],"
                "\"valid-lifetime\": 4000 } }";
198
    testParser(txt, Parser4Context::PARSER_DHCP4, false);
199 200
}

201

202 203 204 205 206
/// @brief Loads specified example config file
///
/// This test loads specified example file twice: first, using the legacy
/// JSON file and then second time using bison parser. Two created Element
/// trees are then compared. The input is decommented before it is passed
207
/// to legacy parser (as legacy support for comments is very limited).
208 209
///
/// @param fname name of the file to be loaded
210
void testFile(const std::string& fname) {
211 212 213
    ElementPtr reference_json;
    ConstElementPtr test_json;

214
    string decommented = decommentJSONfile(fname);
215

216
    cout << "Parsing file " << fname << " (" << decommented << ")" << endl;
217 218 219 220 221

    EXPECT_NO_THROW(reference_json = Element::fromJSONFile(decommented, true));

    // remove the temporary file
    EXPECT_NO_THROW(::remove(decommented.c_str()));
222 223 224 225 226 227 228 229 230 231 232 233 234

    EXPECT_NO_THROW(
    try {
        Parser4Context ctx;
        test_json = ctx.parseFile(fname, Parser4Context::PARSER_DHCP4);
    } catch (const std::exception &x) {
        cout << "EXCEPTION: " << x.what() << endl;
        throw;
    });

    ASSERT_TRUE(reference_json);
    ASSERT_TRUE(test_json);

235
    compareJSON(reference_json, test_json);
236 237 238 239
}

// This test loads all available existing files. Each config is loaded
// twice: first with the existing Element::fromJSONFile() and then
Francis Dupont's avatar
Francis Dupont committed
240
// the second time with Parser4. Both JSON trees are then compared.
241
TEST(ParserTest, file) {
242 243 244
    vector<string> configs = { "advanced.json" ,
                               "backends.json",
                               "classify.json",
245
                               "cassandra.json",
246 247 248 249 250 251 252 253
                               "dhcpv4-over-dhcpv6.json",
                               "hooks.json",
                               "leases-expiration.json",
                               "multiple-options.json",
                               "mysql-reservations.json",
                               "pgsql-reservations.json",
                               "reservations.json",
                               "several-subnets.json",
254 255
                               "single-subnet.json",
                               "with-ddns.json" };
256 257

    for (int i = 0; i<configs.size(); i++) {
258
        testFile(string(CFG_EXAMPLES) + "/" + configs[i]);
259 260 261
    }
}

262 263 264 265 266
/// @brief Tests error conditions in Dhcp4Parser
///
/// @param txt text to be parsed
/// @param parser_type type of the parser to be used in the test
/// @param msg expected content of the exception
267 268 269 270 271 272 273
void testError(const std::string& txt,
               Parser4Context::ParserType parser_type,
               const std::string& msg)
{
    try {
        Parser4Context ctx;
        ConstElementPtr parsed = ctx.parseString(txt, parser_type);
Francis Dupont's avatar
Francis Dupont committed
274
        FAIL() << "Expected Dhcp4ParseError but nothing was raised (expected: "
275 276 277 278 279 280
               << msg << ")";
    }
    catch (const Dhcp4ParseError& ex) {
        EXPECT_EQ(msg, ex.what());
    }
    catch (...) {
Francis Dupont's avatar
Francis Dupont committed
281
        FAIL() << "Expected Dhcp4ParseError but something else was raised";
282 283 284
    }
}

285
// Verify that error conditions are handled correctly.
286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349
TEST(ParserTest, errors) {
    // no input
    testError("", Parser4Context::PARSER_JSON,
              "<string>:1.1: syntax error, unexpected end of file");
    testError(" ", Parser4Context::PARSER_JSON,
              "<string>:1.2: syntax error, unexpected end of file");
    testError("\n", Parser4Context::PARSER_JSON,
              "<string>:2.1: syntax error, unexpected end of file");
    testError("\t", Parser4Context::PARSER_JSON,
              "<string>:1.2: syntax error, unexpected end of file");
    testError("\r", Parser4Context::PARSER_JSON,
              "<string>:1.2: syntax error, unexpected end of file");

    // comments
    testError("# nothing\n",
              Parser4Context::PARSER_JSON,
              "<string>:2.1: syntax error, unexpected end of file");
    testError(" #\n",
              Parser4Context::PARSER_JSON,
              "<string>:2.1: syntax error, unexpected end of file");
    testError("// nothing\n",
              Parser4Context::PARSER_JSON,
              "<string>:2.1: syntax error, unexpected end of file");
    testError("/* nothing */\n",
              Parser4Context::PARSER_JSON,
              "<string>:2.1: syntax error, unexpected end of file");
    testError("/* no\nthing */\n",
              Parser4Context::PARSER_JSON,
              "<string>:3.1: syntax error, unexpected end of file");
    testError("/* no\nthing */\n\n",
              Parser4Context::PARSER_JSON,
              "<string>:4.1: syntax error, unexpected end of file");
    testError("/* nothing\n",
              Parser4Context::PARSER_JSON,
              "Comment not closed. (/* in line 1");
    testError("\n\n\n/* nothing\n",
              Parser4Context::PARSER_JSON,
              "Comment not closed. (/* in line 4");
    testError("{ /* */*/ }\n",
              Parser4Context::PARSER_JSON,
              "<string>:1.3-8: Invalid character: *");
    testError("{ /* // *// }\n",
              Parser4Context::PARSER_JSON,
              "<string>:1.3-11: Invalid character: /");
    testError("{ /* // *///  }\n",
              Parser4Context::PARSER_JSON,
              "<string>:2.1: syntax error, unexpected end of file, "
              "expecting }");

    // includes
    testError("<?\n",
              Parser4Context::PARSER_JSON,
              "Directive not closed.");
    testError("<?include\n",
              Parser4Context::PARSER_JSON,
              "Directive not closed.");
    string file = string(CFG_EXAMPLES) + "/" + "single-subnet.json";
    testError("<?include \"" + file + "\"\n",
              Parser4Context::PARSER_JSON,
              "Directive not closed.");
    testError("<?include \"/foo/bar\" ?>/n",
              Parser4Context::PARSER_JSON,
              "Can't open include file /foo/bar");

350
    // JSON keywords
351
    testError("{ \"foo\": True }",
352
              Parser4Context::PARSER_JSON,
353
              "<string>:1.10-13: JSON true reserved keyword is lower case only");
354 355
    testError("{ \"foo\": False }",
              Parser4Context::PARSER_JSON,
356
              "<string>:1.10-14: JSON false reserved keyword is lower case only");
357 358
    testError("{ \"foo\": NULL }",
              Parser4Context::PARSER_JSON,
359
              "<string>:1.10-13: JSON null reserved keyword is lower case only");
360
    testError("{ \"foo\": Tru }",
361 362
              Parser4Context::PARSER_JSON,
              "<string>:1.10: Invalid character: T");
363
    testError("{ \"foo\": nul }",
364
              Parser4Context::PARSER_JSON,
365
              "<string>:1.10: Invalid character: n");
366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414

    // numbers
    testError("123",
              Parser4Context::PARSER_DHCP4,
              "<string>:1.1-3: syntax error, unexpected integer, "
              "expecting {");
    testError("-456",
              Parser4Context::PARSER_DHCP4,
              "<string>:1.1-4: syntax error, unexpected integer, "
              "expecting {");
    testError("-0001",
              Parser4Context::PARSER_DHCP4,
              "<string>:1.1-5: syntax error, unexpected integer, "
              "expecting {");
    testError("1234567890123456789012345678901234567890",
              Parser4Context::PARSER_JSON,
              "<string>:1.1-40: Failed to convert "
              "1234567890123456789012345678901234567890"
              " to an integer.");
    testError("-3.14e+0",
              Parser4Context::PARSER_DHCP4,
              "<string>:1.1-8: syntax error, unexpected floating point, "
              "expecting {");
    testError("1e50000",
              Parser4Context::PARSER_JSON,
              "<string>:1.1-7: Failed to convert 1e50000 "
              "to a floating point.");

    // strings
    testError("\"aabb\"",
              Parser4Context::PARSER_DHCP4,
              "<string>:1.1-6: syntax error, unexpected constant string, "
              "expecting {");
    testError("{ \"aabb\"err",
              Parser4Context::PARSER_JSON,
              "<string>:1.9: Invalid character: e");
    testError("{ err\"aabb\"",
              Parser4Context::PARSER_JSON,
              "<string>:1.3: Invalid character: e");
    testError("\"a\n\tb\"",
              Parser4Context::PARSER_JSON,
              "<string>:1.1-6: Invalid control in \"a\n\tb\"");
    testError("\"a\\n\\tb\"",
              Parser4Context::PARSER_DHCP4,
              "<string>:1.1-8: syntax error, unexpected constant string, "
              "expecting {");
    testError("\"a\\x01b\"",
              Parser4Context::PARSER_JSON,
              "<string>:1.1-8: Bad escape in \"a\\x01b\"");
415
    testError("\"a\\u0162\"",
416
              Parser4Context::PARSER_JSON,
417
              "<string>:1.1-9: Unsupported unicode escape in \"a\\u0162\"");
418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504
    testError("\"a\\u062z\"",
              Parser4Context::PARSER_JSON,
              "<string>:1.1-9: Bad escape in \"a\\u062z\"");
    testError("\"abc\\\"",
              Parser4Context::PARSER_JSON,
              "<string>:1.1-6: Overflow escape in \"abc\\\"");

    // from data_unittest.c
    testError("\\a",
              Parser4Context::PARSER_JSON,
              "<string>:1.1: Invalid character: \\");
    testError("\\",
              Parser4Context::PARSER_JSON,
              "<string>:1.1: Invalid character: \\");
    testError("\\\"\\\"",
              Parser4Context::PARSER_JSON,
              "<string>:1.1: Invalid character: \\");

    // want a map
    testError("[]\n",
              Parser4Context::PARSER_DHCP4,
              "<string>:1.1: syntax error, unexpected [, "
              "expecting {");
    testError("[]\n",
              Parser4Context::PARSER_DHCP4,
              "<string>:1.1: syntax error, unexpected [, "
              "expecting {");
    testError("{ 123 }\n",
              Parser4Context::PARSER_JSON,
              "<string>:1.3-5: syntax error, unexpected integer, "
              "expecting }");
    testError("{ 123 }\n",
              Parser4Context::PARSER_DHCP4,
              "<string>:1.3-5: syntax error, unexpected integer");
    testError("{ \"foo\" }\n",
              Parser4Context::PARSER_JSON,
              "<string>:1.9: syntax error, unexpected }, "
              "expecting :");
    testError("{ \"foo\" }\n",
              Parser4Context::PARSER_DHCP4,
              "<string>:1.9: syntax error, unexpected }, expecting :");
    testError("{ \"foo\":null }\n",
              Parser4Context::PARSER_DHCP4,
              "<string>:1.3-7: got unexpected keyword "
              "\"foo\" in toplevel map.");
    testError("{ \"Dhcp4\" }\n",
              Parser4Context::PARSER_DHCP4,
              "<string>:1.11: syntax error, unexpected }, "
              "expecting :");
    testError("{ \"Dhcp6\":[]\n",
              Parser4Context::PARSER_DHCP4,
              "<string>:2.1: syntax error, unexpected end of file, "
              "expecting \",\" or }");
    testError("{}{}\n",
              Parser4Context::PARSER_JSON,
              "<string>:1.3: syntax error, unexpected {, "
              "expecting end of file");

    // bad commas
    testError("{ , }\n",
              Parser4Context::PARSER_JSON,
              "<string>:1.3: syntax error, unexpected \",\", "
              "expecting }");
    testError("{ , \"foo\":true }\n",
              Parser4Context::PARSER_JSON,
              "<string>:1.3: syntax error, unexpected \",\", "
              "expecting }");
    testError("{ \"foo\":true, }\n",
              Parser4Context::PARSER_JSON,
              "<string>:1.15: syntax error, unexpected }, "
              "expecting constant string");

    // bad type
    testError("{ \"Dhcp4\":{\n"
              "  \"valid-lifetime\":false }}\n",
              Parser4Context::PARSER_DHCP4,
              "<string>:2.20-24: syntax error, unexpected boolean, "
              "expecting integer");

    // unknown keyword
    testError("{ \"Dhcp4\":{\n"
              " \"valid_lifetime\":600 }}\n",
              Parser4Context::PARSER_DHCP4,
              "<string>:2.2-17: got unexpected keyword "
              "\"valid_lifetime\" in Dhcp4 map.");
}

505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526
// Check unicode escapes
TEST(ParserTest, unicodeEscapes) {
    ConstElementPtr result;
    string json;

    // check we can reread output
    for (char c = -128; c < 127; ++c) {
        string ins(" ");
        ins[1] = c;
        ConstElementPtr e(new StringElement(ins));
        json = e->str();
        ASSERT_NO_THROW(
        try {
            Parser4Context ctx;
            result = ctx.parseString(json, Parser4Context::PARSER_JSON);
        } catch (const std::exception &x) {
            cout << "EXCEPTION: " << x.what() << endl;
            throw;
        });
        ASSERT_EQ(Element::string, result->getType());
        EXPECT_EQ(ins, result->stringValue());
    }
527
}
528

529
// This test checks that all representations of a slash are recognized properly.
530
TEST(ParserTest, unicodeSlash) {
531
    // check the 4 possible encodings of solidus '/'
532 533
    ConstElementPtr result;
    string json = "\"/\\/\\u002f\\u002F\"";
534 535 536 537 538 539 540 541 542 543
    ASSERT_NO_THROW(
    try {
        Parser4Context ctx;
        result = ctx.parseString(json, Parser4Context::PARSER_JSON);
    } catch (const std::exception &x) {
        cout << "EXCEPTION: " << x.what() << endl;
        throw;
    });
    ASSERT_EQ(Element::string, result->getType());
    EXPECT_EQ("////", result->stringValue());
544
}
545

546
};
547 548
};
};