Commit 016e77e5 authored by Tomek Mrugalski's avatar Tomek Mrugalski 🛰

[3400] Comment removal implemented in lib/cc/data

parent ecb53554
...@@ -594,17 +594,18 @@ Element::nameToType(const std::string& type_name) { ...@@ -594,17 +594,18 @@ Element::nameToType(const std::string& type_name) {
} }
ElementPtr ElementPtr
Element::fromJSON(std::istream& in) throw(JSONError) { Element::fromJSON(std::istream& in, bool preproc) throw(JSONError) {
int line = 1, pos = 1; int line = 1, pos = 1;
return (fromJSON(in, "<istream>", line, pos)); return (fromJSON(preproc?preprocess(in):in, "<istream>", line, pos));
} }
ElementPtr ElementPtr
Element::fromJSON(std::istream& in, const std::string& file_name) Element::fromJSON(std::istream& in, const std::string& file_name, bool preproc)
throw(JSONError) throw(JSONError)
{ {
int line = 1, pos = 1; int line = 1, pos = 1;
return (fromJSON(in, file_name, line, pos)); return (fromJSON(preproc?preprocess(in):in, file_name, line, pos));
} }
ElementPtr ElementPtr
...@@ -682,11 +683,13 @@ Element::fromJSON(std::istream& in, const std::string& file, int& line, ...@@ -682,11 +683,13 @@ Element::fromJSON(std::istream& in, const std::string& file, int& line,
} }
ElementPtr ElementPtr
Element::fromJSON(const std::string& in) { Element::fromJSON(const std::string& in, bool preproc) {
std::stringstream ss; std::stringstream ss;
ss << in; ss << in;
int line = 1, pos = 1; int line = 1, pos = 1;
ElementPtr result(fromJSON(ss, "<string>", line, pos)); ElementPtr result(fromJSON(preproc?preprocess(ss):ss, "<string>",
line, pos));
skipChars(ss, WHITESPACE, line, pos); skipChars(ss, WHITESPACE, line, pos);
// ss must now be at end // ss must now be at end
if (ss.peek() != EOF) { if (ss.peek() != EOF) {
...@@ -1013,5 +1016,35 @@ merge(ElementPtr element, ConstElementPtr other) { ...@@ -1013,5 +1016,35 @@ merge(ElementPtr element, ConstElementPtr other) {
} }
} }
std::istream& Element::preprocess(std::istream& in) {
// There is no assignment operator defined for streams, so we can't return
// stream as an object. If we return a pointer, someone would have to free
// it. So the most convenient way is to return a reference. However, since
// it's not allowed to return references to automatic variables, we must
// make sure that the reference is valid after this method returns. So
// returning a value to static object seems the only feasible way to go.
static stringstream filtered;
std::string line;
filtered.clear();
filtered.str("");
while (std::getline(in, line)) {
// If this is a comments line, replace it with empty line
// (so the line numbers will still match
if (!line.empty() && line[0] == '#') {
line = "";
}
// getline() removes end line charaters. Unfortunately, we need
// it for getting the line numbers right (in case we report an
// error.
filtered << line;
filtered << "\n";
}
return filtered;
}
} }
} }
...@@ -138,6 +138,7 @@ protected: ...@@ -138,6 +138,7 @@ protected:
: type_(t), position_(pos) { : type_(t), position_(pos) {
} }
public: public:
// any is a special type used in list specifications, specifying // any is a special type used in list specifications, specifying
...@@ -388,18 +389,35 @@ public: ...@@ -388,18 +389,35 @@ public:
//@{ //@{
/// Creates an Element from the given JSON string /// Creates an Element from the given JSON string
/// \param in The string to parse the element from /// \param in The string to parse the element from
/// \param preproc specified whether preprocessing (e.g. comment removal)
/// should be performed
/// \return An ElementPtr that contains the element(s) specified /// \return An ElementPtr that contains the element(s) specified
/// in the given string. /// in the given string.
static ElementPtr fromJSON(const std::string& in); static ElementPtr fromJSON(const std::string& in, bool preproc = false);
/// Creates an Element from the given input stream containing JSON
/// formatted data.
///
/// \param in The string to parse the element from
/// \param preproc specified whether preprocessing (e.g. comment removal)
/// should be performed
/// \return An ElementPtr that contains the element(s) specified
/// in the given input stream.
static ElementPtr fromJSON(std::istream& in, bool preproc = false)
throw(JSONError);
/// Creates an Element from the given input stream containing JSON /// Creates an Element from the given input stream containing JSON
/// formatted data. /// formatted data.
/// ///
/// \param in The string to parse the element from /// \param in The string to parse the element from
/// \param file_name specified input file name (used in error reporting)
/// \param preproc specified whether preprocessing (e.g. comment removal)
/// should be performed
/// \return An ElementPtr that contains the element(s) specified /// \return An ElementPtr that contains the element(s) specified
/// in the given input stream. /// in the given input stream.
static ElementPtr fromJSON(std::istream& in) throw(JSONError); static ElementPtr fromJSON(std::istream& in, const std::string& file_name,
static ElementPtr fromJSON(std::istream& in, const std::string& file_name) bool preproc = false)
throw(JSONError); throw(JSONError);
/// Creates an Element from the given input stream, where we keep /// Creates an Element from the given input stream, where we keep
...@@ -435,6 +453,21 @@ public: ...@@ -435,6 +453,21 @@ public:
/// \return the corresponding type value /// \return the corresponding type value
static Element::types nameToType(const std::string& type_name); static Element::types nameToType(const std::string& type_name);
/// \brief input text preprocessor
///
/// This method performs preprocessing of the input stream (which is
/// expected to contain a text version of to be parsed JSON). For now the
/// sole supported operation is bash-style (line starting with #) comment
/// removal, but it will be extended later to cover more cases (C, C++ style
/// comments, file inclusions, maybe macro replacements?)
///
/// It reads all contents of the input stream, filters the content and
/// returns the result in a different stream.
///
/// @param in input stream to be preprocessed
/// @return a new stream that has the content filtered.
static std::istream& preprocess(std::istream& in);
/// \name Wire format factory functions /// \name Wire format factory functions
/// These function pparse the wireformat at the given stringstream /// These function pparse the wireformat at the given stringstream
......
...@@ -930,6 +930,60 @@ TEST(Element, merge) { ...@@ -930,6 +930,60 @@ TEST(Element, merge) {
} }
// This test checks whether it is possible to ignore comments. It also checks
// that the comments are ignored only when told to.
TEST(Element, preprocessor) {
string no_comment = "{ \"a\": 1,\n"
" \"b\": 2}";
string head_comment = "# this is a comment, ignore me\n"
"{ \"a\": 1,\n"
" \"b\": 2}";
string mid_comment = "{ \"a\": 1,\n"
"# this is a comment, ignore me\n"
" \"b\": 2}";
string tail_comment = "{ \"a\": 1,\n"
" \"b\": 2}"
"# this is a comment, ignore me\n";
string dbl_head_comment = "# this is a comment, ignore me\n"
"# second line, still ignored\n"
"{ \"a\": 1,\n"
" \"b\": 2}";
string dbl_mid_comment = "{ \"a\": 1,\n"
"# this is a comment, ignore me\n"
"# second line, still ignored\n"
" \"b\": 2}";
string dbl_tail_comment = "{ \"a\": 1,\n"
" \"b\": 2}"
"# this is a comment, ignore me\n"
"# second line, still ignored\n";
// This is what we expect in all cases.
ElementPtr exp = Element::fromJSON(no_comment);
// Let's convert them all and see that the result it the same every time
EXPECT_TRUE(exp->equals(*Element::fromJSON(head_comment, true)));
EXPECT_TRUE(exp->equals(*Element::fromJSON(mid_comment, true)));
EXPECT_TRUE(exp->equals(*Element::fromJSON(tail_comment, true)));
EXPECT_TRUE(exp->equals(*Element::fromJSON(dbl_head_comment, true)));
EXPECT_TRUE(exp->equals(*Element::fromJSON(dbl_mid_comment, true)));
EXPECT_TRUE(exp->equals(*Element::fromJSON(dbl_tail_comment, true)));
// With preprocessing disabled, it should fail all around
EXPECT_THROW(Element::fromJSON(head_comment), JSONError);
EXPECT_THROW(Element::fromJSON(mid_comment), JSONError);
EXPECT_THROW(Element::fromJSON(tail_comment), JSONError);
EXPECT_THROW(Element::fromJSON(dbl_head_comment), JSONError);
EXPECT_THROW(Element::fromJSON(dbl_mid_comment), JSONError);
EXPECT_THROW(Element::fromJSON(dbl_tail_comment), JSONError);
}
TEST(Element, getPosition) { TEST(Element, getPosition) {
// Create a JSON string holding different type of values. Some of the // Create a JSON string holding different type of values. Some of the
// values in the config string are not aligned, so as we can check that // values in the config string are not aligned, so as we can check that
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment