From: Tomek Mrugalski Date: Thu, 8 May 2014 14:01:07 +0000 (+0200) Subject: [3400] Comment removal implemented in lib/cc/data X-Git-Tag: trac3434_base~32^2~19 X-Git-Url: http://git.ipfire.org/gitweb.cgi?a=commitdiff_plain;h=016e77e5e8b5c280426000b1cdc993fc6ede88cc;p=thirdparty%2Fkea.git [3400] Comment removal implemented in lib/cc/data --- diff --git a/src/lib/cc/data.cc b/src/lib/cc/data.cc index f833230db1..4ab3526016 100644 --- a/src/lib/cc/data.cc +++ b/src/lib/cc/data.cc @@ -594,17 +594,18 @@ Element::nameToType(const std::string& type_name) { } ElementPtr -Element::fromJSON(std::istream& in) throw(JSONError) { +Element::fromJSON(std::istream& in, bool preproc) throw(JSONError) { + int line = 1, pos = 1; - return (fromJSON(in, "", line, pos)); + return (fromJSON(preproc?preprocess(in):in, "", line, pos)); } ElementPtr -Element::fromJSON(std::istream& in, const std::string& file_name) +Element::fromJSON(std::istream& in, const std::string& file_name, bool preproc) throw(JSONError) { int line = 1, pos = 1; - return (fromJSON(in, file_name, line, pos)); + return (fromJSON(preproc?preprocess(in):in, file_name, line, pos)); } ElementPtr @@ -682,11 +683,13 @@ Element::fromJSON(std::istream& in, const std::string& file, int& line, } ElementPtr -Element::fromJSON(const std::string& in) { +Element::fromJSON(const std::string& in, bool preproc) { std::stringstream ss; ss << in; + int line = 1, pos = 1; - ElementPtr result(fromJSON(ss, "", line, pos)); + ElementPtr result(fromJSON(preproc?preprocess(ss):ss, "", + line, pos)); skipChars(ss, WHITESPACE, line, pos); // ss must now be at end if (ss.peek() != EOF) { @@ -1013,5 +1016,35 @@ merge(ElementPtr element, ConstElementPtr other) { } } +std::istream& Element::preprocess(std::istream& in) { + + // There is no assignment operator defined for streams, so we can't return + // stream as an object. If we return a pointer, someone would have to free + // it. So the most convenient way is to return a reference. However, since + // it's not allowed to return references to automatic variables, we must + // make sure that the reference is valid after this method returns. So + // returning a value to static object seems the only feasible way to go. + static stringstream filtered; + std::string line; + + filtered.clear(); + filtered.str(""); + while (std::getline(in, line)) { + // If this is a comments line, replace it with empty line + // (so the line numbers will still match + if (!line.empty() && line[0] == '#') { + line = ""; + } + + // getline() removes end line charaters. Unfortunately, we need + // it for getting the line numbers right (in case we report an + // error. + filtered << line; + filtered << "\n"; + } + + return filtered; +} + } } diff --git a/src/lib/cc/data.h b/src/lib/cc/data.h index 0850abfa38..207a9d8ed0 100644 --- a/src/lib/cc/data.h +++ b/src/lib/cc/data.h @@ -138,6 +138,7 @@ protected: : type_(t), position_(pos) { } + public: // any is a special type used in list specifications, specifying @@ -388,18 +389,35 @@ public: //@{ /// Creates an Element from the given JSON string /// \param in The string to parse the element from + /// \param preproc specified whether preprocessing (e.g. comment removal) + /// should be performed /// \return An ElementPtr that contains the element(s) specified /// in the given string. - static ElementPtr fromJSON(const std::string& in); + static ElementPtr fromJSON(const std::string& in, bool preproc = false); + + /// Creates an Element from the given input stream containing JSON + /// formatted data. + /// + /// \param in The string to parse the element from + /// \param preproc specified whether preprocessing (e.g. comment removal) + /// should be performed + /// \return An ElementPtr that contains the element(s) specified + /// in the given input stream. + static ElementPtr fromJSON(std::istream& in, bool preproc = false) + throw(JSONError); + /// Creates an Element from the given input stream containing JSON /// formatted data. /// /// \param in The string to parse the element from + /// \param file_name specified input file name (used in error reporting) + /// \param preproc specified whether preprocessing (e.g. comment removal) + /// should be performed /// \return An ElementPtr that contains the element(s) specified /// in the given input stream. - static ElementPtr fromJSON(std::istream& in) throw(JSONError); - static ElementPtr fromJSON(std::istream& in, const std::string& file_name) + static ElementPtr fromJSON(std::istream& in, const std::string& file_name, + bool preproc = false) throw(JSONError); /// Creates an Element from the given input stream, where we keep @@ -435,6 +453,21 @@ public: /// \return the corresponding type value static Element::types nameToType(const std::string& type_name); + /// \brief input text preprocessor + /// + /// This method performs preprocessing of the input stream (which is + /// expected to contain a text version of to be parsed JSON). For now the + /// sole supported operation is bash-style (line starting with #) comment + /// removal, but it will be extended later to cover more cases (C, C++ style + /// comments, file inclusions, maybe macro replacements?) + /// + /// It reads all contents of the input stream, filters the content and + /// returns the result in a different stream. + /// + /// @param in input stream to be preprocessed + /// @return a new stream that has the content filtered. + static std::istream& preprocess(std::istream& in); + /// \name Wire format factory functions /// These function pparse the wireformat at the given stringstream diff --git a/src/lib/cc/tests/data_unittests.cc b/src/lib/cc/tests/data_unittests.cc index 8e31c32a15..6a5e839957 100644 --- a/src/lib/cc/tests/data_unittests.cc +++ b/src/lib/cc/tests/data_unittests.cc @@ -930,6 +930,60 @@ TEST(Element, merge) { } +// This test checks whether it is possible to ignore comments. It also checks +// that the comments are ignored only when told to. +TEST(Element, preprocessor) { + + string no_comment = "{ \"a\": 1,\n" + " \"b\": 2}"; + + string head_comment = "# this is a comment, ignore me\n" + "{ \"a\": 1,\n" + " \"b\": 2}"; + + string mid_comment = "{ \"a\": 1,\n" + "# this is a comment, ignore me\n" + " \"b\": 2}"; + + string tail_comment = "{ \"a\": 1,\n" + " \"b\": 2}" + "# this is a comment, ignore me\n"; + + string dbl_head_comment = "# this is a comment, ignore me\n" + "# second line, still ignored\n" + "{ \"a\": 1,\n" + " \"b\": 2}"; + + string dbl_mid_comment = "{ \"a\": 1,\n" + "# this is a comment, ignore me\n" + "# second line, still ignored\n" + " \"b\": 2}"; + + string dbl_tail_comment = "{ \"a\": 1,\n" + " \"b\": 2}" + "# this is a comment, ignore me\n" + "# second line, still ignored\n"; + + // This is what we expect in all cases. + ElementPtr exp = Element::fromJSON(no_comment); + + // Let's convert them all and see that the result it the same every time + EXPECT_TRUE(exp->equals(*Element::fromJSON(head_comment, true))); + EXPECT_TRUE(exp->equals(*Element::fromJSON(mid_comment, true))); + EXPECT_TRUE(exp->equals(*Element::fromJSON(tail_comment, true))); + EXPECT_TRUE(exp->equals(*Element::fromJSON(dbl_head_comment, true))); + EXPECT_TRUE(exp->equals(*Element::fromJSON(dbl_mid_comment, true))); + EXPECT_TRUE(exp->equals(*Element::fromJSON(dbl_tail_comment, true))); + + // With preprocessing disabled, it should fail all around + EXPECT_THROW(Element::fromJSON(head_comment), JSONError); + EXPECT_THROW(Element::fromJSON(mid_comment), JSONError); + EXPECT_THROW(Element::fromJSON(tail_comment), JSONError); + EXPECT_THROW(Element::fromJSON(dbl_head_comment), JSONError); + EXPECT_THROW(Element::fromJSON(dbl_mid_comment), JSONError); + EXPECT_THROW(Element::fromJSON(dbl_tail_comment), JSONError); +} + TEST(Element, getPosition) { // Create a JSON string holding different type of values. Some of the // values in the config string are not aligned, so as we can check that