How to generalize the spirit parser to take lists in random order?

Question

I have a simple parser that can parse lists of ints or quoted strings.

If I use SIMPLE_CASE where I accept the input:

std::string input1 = "{ INT: 42, 24 STR: \"Smith\", \"John\" }"; 

it is correctly parsed into my_record , which contains an int list and a std :: string list.

I want to change this code to general so that it can accept zero or more INT lists and zero or more STR lists in random order and my_record them in my_record in the correct order. I would like my second, more general test case:

 std::string input1 = "{ STR: \"Joe\" INT: 42, 24 STR: \"Smith\", \"John\" }"; 

analyze as:

 client::my_record expected1 { { 42, 24 }, {"Joe", "Smith", "John"} }; 

The code below works fine if I run:

 /tmp$ g++ -DSIMPLE_CASE -g -std=c++11 sandbox.cpp -o sandbox && ./sandbox 

but I'm not sure how to make the general case work at startup:

 /tmp$ g++ -g -std=c++11 sandbox.cpp -o sandbox && ./sandbox 

Code for sandbox.cpp

 #include <boost/config/warning_disable.hpp> #include <boost/spirit/include/qi.hpp> #include <boost/spirit/include/phoenix_core.hpp> #include <boost/spirit/include/phoenix_operator.hpp> #include <boost/spirit/include/phoenix_object.hpp> #include <boost/fusion/include/adapt_struct.hpp> #include <boost/fusion/include/io.hpp> #include <string> #include <complex> #include <algorithm> namespace client { namespace qi = boost::spirit::qi; namespace ascii = boost::spirit::ascii; struct my_record { std::vector<int> m_ints; std::vector<std::string> m_strs; bool operator==( const my_record& other ) const { return std::equal( m_ints.begin(), m_ints.end(), other.m_ints.begin() ) && std::equal( m_strs.begin(), m_strs.end(), other.m_strs.begin() ); } bool operator!=( const my_record& other ) const { return ! operator==( other ); } friend std::ostream& operator<<( std::ostream& os, const my_record& rec ); }; std::ostream& operator<<( std::ostream& os, const my_record& rec ) { for( const auto& x : rec.m_ints ) std::cerr << x << ' '; std::cerr << std::endl; for( const auto& x : rec.m_strs ) std::cerr << x << ' '; std::cerr << std::endl; } } BOOST_FUSION_ADAPT_STRUCT( client::my_record, (std::vector<int>, m_ints) (std::vector<std::string>, m_strs) ) namespace client { template <typename Iterator> struct employee_parser : qi::grammar<Iterator, my_record(), ascii::space_type> { employee_parser() : employee_parser::base_type(start) { using qi::int_; using qi::lit; using qi::double_; using qi::lexeme; using ascii::char_; quoted_string %= lexeme['"' >> +(char_ - '"') >> '"']; #ifdef SIMPLE_CASE start %= '{' >> int_list >> str_list >> '}' ; #else // not sure how to approach this start %= '{' >> *(int_list) // want zero or more of these, in any order >> *(str_list) // want zero or more of these, in any order >> '}' ; #endif str_list %= lit( "STR:" ) >> quoted_string % ',' ; int_list %= lit( "INT:" ) >> int_ % ',' ; } qi::rule<Iterator, std::string(), ascii::space_type> quoted_string; qi::rule<Iterator, std::vector<std::string>(), ascii::space_type> str_list; qi::rule<Iterator, std::vector<int>(), ascii::space_type> int_list; qi::rule<Iterator, my_record(), ascii::space_type> start; }; } static int TryParse( const std::string& input, const client::my_record& expected ) { using boost::spirit::ascii::space; client::my_record rec; auto iter = input.begin(), end = input.end(); client::employee_parser<decltype(iter)> g; phrase_parse( iter, end, g, space, rec ); if ( iter!=end ) { std::cerr << "failed to parse completely" << std::endl; return -1; } else if ( rec!=expected ) { std::cerr << "unexpected result in parse" << std::endl; std::cerr << rec; return -1; } return 0; } int main(int argc, char* argv[]) { #ifdef SIMPLE_CASE client::my_record expected1 { { 42, 24 }, {"Smith", "John"} }, emp; std::string input1 = "{ INT: 42, 24 STR: \"Smith\", \"John\" }"; return TryParse( input1, expected1 ); #else client::my_record expected1 { { 42, 24 }, {"Joe", "Smith", "John"} }, emp; std::string input1 = "{ STR: \"Joe\" INT: 42, 24 STR: \"Smith\", \"John\" }"; return TryParse( input1, expected1 ); #endif } 

Answer 1

Your grammar is incorrect

  start %= '{' >> *(int_list) // want zero or more of these, in any order >> *(str_list) // want zero or more of these, in any order >> '}' ; 

This means accepting any int number, followed by any string number. You cannot have int , string , int or any other combination.

You need something like

  start %= '{' >> *( int_list // want zero or more of these, in any order | str_list // want zero or more of these, in any order ) >> '}' ; 

But, obviously, you need to merge this into a data structure, fearing that you might have to use semantic actions.

ALSO:

while I am here, I cannot allow this slide:

  std::ostream& operator<<( std::ostream& os, const my_record& rec ) { for( const auto& x : rec.m_ints ) std::cerr << x << ' '; std::cerr << std::endl; for( const auto& x : rec.m_strs ) std::cerr << x << ' '; std::cerr << std::endl; } 

should be attached to os as:

  for( const auto& x : rec.m_ints ) os << x << ' '; os << '\n'; 

Also try and avoid endl ing in the stream insert statement, use \n if you need a new line.

DECISION:

In the end, the phoenix, push_back, and binding functions needed to be used.

 template<typename Iterator> struct my_grammar : qi::grammar<Iterator, my_record(), ascii::space_type> { my_grammar() : my_grammar::base_type(start) { quoted_string %= qi::lexeme['"' >> +(qi::char_ - '"') >> '"']; start = qi::lit("{") >> *( "INT:" >> qi::int_ [ phx::push_back( phx::at_c<0>( qi::_val ), qi::_1 ) ] % "," | "STR:" >> quoted_string [ phx::push_back( phx::bind( &my_record::m_strs, qi::_val ), qi::_1 ) ] % "," ) >> "}" ; } qi::rule<Iterator, std::string(), ascii::space_type> quoted_string; qi::rule<Iterator, my_record(), ascii::space_type> start; }; 

The entire list of codes can be seen here:

http://ideone.com/XW18Z2

Answer 2

Alternative using is_container and push_back_container instead of semantic actions:

Step1: delete the macro BOOST_FUSION_ADAPT_STRUCT .

Step 2: change the start rule.

 start %= '{' >> *(int_list // want zero or more of these, in any order | str_list) // want zero or more of these, in any order >> '}' ; 

Step 3. Add the following specializations.

 namespace boost { namespace spirit { namespace traits { template <> struct is_container<client::my_record>: mpl::true_//my_record acts as a container {}; template <> struct container_value<client::my_record> { typedef boost::variant<std::vector<int>,std::vector<std::string>> type;//The elements to add to that container are either vector<int> or vector<string> }; template <> struct push_back_container<client::my_record,std::vector<int>>//when you add a vector of ints... { static bool call(client::my_record& c, std::vector<int> const& val) { c.m_ints.insert(c.m_ints.end(),val.begin(), val.end());//insert it at the end of your acumulated vector of ints return true; } }; template <> struct push_back_container<client::my_record,std::vector<std::string>>//when you add a vector of strings { static bool call(client::my_record& c, std::vector<std::string> const& val)//insert it at the end of your acumulated vector of strings { c.m_strs.insert(c.m_strs.end(),val.begin(),val.end()); return true; } }; }}} 

Here is the complete code on request (compiled with g ++ 4.7.1 and msvc11 if I create the expected result with a few push_backs):

Updated example of adding another member vector of adapted structures.

 #include <boost/config/warning_disable.hpp> #include <boost/spirit/include/qi.hpp> #include <boost/fusion/adapted/struct.hpp> #include <string> #include <vector> #include <iostream> namespace client { struct my_subrec { double foo; double bar; bool operator==( const my_subrec& other ) const { return foo==other.foo && bar==other.bar; } }; std::ostream& operator<<( std::ostream& os, const my_subrec& rec ) { os << rec.foo << "->" << rec.bar; return os; } } BOOST_FUSION_ADAPT_STRUCT(client::my_subrec, (double, foo) (double, bar) ) namespace client { namespace qi = boost::spirit::qi; namespace ascii = boost::spirit::ascii; struct my_record { std::vector<int> m_ints; std::vector<std::string> m_strs; std::vector<my_subrec> m_recs; bool operator==( const my_record& other ) const { return std::equal( m_ints.begin(), m_ints.end(), other.m_ints.begin() ) && std::equal( m_strs.begin(), m_strs.end(), other.m_strs.begin() ) && std::equal( m_recs.begin(), m_recs.end(), other.m_recs.begin() ); } bool operator!=( const my_record& other ) const { return ! operator==( other ); } friend std::ostream& operator<<( std::ostream& os, const my_record& rec ); }; std::ostream& operator<<( std::ostream& os, const my_record& rec ) { for( const auto& x : rec.m_ints ) os << x << ' '; os << '\n'; for( const auto& x : rec.m_strs ) os << x << ' '; os << '\n'; for( const auto& x : rec.m_recs ) os << x << ' '; return os; } } //BOOST_FUSION_ADAPT_STRUCT( // client::my_record, // (std::vector<int>, m_ints) // (std::vector<std::string>, m_strs) //) namespace client { template <typename Iterator> struct employee_parser : qi::grammar<Iterator, my_record(), ascii::space_type> { employee_parser() : employee_parser::base_type(start) { using qi::int_; using qi::lit; using qi::double_; using qi::lexeme; using ascii::char_; quoted_string %= lexeme['"' >> +(char_ - '"') >> '"']; #ifdef SIMPLE_CASE start %= '{' >> int_list >> str_list >> '}' ; #else // not sure how to approach this start %= '{' >> *(int_list // want zero or more of these, in any order | str_list // want zero or more of these, in any order | rec_list) >> '}' ; #endif str_list %= lit( "STR:" ) >> quoted_string % ',' ; int_list %= lit( "INT:" ) >> int_ % ',' ; rec_list = lit( "REC:" ) >> rec % ',' ; rec = double_ >> lit('-') >> double_ ; } qi::rule<Iterator, std::string(), ascii::space_type> quoted_string; qi::rule<Iterator, std::vector<std::string>(), ascii::space_type> str_list; qi::rule<Iterator, std::vector<int>(), ascii::space_type> int_list; qi::rule<Iterator, client::my_subrec(), ascii::space_type> rec; qi::rule<Iterator, std::vector<client::my_subrec>(),ascii::space_type> rec_list; qi::rule<Iterator, my_record(), ascii::space_type> start; }; } namespace boost { namespace spirit { namespace traits { template <> struct is_container<client::my_record>: mpl::true_//my_record acts as a container {}; template <> struct container_value<client::my_record> { typedef boost::variant<std::vector<int>,std::vector<std::string>,std::vector<client::my_subrec> >type; //The elements to add to that container are vector<int>, vector<string> or vector<my_subrec> }; template <> struct push_back_container<client::my_record,std::vector<int>>//when you add a vector of ints... { static bool call(client::my_record& c, std::vector<int> const& val) { c.m_ints.insert(c.m_ints.end(),val.begin(), val.end());//insert it at the end of your acumulated vector of ints return true; } }; template <> struct push_back_container<client::my_record,std::vector<std::string>>//when you add a vector of strings { static bool call(client::my_record& c, std::vector<std::string> const& val)//insert it at the end of your acumulated vector of strings { c.m_strs.insert(c.m_strs.end(),val.begin(),val.end()); return true; } }; template <> struct push_back_container<client::my_record,std::vector<client::my_subrec>>//when you add a vector of subrecs { static bool call(client::my_record& c, std::vector<client::my_subrec> const& val)//insert it at the end of your acumulated vector of subrecs { c.m_recs.insert(c.m_recs.end(),val.begin(),val.end()); return true; } }; }}} static int TryParse( const std::string& input, const client::my_record& expected ) { using boost::spirit::ascii::space; client::my_record rec; auto iter = input.begin(), end = input.end(); client::employee_parser<decltype(iter)> g; phrase_parse( iter, end, g, space, rec ); if ( iter!=end ) { std::cerr << "failed to parse completely" << std::endl; return -1; } else if ( rec!=expected ) { std::cerr << "unexpected result in parse" << std::endl; std::cerr << rec; return -1; } std::cout << rec << std::endl; return 0; } int main(int argc, char* argv[]) { #ifdef SIMPLE_CASE client::my_record expected1 { {42, 24 }, {"Smith", "John"} }, emp; std::string input1 = "{ INT: 42, 24 STR: \"Smith\", \"John\" }"; return TryParse( input1, expected1 ); #else client::my_record expected1 { { 42, 24,240 }, {"Joe", "Smith", "John"}, {{1.5,2.5}} }, emp; std::string input1 = "{ STR: \"Joe\" INT: 42, 24 STR: \"Smith\", \"John\" INT: 240 REC: 1.5-2.5 }"; return TryParse( input1, expected1 ); #endif }