All geek questions in one place

BNF recursive rule using forced spirit

I am trying to write a parser for the following BNF rules using boost spirit (Boost v1.64)
Rules:

<numeric-literal>::= integer  
<type-name> ::= "in" | "out" | "in_out"  
<array-type-spec> ::= <type-spec> "[" [<numeric-literal>] "]"  
<tuple-type-spec> ::= "(" <type-spec> ("," <type-spec>)+ ")"
<type-spec> ::= <type-name> | <array-type-spec> | <tuple-type-spec>

Below is my attempt using boost::make_recursive_variant
It seems to be working fine on a string in
But it fails on in[2].
Where is my mistake? What would be an elegant solution?

namespace Ast {
enum class TypeName { IN, OUT, INOUT};
using NumericLiteral = int;
    using TypeSpec = boost::make_recursive_variant
    <
    TypeName,
    std::pair<boost::recursive_variant_, NumericLiteral>,
    std::vector < boost::recursive_variant_ >
    >::type;
}
//grammar:
namespace myGrammar {
namespace qi = boost::spirit::qi;

template <typename Iterator = char const*,typename Signature = Ast::TypeSpec()>
struct myRules : qi::grammar < Iterator, Signature> {

    myRules() : myRules::base_type(start) {
        fillSymbols();
        rNumericLiteral = qi::int_;
        rTypeName = sTypeName;
        rTypeSpec = rTypeName | (rTypeSpec >> '[' >> rNumericLiteral >> ']') | ('(' >> qi::repeat(2, qi::inf)[(rTypeSpec % ',')] >> ')');

        start = qi::skip(qi::space)[rTypeSpec];
    }

private:
    using Skipper = qi::space_type;
    qi::rule<Iterator,  Ast::TypeSpec()> start;
    qi::rule<Iterator, Ast::NumericLiteral(), Skipper> rNumericLiteral;

    qi::rule<Iterator, Ast::TypeName(), Skipper> rTypeName;
    qi::rule<Iterator, Ast::TypeSpec(), Skipper> rTypeSpec;


    //symbols
    qi::symbols<char, Ast::TypeName>sTypeName;
    void fillSymbols()
    {
        using namespace Ast;
        sTypeName.add
            ("in", TypeName::IN)
            ("out", TypeName::OUT)
            ("in_out", TypeName::INOUT)
    }

};
}
+4
source share
1 answer

There is a problem translating this 1: 1 grammar into the PEG grammar, since left recursion leads to infinite recursion.

You can still change the rules anyway, so left recursion does not occur, but you will have more problems with the AST synthesis you want.

, :

Live On Coliru

//#define BOOST_SPIRIT_DEBUG
#include <boost/spirit/include/qi.hpp>
#include <boost/fusion/adapted/std_pair.hpp>

/*
    <numeric-literal> ::= integer
    <type-name>       ::= "in" | "out" | "in_out"
    <array-type-spec> ::= <type-spec> "[" [<numeric-literal>] "]"
    <tuple-type-spec> ::= "(" <type-spec> ("," <type-spec>)+ ")"
    <type-spec>       ::= <type-name> | <array-type-spec> | <tuple-type-spec>
*/

namespace Ast {
    enum class TypeName { IN, OUT, INOUT };

    static inline std::ostream& operator<<(std::ostream& os, TypeName tn) {
        switch(tn) {
            case TypeName::IN:    return os << "IN";
            case TypeName::OUT:   return os << "OUT";
            case TypeName::INOUT: return os << "INOUT";
        }
        return os << "?";
    }

    using NumericLiteral = int;

    using TypeSpec = boost::make_recursive_variant<
        TypeName,
        std::pair<boost::recursive_variant_, NumericLiteral>,
        std::vector<boost::recursive_variant_>
    >::type;

    using ArraySpec = std::pair<TypeSpec, NumericLiteral>;
    using TupleSpec = std::vector<TypeSpec>;
}

// grammar:
namespace myGrammar {
    namespace qi = boost::spirit::qi;

    template <typename Iterator = char const *, typename Signature = Ast::TypeSpec()>
        struct myRules : qi::grammar<Iterator, Signature> {

            myRules() : myRules::base_type(start) {
                rNumericLiteral = qi::int_;
                rTypeName       = sTypeName >> !qi::alpha;
                rTupleSpec      = '(' >> rTypeSpec >> +(',' >> rTypeSpec) >> ')'; 
                rScalarSpec     = rTypeName | rTupleSpec;
                rArraySpec      = rScalarSpec >> '[' >> rNumericLiteral >> ']';
                rTypeSpec       = rArraySpec | rScalarSpec;

                start = qi::skip(qi::space)[rTypeSpec >> qi::eoi];

                BOOST_SPIRIT_DEBUG_NODES((start)(rTypeSpec)(rTypeName)(rArraySpec)(rScalarSpec)(rTypeSpec)(rNumericLiteral))
            }

          private:
            using Skipper = qi::space_type;
            qi::rule<Iterator, Ast::TypeSpec()> start;
            qi::rule<Iterator, Ast::NumericLiteral(), Skipper> rNumericLiteral;
            qi::rule<Iterator, Ast::ArraySpec(), Skipper> rArraySpec;
            qi::rule<Iterator, Ast::TypeSpec(), Skipper> rTypeSpec, rScalarSpec;
            qi::rule<Iterator, Ast::TupleSpec(), Skipper> rTupleSpec;
            // implicit lexeme
            qi::rule<Iterator, Ast::TypeName()> rTypeName;

            // symbols
            struct TypeName_r : qi::symbols<char, Ast::TypeName> { 
                TypeName_r() {
                    using Ast::TypeName;
                    add ("in", TypeName::IN)
                        ("out", TypeName::OUT)
                        ("in_out", TypeName::INOUT);
                }
            } sTypeName;
        };
}

static inline std::ostream& operator<<(std::ostream& os, Ast::TypeSpec tn) {
    struct {
        std::ostream& _os;

        void operator()(Ast::TypeSpec const& ts) const {
            apply_visitor(*this, ts);
        }
        void operator()(Ast::TypeName tn) const { std::cout << tn; }
        void operator()(Ast::TupleSpec const& tss) const { 
            std::cout << "(";
            for (auto const& ts: tss) {
                (*this)(ts); 
                std::cout << ", ";
            }
            std::cout << ")";
        }
        void operator()(Ast::ArraySpec const& as) const { 
            (*this)(as.first);
            std::cout << '[' << as.second << ']';
        }
    } const dumper{os};

    dumper(tn);
    return os;
}

int main() {
    using It = std::string::const_iterator;
    myGrammar::myRules<It> const parser;

    std::string const test_ok[] = {
        "in",
        "out",
        "in_out",
        "(in, out)",
        "(out, in)",
        "(in, in, in, out, in_out)",
        "in[13]",
        "in[0]",
        "in[-2]",
        "in[1][2][3]",
        "in[3][3][3]",
        "(in[3][3][3], out, in_out[0])",
        "(in[3][3][3], out, in_out[0])",
        "(in, out)[13]",
        "(in, out)[13][0]",
    };

    std::string const test_fail[] = {
        "",
        "i n",
        "inout",
        "()",
        "(in)",
        "(out)",
        "(in_out)",
        "IN",
    };

    auto expect = [&](std::string const& sample, bool expected) {
        It f = sample.begin(), l = sample.end(); 

        Ast::TypeSpec spec;
        bool ok = parse(f, l, parser, spec);

        std::cout << "Test passed:" << std::boolalpha << (expected == ok) << "\n";

        if (expected || (expected != ok)) {
            if (ok) {
                std::cout << "Parsed: " << spec << "\n";
            } else {
                std::cout << "Parse failed\n";
            }
        }

        if (f!=l) {
            std::cout << "Remaining unparsed: '" << std::string(f,l) << "'\n";
        }
    };

    for (std::string const sample : test_ok)   expect(sample, true); 
    for (std::string const sample : test_fail) expect(sample, false); 
}


Test passed:true
Parsed: IN
Test passed:true
Parsed: OUT
Test passed:true
Parsed: INOUT
Test passed:true
Parsed: (IN, OUT, )
Test passed:true
Parsed: (OUT, IN, )
Test passed:true
Parsed: (IN, IN, IN, OUT, INOUT, )
Test passed:true
Parsed: IN[13]
Test passed:true
Parsed: IN[0]
Test passed:true
Parsed: IN[-2]
Test passed:false
Parse failed
Remaining unparsed: 'in[1][2][3]'
Test passed:false
Parse failed
Remaining unparsed: 'in[3][3][3]'
Test passed:false
Parse failed
Remaining unparsed: '(in[3][3][3], out, in_out[0])'
Test passed:false
Parse failed
Remaining unparsed: '(in[3][3][3], out, in_out[0])'
Test passed:true
Parsed: (IN, OUT, )[13]
Test passed:false
Parse failed
Remaining unparsed: '(in, out)[13][0]'
Test passed:true
Test passed:true
Remaining unparsed: 'i n'
Test passed:true
Remaining unparsed: 'inout'
Test passed:true
Remaining unparsed: '()'
Test passed:true
Remaining unparsed: '(in)'
Test passed:true
Remaining unparsed: '(out)'
Test passed:true
Remaining unparsed: '(in_out)'
Test passed:true
Remaining unparsed: 'IN'

, , , in[1][2]. , , "" :

rScalarSpec     = rTypeName | rTupleSpec;
rArraySpec      = rScalarSpec >> '[' >> rNumericLiteral >> ']';
rTypeSpec       = rArraySpec | rScalarSpec;

, , . , , .

, , [ :

    rArraySpec      = rScalarSpec >> '[' >> rNumericLiteral >> ']' >> !qi::lit('[')
                    | rArraySpec  >> '[' >> rNumericLiteral >> ']';

- BOOM - - ( , in[1][).

.

.

  • , scalar/array AST. , , .

  • , , , '['. , (very long spec)[1][1][1][1][1][1][1][1][1][1].

, -:)

AST

TypeSpec (, ) :

namespace Ast {
    enum class TypeName { IN, OUT, INOUT };

    static inline std::ostream& operator<<(std::ostream& os, TypeName tn) {
        switch(tn) {
            case TypeName::IN:    return os << "IN";
            case TypeName::OUT:   return os << "OUT";
            case TypeName::INOUT: return os << "INOUT";
        }
        return os << "?";
    }

    struct TypeSpec;

    using ScalarSpec = boost::make_recursive_variant<
        TypeName,
        std::vector<TypeSpec>
    >::type;

    struct TypeSpec {
        ScalarSpec            spec;
        std::vector<unsigned> dim;
    };

    using TupleSpec = std::vector<TypeSpec>;
}

, , . , 0 . "" " ".

:

rRank      %= qi::uint_ [qi::_pass = (qi::_1 > 0)];
rTypeName   = sTypeName;
rTupleSpec  = '(' >> rTypeSpec >> +(',' >> rTypeSpec) >> ')'; 
rScalarSpec = rTypeName | rTupleSpec;
rTypeSpec   = rScalarSpec >> *('[' >> rRank >> ']');

Phoenix, , 0

, :

FULL DEMO

Live On Coliru

//#define BOOST_SPIRIT_DEBUG
#include <boost/spirit/include/qi.hpp>
#include <boost/spirit/include/phoenix.hpp>
#include <boost/fusion/adapted.hpp>

/*
    <numeric-literal> ::= integer
    <type-name>       ::= "in" | "out" | "in_out"
    <array-type-spec> ::= <type-spec> "[" [<numeric-literal>] "]"
    <tuple-type-spec> ::= "(" <type-spec> ("," <type-spec>)+ ")"
    <type-spec>       ::= <type-name> | <array-type-spec> | <tuple-type-spec>
*/

namespace Ast {
    enum class TypeName { IN, OUT, INOUT };

    static inline std::ostream& operator<<(std::ostream& os, TypeName tn) {
        switch(tn) {
            case TypeName::IN:    return os << "IN";
            case TypeName::OUT:   return os << "OUT";
            case TypeName::INOUT: return os << "INOUT";
        }
        return os << "?";
    }

    struct TypeSpec;

    using ScalarSpec = boost::make_recursive_variant<
        TypeName,
        std::vector<TypeSpec>
    >::type;

    struct TypeSpec {
        ScalarSpec            spec;
        std::vector<unsigned> dim;
    };

    using TupleSpec = std::vector<TypeSpec>;
}

BOOST_FUSION_ADAPT_STRUCT(Ast::TypeSpec, spec, dim)

// grammar:
namespace myGrammar {
    namespace qi = boost::spirit::qi;

    template <typename Iterator = char const *, typename Signature = Ast::TypeSpec()>
        struct myRules : qi::grammar<Iterator, Signature> {

            myRules() : myRules::base_type(start) {
                rRank      %= qi::uint_ [qi::_pass = (qi::_1 > 0)];
                rTypeName   = sTypeName;
                rTupleSpec  = '(' >> rTypeSpec >> +(',' >> rTypeSpec) >> ')'; 
                rScalarSpec = rTypeName | rTupleSpec;
                rTypeSpec   = rScalarSpec >> *('[' >> rRank >> ']');

                start = qi::skip(qi::space)[rTypeSpec >> qi::eoi];

                BOOST_SPIRIT_DEBUG_NODES((start)(rTypeSpec)(rTypeName)(rScalarSpec)(rTypeSpec)(rRank))
            }

          private:
            using Skipper = qi::space_type;
            qi::rule<Iterator, Ast::TypeSpec()> start;
            qi::rule<Iterator, Ast::ScalarSpec(), Skipper> rScalarSpec;
            qi::rule<Iterator, Ast::TypeSpec(),   Skipper> rTypeSpec;
            qi::rule<Iterator, Ast::TupleSpec(),  Skipper> rTupleSpec;
            // implicit lexeme
            qi::rule<Iterator, Ast::TypeName()> rTypeName;
            qi::rule<Iterator, unsigned()>      rRank;

            // symbols
            struct TypeName_r : qi::symbols<char, Ast::TypeName> { 
                TypeName_r() {
                    using Ast::TypeName;
                    add ("in", TypeName::IN)
                        ("out", TypeName::OUT)
                        ("in_out", TypeName::INOUT);
                }
            } sTypeName;
        };
}

static inline std::ostream& operator<<(std::ostream& os, Ast::TypeSpec tn) {
    struct {
        std::ostream& _os;

        void operator()(Ast::ScalarSpec const& ts) const {
            apply_visitor(*this, ts);
        }
        void operator()(Ast::TypeName tn) const { std::cout << tn; }
        void operator()(Ast::TupleSpec const& tss) const { 
            std::cout << "(";
            for (auto const& ts: tss) {
                (*this)(ts); 
                std::cout << ", ";
            }
            std::cout << ")";
        }
        void operator()(Ast::TypeSpec const& as) const { 
            (*this)(as.spec);
            for (auto rank : as.dim)
                std::cout << '[' << rank << ']';
        }
    } const dumper{os};

    dumper(tn);
    return os;
}

int main() {
    using It = std::string::const_iterator;
    myGrammar::myRules<It> const parser;

    std::string const test_ok[] = {
        "in",
        "out",
        "in_out",
        "(in, out)",
        "(out, in)",
        "(in, in, in, out, in_out)",
        "in[13]",
        "in[1][2][3]",
        "in[3][3][3]",
        "(in[3][3][3], out, in_out[1])",
        "(in[3][3][3], out, in_out[1])",
        "(in, out)[13]",
        "(in, out)[13][14]",
    };

    std::string const test_fail[] = {
        "",
        "i n",
        "inout",
        "()",
        "(in)",
        "(out)",
        "(in_out)",
        "IN",
        "in[0]",
        "in[-2]",
        "(in[3][3][3], out, in_out[0])",
        "(in[3][3][3], out, in_out[0])",
    };

    auto expect = [&](std::string const& sample, bool expected) {
        It f = sample.begin(), l = sample.end(); 

        Ast::TypeSpec spec;
        bool ok = parse(f, l, parser, spec);

        std::cout << "Test passed:" << std::boolalpha << (expected == ok) << "\n";

        if (expected || (expected != ok)) {
            if (ok) {
                std::cout << "Parsed: " << spec << "\n";
            } else {
                std::cout << "Parse failed\n";
            }
        }

        if (f!=l) {
            std::cout << "Remaining unparsed: '" << std::string(f,l) << "'\n";
        }
    };

    for (std::string const sample : test_ok)   expect(sample, true); 
    for (std::string const sample : test_fail) expect(sample, false); 
}


Test passed:true
Parsed: IN
Test passed:true
Parsed: OUT
Test passed:true
Parsed: INOUT
Test passed:true
Parsed: (IN, OUT, )
Test passed:true
Parsed: (OUT, IN, )
Test passed:true
Parsed: (IN, IN, IN, OUT, INOUT, )
Test passed:true
Parsed: IN[13]
Test passed:true
Parsed: IN[1][2][3]
Test passed:true
Parsed: IN[3][3][3]
Test passed:true
Parsed: (IN[3][3][3], OUT, INOUT[1], )
Test passed:true
Parsed: (IN[3][3][3], OUT, INOUT[1], )
Test passed:true
Parsed: (IN, OUT, )[13]
Test passed:true
Parsed: (IN, OUT, )[13][14]
Test passed:true
Test passed:true
Remaining unparsed: 'i n'
Test passed:true
Remaining unparsed: 'inout'
Test passed:true
Remaining unparsed: '()'
Test passed:true
Remaining unparsed: '(in)'
Test passed:true
Remaining unparsed: '(out)'
Test passed:true
Remaining unparsed: '(in_out)'
Test passed:true
Remaining unparsed: 'IN'
Test passed:true
Remaining unparsed: 'in[0]'
Test passed:true
Remaining unparsed: 'in[-2]'
Test passed:true
Remaining unparsed: '(in[3][3][3], out, in_out[0])'
Test passed:true
Remaining unparsed: '(in[3][3][3], out, in_out[0])'
+2

Source: https://habr.com/ru/post/1681892/

More articles:


All Articles