Is it a boost spirit regression? - boost

I have a boost spirit parser which works very well with boost 1.46.1, and which doesn't work with boost 1.54.
This parser extract informations from the following sentence, which is a variable initialization in a DSEL : "Position #start=0;0;0".
Informations extracted from this sentence are stored in a structure :
Type of the variable will be stored (here position) ;
Name of variable (here start) ;
The "#" means that the variable is "static" ;
The value of the variable (0;0;0).
The code to extract these inforamtions is as follow :
#include <iostream>
#include <boost/spirit/include/classic.hpp>
#include <boost/spirit/include/qi.hpp>
#include <boost/fusion/include/adapt_struct.hpp>
#include <boost/spirit/include/qi_char_class.hpp>
using namespace std;
using namespace boost::spirit;
struct VariableInitialization
{
std::string m_type;
bool m_is_static;
std::string m_name;
std::string m_value;
};
BOOST_FUSION_ADAPT_STRUCT(
VariableInitialization,
(std::string, m_type)
(bool, m_is_static)
(std::string, m_name)
(std::string, m_value)
)
template <typename Iterator>
struct VariableInitializationParser : qi::grammar<Iterator, VariableInitialization(), ascii::space_type> {
/*!
* IsStatic is a mapping betwen a char and a boolean
*/
struct IsStatic_ : qi::symbols<char, bool> {
IsStatic_()
{
add("#", false)("#", true);
}
}IsStatic;
VariableInitializationParser() :
VariableInitializationParser::base_type(start) {
using qi::lit;
using ascii::char_;
using qi::_val;
using qi::_1;
/*!
* For now, type is one of the three following litterals :
*/
var_type %= lit("Position")|lit("String")|lit("Numeric")|lit("Integer")|lit("Trajectory");
/*!
* identifier is how a variable can be named. Name of variable is an alpha (a-zA-Z) or an _,followed
* by any alpha numeric (a-zA-Z0-9) or a _. The followings are correct :
* _toto _T5ot_To t1oTo ...
* The following are incorrect
* 12toto -tiotp ...
*/
identifier %= ((ascii::alpha|char_('_')) >> *(ascii::alnum|char_('_')));
/*!
* var value can be anything because it's parsed by someone else.
*/
var_value %= qi::lexeme[*(char_)];
start = var_type >> IsStatic >> identifier >> '=' >> var_value;
}
qi::rule<Iterator, std::string(), ascii::space_type> var_type;
qi::rule<Iterator, std::string(), ascii::space_type> identifier;
qi::rule<Iterator, std::string(), ascii::space_type> var_value;
qi::rule<Iterator, VariableInitialization(), ascii::space_type> start;
};
int main()
{
VariableInitialization variable;
std::string input = "Position #toto=1;2;2";
std::string::const_iterator iter = input.begin();
std::string::const_iterator end = input.end();
// The phrase_parse call wil fill the structure "variable" with the good values if the syntax is correct.
// if the syntax is not correct, the method will return false.
// So if input = "Integer #toto= 6", variable.m_type == "Integer", variable.m_isStatic==true,
// variable.m_name=="toto" and variable.m_vale="6".
VariableInitializationParser<std::string::const_iterator> m_parser;
bool ok = phrase_parse(iter, end, m_parser, boost::spirit::ascii::space, variable);
if(!ok) return false;
std::cout << "Boost version : " << BOOST_VERSION << std::endl;
std::cout << "Type : " << variable.m_type << std::endl
<< "Is Static : " << variable.m_is_static << std::endl
<< "Name :" << variable.m_name << std::endl
<< "Value :" << variable.m_value << std::endl;
return 0;
}
Output of the following code is different in Boost 1.46.1 and in boost 1.53.
With boost 1.46.1 I have the following Output :
Boost version : 104601
Type : Position
Is Static : 1
Name :toto
Value :1;2;2
With boost 1.54 I have :
Boost version : 105400
Type :
Is Static : 1
Name :toto
Value :1;2;2
As you can see in boost 1.54 the Position output is not filled by the parser.
I read (maybe not carefully) the changelog of Fusion and Spirit, and I can't find why this happens.
Does someone have an explaination ?

No this is not a regression. Quite clearly you depended on undocumented (possibly undefined) behaviour.
Just change lit into string, or just slightly more compact:
var_type = qi::raw[lit("Position")|"String"|"Numeric"|"Integer"|"Trajectory"];
See it Live On Coliru
#include <iostream>
#include <boost/spirit/include/qi.hpp>
#include <boost/fusion/include/adapt_struct.hpp>
using namespace std;
using namespace boost::spirit;
struct VariableInitialization
{
std::string m_type;
bool m_is_static;
std::string m_name;
std::string m_value;
};
BOOST_FUSION_ADAPT_STRUCT(
VariableInitialization,
(std::string, m_type)
(bool, m_is_static)
(std::string, m_name)
(std::string, m_value)
)
template <typename Iterator>
struct VariableInitializationParser : qi::grammar<Iterator, VariableInitialization(), ascii::space_type> {
struct IsStatic_ : qi::symbols<char, bool> {
IsStatic_() {
add("#", false)("#", true);
}
} IsStatic;
VariableInitializationParser() :
VariableInitializationParser::base_type(start) {
using qi::lit;
using ascii::char_;
var_type = qi::raw[lit("Position")|"String"|"Numeric"|"Integer"|"Trajectory"];
identifier = (ascii::alpha|'_') >> *(ascii::alnum|'_');
var_value = qi::lexeme[*(char_)];
start = var_type >> IsStatic >> identifier >> '=' >> var_value;
}
qi::rule<Iterator, std::string(), ascii::space_type> var_type;
qi::rule<Iterator, std::string(), ascii::space_type> identifier;
qi::rule<Iterator, std::string(), ascii::space_type> var_value;
qi::rule<Iterator, VariableInitialization(), ascii::space_type> start;
};
int main()
{
VariableInitialization variable;
std::string input = "Position #toto=1;2;2";
std::string::const_iterator iter = input.begin();
std::string::const_iterator end = input.end();
VariableInitializationParser<std::string::const_iterator> m_parser;
bool ok = phrase_parse(iter, end, m_parser, boost::spirit::ascii::space, variable);
if(!ok) return false;
std::cout << "Boost version : " << BOOST_VERSION << "\n";
std::cout << "Type : " << variable.m_type << "\n"
<< "Is Static : " << variable.m_is_static << "\n"
<< "Name :" << variable.m_name << "\n"
<< "Value :" << variable.m_value << "\n";
}

Related

Q: Boost Program Options using std::filesystem::path as option fails when the given path contains spaces

I have a windows command line program using Boost.Program_Options. One option uses a std::filesystem::path variable.
namespace fs = std::filesystem;
namespace po = boost::program_options;
fs::path optionsFile;
po::options_description desc( "Options" );
desc.add_options()
("help,h", "Help screen")
("options,o", po::value<fs::path>( &optionsFile ), "file with options");
calling the program with -o c:\temp\options.txt or with -o "c:\temp\options.txt" works fine, but calling the program with -o "c:\temp\options 1.txt" fails with this error:
error: the argument( 'c:\temp\options 1.txt' ) for option '--options' is invalid
The content of argv in this case is:
argv[0] = Exepath
argv[1] = -o
argv[2] = c:\temp\options 1.txt
This is the full code:
#include <boost/program_options.hpp>
#include <filesystem>
#include <iostream>
namespace fs = std::filesystem;
namespace po = boost::program_options;
int wmain( int argc, wchar_t * argv[] )
{
try
{
fs::path optionsFile;
po::options_description desc( "Options" );
desc.add_options()
("help,h", "Help screen")
("options,o", po::value<fs::path>( &optionsFile ), "File containing the command and the arguments");
po::wcommand_line_parser parser{ argc, argv };
parser.options( desc ).allow_unregistered().style(
po::command_line_style::default_style |
po::command_line_style::allow_slash_for_short );
po::wparsed_options parsed_options = parser.run();
po::variables_map vm;
store( parsed_options, vm );
notify( vm );
if( vm.count( "help" ) )
{
std::cout << desc << '\n';
return 0;
}
std::cout << "optionsFile = " << optionsFile << "\n";
}
catch( const std::exception & e )
{
std::cerr << "error: " << e.what() << "\n";
return 1;
}
return 0;
}
How can I handle paths containing whitespace correctly? Is that even possible using std::filesystem::path or do I have to use std::wstring?
Indeed I could reproduce this. Replacing fs::path with std::string fixed it.
Here's a side-by-side reproducer:
Live On Coliru
#include <boost/program_options.hpp>
#include <filesystem>
#include <iostream>
namespace po = boost::program_options;
template <typename Path> static constexpr auto Type = "[unknown]";
template <> constexpr auto Type<std::string> = "std::string";
template <> constexpr auto Type<std::filesystem::path> = "fs::path";
template <typename Path>
bool do_test(int argc, char const* argv[]) try {
Path optionsFile;
po::options_description desc("Options");
desc.add_options() //
("help,h", "Help screen") //
("options,o", po::value<Path>(&optionsFile),
"File containing the command and the arguments");
po::command_line_parser parser{argc, argv};
parser.options(desc).allow_unregistered().style(
po::command_line_style::default_style |
po::command_line_style::allow_slash_for_short);
auto parsed_options = parser.run();
po::variables_map vm;
store(parsed_options, vm);
notify(vm);
if (vm.count("help")) {
std::cout << desc << '\n';
return true;
}
std::cout << "Using " << Type<Path> << "\toptionsFile = " << optionsFile << "\n";
return true;
} catch (const std::exception& e) {
std::cout << "Using " << Type<Path> << "\terror: " << e.what() << "\n";
return false;
}
int main() {
for (auto args : {
std::vector{"Exepath", "-o", "c:\\temp\\options1.txt"},
std::vector{"Exepath", "-o", "c:\\temp\\options 1.txt"},
})
{
std::cout << "\n -- Input: ";
for (auto& arg : args) {
std::cout << " " << std::quoted(arg);
}
std::cout << "\n";
int argc = args.size();
args.push_back(nullptr);
do_test<std::string>(argc, args.data());
do_test<std::filesystem::path>(argc, args.data());
}
}
Prints
-- Input: "Exepath" "-o" "c:\\temp\\options1.txt"
Using std::string optionsFile = c:\temp\options1.txt
Using fs::path optionsFile = "c:\\temp\\options1.txt"
-- Input: "Exepath" "-o" "c:\\temp\\options 1.txt"
Using std::string optionsFile = c:\temp\options 1.txt
Using fs::path error: the argument ('c:\temp\options 1.txt') for option '--options' is invalid
The reason most likely is that extraction from the command line argument defaults to using operator>> on a stringstream¹. If that has skipws set (as all C++ istreams do by default), then whitespace stops the "parse" and the argument is rejected because it is not fully consumed.
However, modifying the code to include a validate overload that fires for paths, adding std::noskipws didn't help!
template <class CharT>
void validate(boost::any& v, std::vector<std::basic_string<CharT>> const& s,
std::filesystem::path* p, int)
{
assert(s.size() == 1);
std::basic_stringstream<CharT> ss;
for (auto& el : s)
ss << el;
path converted;
ss >> std::noskipws >> converted;
if (!ss.eof())
throw std::runtime_error("Invalid path format");
v = std::move(converted);
}
Apparently, operator>> for fs::path doesn't obey noskipws. A look at the docs confirms:
Performs stream input or output on the path p. std::quoted is used so that spaces do not cause truncation when later read by stream input operator.
This gives us the workaround:
Workaround
template <class CharT>
void validate(boost::any& v, std::vector<std::basic_string<CharT>> const& s,
std::filesystem::path* p, int)
{
assert(s.size() == 1);
std::basic_stringstream<CharT> ss;
for (auto& el : s)
ss << std::quoted(el);
path converted;
ss >> std::noskipws >> converted;
if (ss.peek(); !ss.eof())
throw std::runtime_error("excess path characters");
v = std::move(converted);
}
Here we balance the std::quoted quoting/escaping as required.
Live Demo
Proof Of Concept:
Live On Coliru
#include <boost/program_options.hpp>
#include <filesystem>
#include <iostream>
namespace std::filesystem {
template <class CharT>
void validate(boost::any& v, std::vector<std::basic_string<CharT>> const& s,
std::filesystem::path* p, int)
{
assert(s.size() == 1);
std::basic_stringstream<CharT> ss;
for (auto& el : s)
ss << std::quoted(el);
path converted;
ss >> std::noskipws >> converted;
if (ss.peek(); !ss.eof())
throw std::runtime_error("excess path characters");
v = std::move(converted);
}
}
namespace po = boost::program_options;
template <typename Path> static constexpr auto Type = "[unknown]";
template <> constexpr auto Type<std::string> = "std::string";
template <> constexpr auto Type<std::filesystem::path> = "fs::path";
template <typename Path>
bool do_test(int argc, char const* argv[]) try {
Path optionsFile;
po::options_description desc("Options");
desc.add_options() //
("help,h", "Help screen") //
("options,o", po::value<Path>(&optionsFile),
"File containing the command and the arguments");
po::command_line_parser parser{argc, argv};
parser.options(desc).allow_unregistered().style(
po::command_line_style::default_style |
po::command_line_style::allow_slash_for_short);
auto parsed_options = parser.run();
po::variables_map vm;
store(parsed_options, vm);
notify(vm);
if (vm.count("help")) {
std::cout << desc << '\n';
return true;
}
std::cout << "Using " << Type<Path> << "\toptionsFile = " << optionsFile << "\n";
return true;
} catch (const std::exception& e) {
std::cout << "Using " << Type<Path> << "\terror: " << e.what() << "\n";
return false;
}
int main() {
for (auto args : {
std::vector{"Exepath", "-o", "c:\\temp\\options1.txt"},
std::vector{"Exepath", "-o", "c:\\temp\\options 1.txt"},
})
{
std::cout << "\n -- Input: ";
for (auto& arg : args) {
std::cout << " " << std::quoted(arg);
}
std::cout << "\n";
int argc = args.size();
args.push_back(nullptr);
do_test<std::string>(argc, args.data());
do_test<std::filesystem::path>(argc, args.data());
}
}
Now prints
-- Input: "Exepath" "-o" "c:\\temp\\options1.txt"
Using std::string optionsFile = c:\temp\options1.txt
Using fs::path optionsFile = "c:\\temp\\options1.txt"
-- Input: "Exepath" "-o" "c:\\temp\\options 1.txt"
Using std::string optionsFile = c:\temp\options 1.txt
Using fs::path optionsFile = "c:\\temp\\options 1.txt"
¹ this actually happens inside boost::lexical_cast which comes from Boost Conversion

How to define the AST for this boost spirit rule?

I have a more complex rule, but this one will suffice for this question (I hope). Consider the rule:
result = double_ >> *(char_ > int_);
where result is declared in terms of a struct result in namespace ast:
qi::rule<Iterator, ast::result(), qi::space_type> result;
Then how does ast::result have to look like?
According to the boost::spirit docs (http://www.boost.org/doc/libs/develop/libs/spirit/doc/html/spirit/abstracts/attributes/compound_attributes.html), the attribute of
char_ > int_ is tuple<char, int> or std::pair<char, int>.
So, I tried:
namespace ast
{
using second_type = std::vector<std::pair<char, int>>;
struct result
{
double first;
second_type second;
};
} // namespace ast
in addition to
BOOST_FUSION_ADAPT_STRUCT(
ast::result,
(double, first),
(ast::second_type, second)
)
But this gives me the compile error:
error: no matching function for call to 'std::pair<char, int>::pair(const char&)'
This rule is simple, creating the AST struct that the result will be stored in should be simple too... but how?
Here is a complete test program with my attempt:
#include <boost/config/warning_disable.hpp>
#include <boost/spirit/include/qi.hpp>
#include <boost/fusion/include/adapt_struct.hpp>
#include <iostream>
#include <string>
#include <vector>
namespace ast
{
using second_type = std::vector<std::pair<char, int>>;
struct result
{
double first;
second_type second;
friend std::ostream& operator<<(std::ostream& os, result const& result);
};
std::ostream& operator<<(std::ostream& os, second_type::value_type val)
{
return os << val.first << ' ' << val.second;
}
std::ostream& operator<<(std::ostream& os, result const& result)
{
os << result.first;
for (auto& i : result.second)
os << ' ' << i;
return os;
}
} // namespace ast
BOOST_FUSION_ADAPT_STRUCT(
ast::result,
(double, first),
(ast::second_type, second)
)
namespace client
{
namespace qi = boost::spirit::qi;
namespace ascii = boost::spirit::ascii;
template <typename Iterator>
class test_grammar : public qi::grammar<Iterator, ast::result(), qi::space_type>
{
private:
qi::rule<Iterator, ast::result(), qi::space_type> result;
public:
test_grammar() : test_grammar::base_type(result, "result_grammar")
{
using qi::double_;
using qi::char_;
using qi::int_;
result = double_ >> *(char_ > int_);
}
};
} // namespace client
int main()
{
std::string const input{"3.4 a 5 b 6 c 7"};
using iterator_type = std::string::const_iterator;
using test_grammar = client::test_grammar<iterator_type>;
namespace qi = boost::spirit::qi;
test_grammar program;
iterator_type iter{input.begin()};
iterator_type const end{input.end()};
ast::result out;
bool r = qi::phrase_parse(iter, end, program, qi::space, out);
if (!r || iter != end)
{
std::cerr << "Parsing failed." << std::endl;
return 1;
}
std::cout << "Parsed: " << out << std::endl;
}
SirGuy changed the AST to suit the default synthesized attributes. At the cost of, indeed complicating the AST.
However, you could leverage attribute compatibity rules by adapting std::pair. In fact, that is as simple as including 1 header:
#include <boost/fusion/include/std_pair.hpp>
Then, everything compiles without change, printing:
Parsed: 3.4 a 5 b 6 c 7
I made the following changes:
#include <boost/fusion/tuple.hpp>
using second_type = std::vector<boost::fusion::tuple<char, int>>;
std::ostream& operator<<(std::ostream& os, second_type::value_type val)
{
return os << boost::fusion::get<0>(val) << ' ' << boost::fusion::get<1>(val);
}
and the result compiled for me. There are most certainly other solutions available too.

Use of unicode predefined character classes in Boost Spirit

I am trying to use the letter character class from unicode i.e. \p{L} with Boost Spirit but I have no luck so far. Below is an example where I am trying to use (on line 30) the \p{L} character class. When I replace line 30 with line 29 it works but that is not the intended use as I need any letter from Unicode in my example.
My use case is for UTF8 only. At the end of they day what I am trying to do here is substract a unicode range from all unicode letters when using boost-spirit lexer.
PS
Of course, my example is trimmed down and may not make a lot of sense as a use case but I hope you get the idea.
#include <boost/config/warning_disable.hpp>
#include <boost/spirit/include/lex_lexertl.hpp>
#include <boost/fusion/include/std_pair.hpp>
#include <iostream>
#include <fstream>
#include <chrono>
#include <vector>
using namespace boost;
using namespace boost::spirit;
using namespace std;
using namespace std::chrono;
std::vector<pair<string, string> > getTokenMacros() {
std::vector<pair<string, string> > tokenDefinitionsVector;
tokenDefinitionsVector.emplace_back("JAPANESE_HIRAGANA", "[\u3041-\u3096]");
tokenDefinitionsVector.emplace_back("JAPANESE_HIRAGANA1",
"[\u3099-\u309E]");
tokenDefinitionsVector.emplace_back("ASIAN_NWS", "{JAPANESE_HIRAGANA}|"
"{JAPANESE_HIRAGANA1}");
tokenDefinitionsVector.emplace_back("ASIAN_NWS_WORD", "{ASIAN_NWS}*");
//tokenDefinitionsVector.emplace_back("NON_ASIAN_LETTER", "[A-Za-z0-9]");
tokenDefinitionsVector.emplace_back("NON_ASIAN_LETTER", "[\\p{L}-[{ASIAN_NWS}]]");
tokenDefinitionsVector.emplace_back("WORD", "{NON_ASIAN_LETTER}+");
tokenDefinitionsVector.emplace_back("ANY", ".");
return tokenDefinitionsVector;
}
;
struct distance_func {
template<typename Iterator1, typename Iterator2>
struct result: boost::iterator_difference<Iterator1> {
};
template<typename Iterator1, typename Iterator2>
typename result<Iterator1, Iterator2>::type operator()(Iterator1& begin,
Iterator2& end) const {
return distance(begin, end);
}
};
boost::phoenix::function<distance_func> const distance_fctor = distance_func();
template<typename Lexer>
struct word_count_tokens: lex::lexer<Lexer> {
word_count_tokens() :
asianNwsWord("{ASIAN_NWS_WORD}", lex::min_token_id + 110), word(
"{WORD}", lex::min_token_id + 170), any("{ANY}",
lex::min_token_id + 3000) {
using lex::_start;
using lex::_end;
using boost::phoenix::ref;
std::vector<pair<string, string> > tokenMacros(getTokenMacros());
for (auto start = tokenMacros.begin(), end = tokenMacros.end();
start != end; start++) {
this->self.add_pattern(start->first, start->second);
}
this->self = asianNwsWord | word | any;
}
lex::token_def<> asianNwsWord, word, any;
};
int main(int argc, char* argv[]) {
typedef lex::lexertl::token<string::iterator> token_type;
typedef lex::lexertl::actor_lexer<token_type> lexer_type;
word_count_tokens<lexer_type> word_count_lexer;
// read in the file int memory
ifstream sampleFile("/home/dan/Documents/wikiSample.txt");
string str = "abc efg ぁあ";
string::iterator first = str.begin();
string::iterator last = str.end();
lexer_type::iterator_type iter = word_count_lexer.begin(first, last);
lexer_type::iterator_type end = word_count_lexer.end();
typedef boost::iterator_range<string::iterator> iterator_range;
vector<iterator_range> parsed_tokens;
while (iter != end && token_is_valid(*iter)) {
cout << (iter->id() - lex::min_token_id) << " " << iter->value()
<< endl;
const iterator_range range = get<iterator_range>(iter->value());
parsed_tokens.push_back(range);
++iter;
}
if (iter != end) {
string rest(first, last);
cout << endl << "!!!!!!!!!" << endl << "Lexical analysis failed\n"
<< "stopped at: \"" << rest << "\"" << endl;
cout << "#" << (int) rest.at(0) << "#" << endl;
}
return 0;
}

How would I implement a forth-style reverse-polish notation parser in boost spirit?

I'm trying to implement a parser for an old forth-based grammar where most of the functions take the form of: "num" "num" "command" where command is a string of some kind.
For example:
0 1 HSFF
41 SENSOR ON
1 12.0 BH 4 LNON
As you can see, the grammar is [mostly] reverse polish notation, with some string of arguments preceding the command. The grammar is pseudo white-space dependent, in that:
0 1 HSFF 41 SENSOR ON
Is as valid as:
0 1 HSFF
41 SENSOR ON
(In other words '\n' is treated just as a space)
Extra whitespace is also skipped, so:
0 1 HSFF 41 SENSOR ON
Is 2 valid commands with a lot of unnecessary whitespace.
All of this seemed simple enough, so I started chugging away at implementing the grammar. Of course, things are never as simple as they seem, and I found that my parser fails on the very first character (in this case an int). So, boiling things down, I tried implementing a single rule:
namespace qi = boost::spirit::qi;
namespace ascii = boost::spirit::ascii;
qi::rule<Iterator> Cmd_TARGETSENSPAIRCMD =
qi::int_ >> (lit("TARGET") | lit("SENSOR") | lit("PAIR") )
>> (lit("ON") | lit("OFF") | lit("ERASE") );
std::string in("0 TARGET ERASE\n");
Iterator = in.begin();
bool success = qi::parse(in.begin(), in.end(), Cmd_TARGETSENSPAIRCMD, ascii::space);
This code block always returns false, indicating that parsing has failed.
As you can see, the rule is that an int must be followed by two literals, in this case indicating whether the command is for a target, sensor, or pair, identified by the int, to be turned on, off, or erased.
If I look at the iterator to see where the parsing has stopped, it shows that it has failed immediately on the int. So I changed the rule to simply be +qi::int_, which succeeds in parsing the int, but fails on the literals. Shortening the rule to simply qi::int_ >> lit("TARGET") also fails.
I think the problem may be in the whitespace skipper I'm using, but I have been unable to determine what I'm doing wrong.
Is there a way to tell spirit that all tokens are separated by whitespace, with the exception of quoted strings (which turn into labels in my grammar)?
I have phantasized a little for you.
The first step I usually take is to come up with an AST model:
namespace Ast
{
enum Command { NO_CMD, TARGET, SENSOR, PAIR };
enum Modifier { NO_MODIFIER, ON, OFF, ERASE };
struct ModifiedCommand
{
Command cmd = NO_CMD;
Modifier mod = NO_MODIFIER;
};
struct OtherCommand
{
std::string token;
OtherCommand(std::string token = "") : token(std::move(token))
{ }
};
typedef boost::variant<int, double> Operand;
typedef boost::variant<Operand, ModifiedCommand, OtherCommand> RpnMachineInstruction;
typedef std::vector<RpnMachineInstruction> RpnMachineProgram;
}
As you can see I intend to distinguish integers and double for operand values, and I treat any "other" commands (like "HSSF") that wasn't actively described in your grammar as free-form tokens (uppercase alphabetical).
Now, we map the rule definitions onto this:
RpnGrammar() : RpnGrammar::base_type(_start)
{
_start = *_instruction;
_instruction = _operand | _mod_command | _other_command;
_operand = _strict_double | qi::int_;
_mod_command = _command >> _modifier;
_other_command = qi::as_string [ +qi::char_("A-Z") ];
// helpers
_command.add("TARGET", Ast::TARGET)("SENSOR", Ast::SENSOR)("PAIR", Ast::PAIR);
_modifier.add("ON", Ast::ON)("OFF", Ast::OFF)("ERASE", Ast::ERASE);
}
The grammar parses the result into a list of instructions (Ast::RpnMachineProgram), where each instruction is either an operand or an operation (a command with modifier, or any other free-form command like "HSSF"). Here are the rule declarations:
qi::rule<It, Ast::RpnMachineProgram(), Skipper> _start;
qi::rule<It, Ast::RpnMachineInstruction(), Skipper> _instruction;
qi::rule<It, Ast::ModifiedCommand(), Skipper> _mod_command;
qi::rule<It, Ast::Operand(), Skipper> _operand;
// note: omitting the Skipper has the same effect as wrapping with `qi::lexeme`
qi::rule<It, Ast::OtherCommand()> _other_command;
qi::real_parser<double, boost::spirit::qi::strict_real_policies<double> > _strict_double;
qi::symbols<char, Ast::Command> _command;
qi::symbols<char, Ast::Modifier> _modifier;
You can see it parse the sample from the question:
Parse succeeded, 10 stack instructions
int:0 int:1 'HSFF'
int:41 SENSOR [ON]
int:1 double:12 'BH'
int:4 'LNON'
The output is created with a sample visitor that you could use as inspiration for an interpreter/executor.
See it Live On Coliru
Full Listing
#include <boost/fusion/adapted/struct.hpp>
#include <boost/spirit/include/qi.hpp>
#include <fstream>
namespace qi = boost::spirit::qi;
namespace Ast
{
enum Command { NO_CMD, TARGET, SENSOR, PAIR };
enum Modifier { NO_MODIFIER, ON, OFF, ERASE };
struct ModifiedCommand
{
Command cmd = NO_CMD;
Modifier mod = NO_MODIFIER;
};
struct OtherCommand
{
std::string token;
OtherCommand(std::string token = "") : token(std::move(token))
{ }
};
typedef boost::variant<int, double> Operand;
typedef boost::variant<Operand, ModifiedCommand, OtherCommand> RpnMachineInstruction;
typedef std::vector<RpnMachineInstruction> RpnMachineProgram;
// for printing, you can adapt this to execute the stack instead
struct Print : boost::static_visitor<std::ostream&>
{
Print(std::ostream& os) : os(os) {}
std::ostream& os;
std::ostream& operator()(Ast::Command cmd) const {
switch(cmd) {
case TARGET: return os << "TARGET" << " ";
case SENSOR: return os << "SENSOR" << " ";
case PAIR: return os << "PAIR" << " ";
case NO_CMD: return os << "NO_CMD" << " ";
default: return os << "#INVALID_COMMAND#" << " ";
}
}
std::ostream& operator()(Ast::Modifier mod) const {
switch(mod) {
case ON: return os << "[ON]" << " ";
case OFF: return os << "[OFF]" << " ";
case ERASE: return os << "[ERASE]" << " ";
case NO_MODIFIER: return os << "[NO_MODIFIER]" << " ";
default: return os << "#INVALID_MODIFIER#" << " ";
}
}
std::ostream& operator()(double d) const { return os << "double:" << d << " "; }
std::ostream& operator()(int i) const { return os << "int:" << i << " "; }
std::ostream& operator()(Ast::OtherCommand const& cmd) const {
return os << "'" << cmd.token << "'\n";
}
std::ostream& operator()(Ast::ModifiedCommand const& cmd) const {
(*this)(cmd.cmd);
(*this)(cmd.mod);
return os << "\n";
}
template <typename... TVariant>
std::ostream& operator()(boost::variant<TVariant...> const& v) const {
return boost::apply_visitor(*this, v);
}
};
}
BOOST_FUSION_ADAPT_STRUCT(Ast::ModifiedCommand, (Ast::Command, cmd)(Ast::Modifier, mod))
template <typename It, typename Skipper = qi::space_type>
struct RpnGrammar : qi::grammar<It, Ast::RpnMachineProgram(), Skipper>
{
RpnGrammar() : RpnGrammar::base_type(_start)
{
_command.add("TARGET", Ast::TARGET)("SENSOR", Ast::SENSOR)("PAIR", Ast::PAIR);
_modifier.add("ON", Ast::ON)("OFF", Ast::OFF)("ERASE", Ast::ERASE);
_start = *_instruction;
_instruction = _operand | _mod_command | _other_command;
_operand = _strict_double | qi::int_;
_mod_command = _command >> _modifier;
_other_command = qi::as_string [ +qi::char_("A-Z") ];
}
private:
qi::rule<It, Ast::RpnMachineProgram(), Skipper> _start;
qi::rule<It, Ast::RpnMachineInstruction(), Skipper> _instruction;
qi::rule<It, Ast::ModifiedCommand(), Skipper> _mod_command;
qi::rule<It, Ast::Operand(), Skipper> _operand;
// note: omitting the Skipper has the same effect as wrapping with `qi::lexeme`
qi::rule<It, Ast::OtherCommand()> _other_command;
qi::real_parser<double, boost::spirit::qi::strict_real_policies<double> > _strict_double;
qi::symbols<char, Ast::Command> _command;
qi::symbols<char, Ast::Modifier> _modifier;
};
int main()
{
std::ifstream ifs("input.txt");
typedef boost::spirit::istream_iterator It;
ifs.unsetf(std::ios::skipws);
RpnGrammar<It> grammar;
It f(ifs), l;
Ast::RpnMachineProgram program;
bool ok = qi::phrase_parse(f, l, grammar, qi::space, program);
if (ok)
{
std::cout << "Parse succeeded, " << program.size() << " stack instructions\n";
std::for_each(
program.begin(),
program.end(),
Ast::Print(std::cout));
}
else
{
std::cout << "Parse failed\n";
}
if (f != l)
{
std::cout << "Remaining unparsed: '" << std::string(f,l) << "'\n";
}
}

boost::spirit::karma output of string in quotation marks

I am trying to escape a string in quotation marks using boost::spirit::karma. This works fine if it's just a string. However, for a string in a boost::variant in a std::vector, it does not. Just printing the string does work however, I do not quite understand why.
Line (1) works fine, but doesn't do what I want. Line (2) should do it, but doesn't.
#include <iostream>
#include <string>
#include <boost/variant.hpp>
#include <boost/spirit/include/karma.hpp>
namespace karma = boost::spirit::karma;
typedef std::vector<boost::variant<int, std::string>> ParameterList;
typedef boost::variant<int, std::string, ParameterList> Parameter;
main()
{
using karma::int_;
using boost::spirit::ascii::string;
using karma::eol;
using karma::lit;
std::string generated;
std::back_insert_iterator<std::string> sink(generated);
// (1)
karma::rule<std::back_insert_iterator<std::string>, ParameterList()> parameterListRule = (int_ | string) % lit(", "); // This works!
// (2)
//karma::rule<std::back_insert_iterator<std::string>, ParameterList()> parameterListRule = (int_ | (lit('"') << string << lit('"'))) % lit(", "); // This does not work
karma::rule<std::back_insert_iterator<std::string>, Parameter()> parameterRule = (int_ | (lit('"') << string << lit('"')) | parameterListRule) << eol; // This does work, even though it also escapes the string in a pair of quotation marks
karma::generate(sink, parameterRule, 1); // Works
karma::generate(sink, parameterRule, "foo"); // Works
karma::generate(sink, parameterRule, Parameter(ParameterList{1, "foo"})); // Only works using rule (1), not with (2)
std::cout << generated;
}
Edited In case recursion was not the goal, here's an edited version that solves the issue and the quote escaping: Live on Coliru (or just source here)
Hmm. It looks like you might have been after a recursive attribute/rule:
typedef boost::make_recursive_variant<int, std::string, std::vector<boost::recursive_variant_> >::type Parameter;
Just in that case, here's a simple approach to generating that:
gen = int_ | string | gen % ", ";
Now, your title suggests that strings containing double-quotes should escape these. I suggest
str = '"' << *('\\' << char_('"') | char_) << '"';
gen = int_ | str | gen % ", ";
Now the following test cases
for (Parameter p : Parameters {
1,
"foo",
Parameters { 1, "foo" },
Parameters { 1, "escape: \"foo\"", Parameters { "2", "bar" } }
})
{
std::cout << karma::format(gen, p) << '\n';
}
result in:
1
"foo"
1, "foo"
1, "escape: \"foo\"", "2", "bar"
If recursion is really a feature, you'd want to see the grouping of nested Parameter lists:
gen = int_ | str | '{' << gen % ", " << '}';
Now prints
1
"foo"
{1, "foo"}
{1, "escape: \"foo\"", {"2", "bar"}}
Full sample program:
#include <boost/variant.hpp>
#include <boost/spirit/include/karma.hpp>
namespace karma = boost::spirit::karma;
typedef boost::make_recursive_variant<int, std::string, std::vector<boost::recursive_variant_> >::type Parameter;
typedef std::vector<Parameter> Parameters;
int main()
{
typedef boost::spirit::ostream_iterator It;
karma::rule<It, Parameter()> gen;
karma::rule<It, std::string()> str;
str = '"' << *('\\' << karma::char_('"') | karma::char_) << '"';
gen = (karma::int_ | str | '{' << gen % ", " << '}');
for (Parameter p : Parameters {
1,
"foo",
Parameters { 1, "foo" },
Parameters { 1, "escape: \"foo\"", Parameters { "2", "bar" } }
})
{
std::cout << karma::format(gen, p) << '\n';
}
}
If you iterate your data types, you should iterate your rules.
#include <iostream>
#include <string>
#include <boost/variant.hpp>
#include <boost/spirit/include/karma.hpp>
namespace karma = boost::spirit::karma;
typedef boost::variant<int, std::string> Item;
typedef std::vector<Item> ParameterList;
typedef boost::variant<int, std::string, ParameterList> Parameter;
int main()
{
using karma::int_;
using boost::spirit::ascii::string;
using karma::eol;
using karma::lit;
std::string generated;
std::back_insert_iterator<std::string> sink(generated);
karma::rule<std::back_insert_iterator<std::string>, Item()> itemRule =
int_ | (lit('"') << string << lit('"'));
karma::rule<std::back_insert_iterator<std::string>, ParameterList()>
parameterListRule = itemRule % lit(", ");
karma::rule<std::back_insert_iterator<std::string>, Parameter()>
parameterRule = (int_ | (lit('"') << string << lit('"')) | parameterListRule) << eol;
karma::generate(sink, parameterRule, 1);
karma::generate(sink, parameterRule, "foo");
karma::generate(sink, parameterRule, Parameter(ParameterList {1, "foo"}));
std::cout << generated;
return 0;
}
#include <iostream>
#include <boost/spirit/include/karma.hpp>
#include <boost/spirit/include/karma_right_alignment.hpp>
using namespace boost;
void foo(char* buffer, uint32_t lhOid) {
boost::spirit::karma::generate(buffer, boost::spirit::right_align(20)[boost::spirit::karma::int_], lhOid);
*buffer = '\0';
}
int main() {
char arr[21];
foo(arr, 1234);
std::cout.write(arr, 21) << std::endl;
return 0;
}

Resources