15 #ifndef VISIONTRANSFER_TOKENIZER_H 16 #define VISIONTRANSFER_TOKENIZER_H 27 Tokenizer(): _separators{
" ",
"\t"}, _comment_initiators{
"#"}, _strip_chars(
""), _collapse(
true), _quoting(
true) {}
28 Tokenizer& separators(
const std::vector<std::string>& seps) { _separators = seps;
return *
this; }
29 Tokenizer& strip_chars(
const std::string& chars) { _strip_chars = chars;
return *
this; }
30 Tokenizer& collapse(
bool coll) { _collapse = coll;
return *
this; }
31 Tokenizer& quoting(
bool quot) { _quoting = quot;
return *
this; }
32 #if VISIONTRANSFER_CPLUSPLUS_VERSION >= 201703L 33 std::vector<std::string> tokenize(
const std::string_view& inp) {
35 std::vector<std::string> tokenize(
const std::string& inp) {
37 std::vector<std::string> toks;
39 bool issep, iscomment;
40 char quotemode =
'\0';
41 for (
size_t i=0; i<inp.size(); ++i) {
42 if (quotemode ==
'\0') {
44 issep =
false; iscomment =
false;
45 for (
const auto& comm: _comment_initiators) {
46 if (inp.substr(i, comm.size())==comm) {
52 for (
const auto& sep: _separators) {
53 if (inp.substr(i, sep.size())==sep) {
63 std::string tmp = ss.str();
64 if (!_collapse || !tmp.empty()) {
68 }
else if (inp[i] ==
'"') {
70 }
else if (inp[i] ==
'\'') {
75 }
else if (quotemode ==
'\'') {
77 if (inp.substr(i, 2) ==
"\\\'") {
80 }
else if (inp.substr(i, 2) ==
"\\\\") {
83 }
else if (inp[i] ==
'\'') {
84 toks.push_back(ss.str());
90 }
else if (quotemode ==
'"') {
92 if (inp.substr(i, 2) ==
"\\\"") {
95 }
else if (inp.substr(i, 2) ==
"\\\\") {
98 }
else if (inp.substr(i, 2) ==
"\\n") {
101 }
else if (inp[i] ==
'"') {
102 toks.push_back(ss.str());
110 std::string tmp = ss.str();
111 if (!_collapse || !tmp.empty()) {
114 if (_strip_chars.size()) {
115 std::vector<std::string> toks2;
117 auto st = s.find_first_not_of(_strip_chars);
118 auto en = s.find_last_not_of(_strip_chars);
119 toks2.push_back((st==en)?std::string():std::string(s.substr(st, en+1-st)));
127 std::vector<std::string> _separators;
128 std::vector<std::string> _comment_initiators;
129 std::string _strip_chars;
130 bool _collapse, _quoting;