header_utils
Loading...
Searching...
No Matches
parsing.h
1
4
5#pragma once
6
7#include "unicode.h"
8#include <variant>
9#include <optional>
10
11namespace ghassanpl::parsing
12{
20
23 [[nodiscard]] size_t find_line_number(std::string_view of_string, std::string_view in_document) noexcept;
24
27 [[nodiscard]] std::pair<size_t, size_t> find_line_and_column(std::string_view of_string, std::string_view in_document) noexcept;
28
30 [[nodiscard]] std::string_view consume_c_identifier(std::string_view& str);
32 [[nodiscard]] std::string_view consume_c_identifier_with(std::string_view& str, std::string_view additional_chars);
33
34#if __cpp_lib_to_chars || defined(DOXYGEN)
35 [[nodiscard]] std::pair<std::string_view, double> consume_c_float(std::string_view& str);
36 [[nodiscard]] std::pair<std::string_view, int64_t> consume_c_integer(std::string_view& str, int base);
37 [[nodiscard]] std::pair<std::string_view, uint64_t> consume_c_unsigned(std::string_view& str, int base );
38#endif
39 template <char DELIMITER>
40 [[nodiscard]] std::pair<std::string_view, std::string> consume_c_string(std::string_view& strv);
41
42
43 bool try_eat(std::string_view& str, std::string_view what);
44 bool try_eat(std::string_view& str, char what);
45 void eat(std::string_view& str, std::string_view what);
46 void eat(std::string_view& str, char what);
47 std::string_view try_eat_identifier(std::string_view& str);
48 std::string_view eat_identifier(std::string_view& str);
49 std::string_view try_eat_identifier_with(std::string_view& str, std::string_view additional_chars);
50 std::string_view eat_identifier_with(std::string_view& str, std::string_view additional_chars);
51 std::string_view eat_whitespace(std::string_view& str);
52 bool try_eat_line_comment(std::string_view& str, std::string_view comment_start);
53 bool try_eat_unsigned(std::string_view& str, uint64_t& result, int base);
54 std::optional<uint64_t> try_eat_unsigned(std::string_view& str, int base);
55 bool try_eat_integer(std::string_view& str, int64_t& result, int base);
56 std::optional<int64_t> try_eat_integer(std::string_view& str, int base);
57 bool try_eat_float(std::string_view& str, double& result);
58 std::optional<double> try_eat_float(std::string_view& str);
59 uint64_t eat_unsigned(std::string_view& str, int base);
60 int64_t eat_integer(std::string_view& str, int base);
62 double eat_float(std::string_view& str);
63 char32_t try_eat_utf8_codepoint(std::string_view& str);
64 char32_t eat_utf8_codepoint(std::string_view& str);
65
67 struct parse_error : std::runtime_error
68 {
69 std::string_view Where;
70
71 template <GHPL_FORMAT_TEMPLATE>
72 parse_error(std::string_view where, GHPL_FORMAT_ARGS)
73 : runtime_error(GHPL_FORMAT_CALL)
74 , Where(where)
75 {
76
77 }
78 };
79
81
83
84 using namespace string_ops;
85
86 // TODO: Use expecteds for results of these functions
87
88 [[nodiscard]] inline size_t find_line_number(std::string_view of_string, std::string_view in_document) noexcept
89 {
90 if (of_string.empty() || in_document.empty())
91 return 0;
92 const auto start_line = of_string.data();
93 const auto start_doc = in_document.data();
94 if (start_line < start_doc || start_line > start_doc + in_document.size())
95 return 0;
96 return std::count(start_doc, start_line, '\n') + 1;
97 }
98
99 [[nodiscard]] inline std::pair<size_t, size_t> find_line_and_column(std::string_view of_string, std::string_view in_document) noexcept
100 {
101 if (of_string.empty() || in_document.empty())
102 return {};
103 const auto start_line = of_string.data();
104 auto start_doc = in_document.data();
105 if (start_line < start_doc || start_line > start_doc + in_document.size())
106 return {};
107
108 size_t count = 0;
109 auto last_line = start_doc;
110
111 for (; start_line != start_doc; ++start_doc) {
112 if (*start_doc == '\n') {
113 ++count;
114 last_line = start_doc + 1;
115 }
116 }
117
118 return { count + 1, start_line - last_line + 1 };
119 }
120
121 [[nodiscard]] inline std::string_view consume_c_identifier(std::string_view& str)
122 {
123 if (str.empty() || !ascii::isidentstart(str[0]))
124 return {};
125
126 const auto start = str.begin();
127 str.remove_prefix(1);
128 trim_while(str, ascii::isident);
129 return make_sv(start, str.begin());
130 }
131
132 [[nodiscard]] inline std::string_view consume_c_identifier_with(std::string_view& str, std::string_view additional_chars)
133 {
134 if (str.empty() || !(ascii::isidentstart(str[0]) || string_contains(additional_chars, str[0])))
135 return {};
136
137 const auto start = str.begin();
138 str.remove_prefix(1);
139 trim_while(str, [additional_chars](char c) { return ascii::isident(c) || string_contains(additional_chars, c); });
140 return make_sv(start, str.begin());
141 }
142
143
144#if __cpp_lib_to_chars || defined(DOXYGEN)
145 [[nodiscard]] inline std::pair<std::string_view, double> consume_c_float(std::string_view& str)
146 {
147 if (str.empty() || !(ascii::isdigit(str[0]) || str[0] == '-'))
148 return {};
149
150 std::pair<std::string_view, double> result;
151
152 const auto from_chars_result = from_chars(str, result.second);
153 if (from_chars_result.ec != std::errc{})
154 return { {}, std::numeric_limits<double>::quiet_NaN() };
155
156 result.first = make_sv(str.data(), from_chars_result.ptr);
157 str.remove_prefix(result.first.size());
158 return result;
159 }
160
161 [[nodiscard]] inline std::pair<std::string_view, int64_t> consume_c_integer(std::string_view& str, int base = 10)
162 {
163 if (str.empty() || !(ascii::isdigit(str[0]) || str[0] == '-'))
164 return {};
165
166 std::pair<std::string_view, int64_t> result;
167
168 const auto from_chars_result = from_chars(str, result.second, base);
169 if (from_chars_result.ec != std::errc{})
170 return { {}, 0 };
171
172 result.first = make_sv(str.data(), from_chars_result.ptr);
173 str.remove_prefix(result.first.size());
174 return result;
175 }
176
177 [[nodiscard]] inline std::pair<std::string_view, uint64_t> consume_c_unsigned(std::string_view& str, int base = 10)
178 {
179 if (str.empty() || !ascii::isdigit(str[0]))
180 return {};
181
182 std::pair<std::string_view, uint64_t> result;
183
184 const auto from_chars_result = std::from_chars(str.data(), str.data() + str.size(), result.second, base);
185 if (from_chars_result.ec != std::errc{})
186 return { {}, 0 };
187
188 result.first = make_sv(str.data(), from_chars_result.ptr);
189 str.remove_prefix(result.first.size());
190 return result;
191 }
192
193 [[deprecated("WARNING: This function is incomplete")]]
194 [[nodiscard]] inline std::tuple<std::string_view, std::variant<double, uint64_t, int64_t>> consume_c_number(std::string_view& str)
195 {
196 if (str.empty())
197 return {};
198
199 if (auto first_char = str[0]; first_char == '-')
200 {
203 return {};
204 }
205 else if (consume(str, "0x"))
206 return consume_c_unsigned(str, 16);
207 else if (consume(str, "0b"))
208 return consume_c_unsigned(str, 1);
209 else if (consume(str, "0"))
210 return consume_c_unsigned(str, 8);
211 else if (ascii::isdigit(first_char))
212 {
213 if (auto result = consume_c_float(str); !result.first.empty())
214 return result;
215
216 if (auto result = consume_c_unsigned(str); !result.first.empty())
217 return result;
218
219 if (auto result = consume_c_integer(str); !result.first.empty())
220 return result;
221 }
222 return {};
223 }
224
225#endif
226
227 template <char DELIMITER = '\''>
228 [[nodiscard]] inline std::pair<std::string_view, std::string> consume_c_string(std::string_view& strv)
229 {
230 if (strv.empty() || strv[0] != DELIMITER)
231 throw std::runtime_error("C string must start with delimiter");
232
233 std::pair<std::string_view, std::string> result;
234
235 auto view = strv;
236 auto start = view.begin();
237 view.remove_prefix(1);
238 while (view[0] != DELIMITER)
239 {
240 auto cp = consume(view);
241 if (cp == '\\')
242 {
243 cp = consume(view);
244 if (view.empty())
245 throw std::runtime_error("unterminated C string");
246
247 switch (cp)
248 {
249 case 'n': result.second += '\n'; break;
250 case '"': result.second += '"'; break;
251 case '\'': result.second += '\''; break;
252 case '\\': result.second += '\\'; break;
253 case '/': result.second += '/'; break;
254 case 'b': result.second += '\b'; break;
255 case 'r': result.second += '\r'; break;
256 case 'f': result.second += '\f'; break;
257 case 't': result.second += '\t'; break;
258 case '0': result.second += '\0'; break;
259 case 'o':
260 {
261 auto num = consume_n(view, 3);
262 if (num.size() < 3 || view.empty()) return {};
263
264 auto parsed = consume_c_integer(num, 8);
265 if (parsed.first.empty() || !num.empty()) return {};
266
267 if (parsed.second > 255) return {};
268 result.second.push_back((char)parsed.second);
269 break;
270 }
271 case 'x':
272 {
273 auto num = consume_n(view, 2);
274 if (num.size() < 2 || view.empty()) return {};
275
276 auto parsed = consume_c_integer(num, 16);
277 if (parsed.first.empty() || !num.empty()) return {};
278
279 //append_utf8(result.second, (char32_t)parsed.second);
280 result.second += (char)parsed.second;
281 break;
282 }
283 case 'u':
284 {
285 auto num = consume_n(view, 4);
286 if (num.size() < 4 || view.empty()) return {};
287
288 auto parsed = consume_c_integer(num, 16);
289 if (parsed.first.empty() || !num.empty()) return {};
290
291 append_utf8(result.second, (char32_t)parsed.second);
292 break;
293 }
294 case 'U':
295 {
296 auto num = consume_n(view, 8);
297 if (num.size() < 8 || view.empty()) return {};
298
299 auto parsed = consume_c_integer(num, 16);
300 if (parsed.first.empty() || !num.empty()) return {};
301
302 append_utf8(result.second, (char32_t)parsed.second);
303 break;
304 }
305 default:
306 throw std::runtime_error("unknown escape character");
307 }
308 }
309 else
310 {
311 result.second += cp;
312 }
313
314 if (view.empty())
315 throw std::runtime_error("unterminated C string");
316 }
317
318 if (!consume(view, DELIMITER))
319 throw std::runtime_error("C string must end with delimiter");
320
321 result.first = make_sv(start, view.begin());
322 strv = view;
323 return result;
324 }
325
326 inline bool try_eat(std::string_view& str, std::string_view what)
327 {
328 string_ops::trim_whitespace_left(str);
329 if (!str.starts_with(what))
330 return false;
331 str.remove_prefix(what.size());
332 return true;
333 }
334
335 inline bool try_eat(std::string_view& str, char what)
336 {
337 string_ops::trim_whitespace_left(str);
338 if (!str.starts_with(what))
339 return false;
340 str.remove_prefix(1);
341 return true;
342 }
343
344 inline void eat(std::string_view& str, std::string_view what)
345 {
346 if (!try_eat(str, what))
347 throw parse_error(str, "expected '{}'", what);
348 }
349
350 inline void eat(std::string_view& str, char what)
351 {
352 if (!try_eat(str, what))
353 throw parse_error(str, "expected '{}'", what);
354 }
355
356 inline std::string_view try_eat_identifier(std::string_view& str)
357 {
358 string_ops::trim_whitespace_left(str);
359 return consume_c_identifier(str);
360 }
361
362 inline std::string_view eat_identifier(std::string_view& str)
363 {
364 const auto result = try_eat_identifier(str);
365 if (result.empty())
366 throw parse_error(str, "expected identifier");
367 return result;
368 }
369
370 inline std::string_view try_eat_identifier_with(std::string_view& str, std::string_view additional_chars)
371 {
372 string_ops::trim_whitespace_left(str);
373 return consume_c_identifier_with(str, additional_chars);
374 }
375
376 inline std::string_view eat_identifier_with(std::string_view& str, std::string_view additional_chars)
377 {
378 const auto result = try_eat_identifier_with(str, additional_chars);
379 if (result.empty())
380 throw parse_error(str, "expected identifier");
381 return result;
382 }
383
384 inline std::string_view eat_whitespace(std::string_view& str)
385 {
386 return string_ops::consume_while(str, string_ops::ascii::isspace);
387 }
388
389 inline bool try_eat_line_comment(std::string_view& str, std::string_view comment_start = "//")
390 {
391 string_ops::trim_whitespace_left(str);
392 if (!try_eat(str, comment_start))
393 return false;
394 std::ignore = string_ops::consume_until(str, '\n');
395 return true;
396 }
397
398 inline bool try_eat_unsigned(std::string_view& str, uint64_t& result, int base = 10)
399 {
400 string_ops::trim_whitespace_left(str);
401 auto [parsed, value] = consume_c_unsigned(str, base);
402 if (parsed.empty()) return false;
403 result = value;
404 return true;
405 }
406
407 inline std::optional<uint64_t> try_eat_unsigned(std::string_view& str, int base = 10)
408 {
409 if (uint64_t result = 0; try_eat_unsigned(str, result, base))
410 return result;
411 return std::nullopt;
412 }
413
414 inline bool try_eat_integer(std::string_view& str, int64_t& result, int base = 10)
415 {
416 string_ops::trim_whitespace_left(str);
417 auto [parsed, value] = consume_c_integer(str, base);
418 if (parsed.empty()) return false;
419 result = value;
420 return true;
421 }
422
423 inline std::optional<int64_t> try_eat_integer(std::string_view& str, int base = 10)
424 {
425 if (int64_t result = 0; try_eat_integer(str, result, base))
426 return result;
427 return std::nullopt;
428 }
429
430 inline bool try_eat_float(std::string_view& str, double& result)
431 {
432 string_ops::trim_whitespace_left(str);
433 auto [parsed, value] = consume_c_float(str);
434 if (parsed.empty()) return false;
435 result = value;
436 return true;
437 }
438
439 inline std::optional<double> try_eat_float(std::string_view& str)
440 {
441 if (double result = 0; try_eat_float(str, result))
442 return result;
443 return std::nullopt;
444 }
445
446 inline uint64_t eat_unsigned(std::string_view& str, int base = 10)
447 {
448 uint64_t result{};
449 if (!try_eat_unsigned(str, result, base))
450 throw parse_error(str, "expected unsigned integer of base {}", base);
451 return result;
452 }
453
454 inline int64_t eat_integer(std::string_view& str, int base = 10)
455 {
456 int64_t result{};
457 if (!try_eat_integer(str, result, base))
458 throw parse_error(str, "expected integer of base {}", base);
459 return result;
460 }
461
462 inline double eat_float(std::string_view& str)
463 {
464 double result{};
465 if (!try_eat_float(str, result))
466 throw parse_error(str, "expected floating-point number");
467 return result;
468 }
469
470 inline char32_t try_eat_utf8_codepoint(std::string_view& str)
471 {
472 return string_ops::consume_utf8(str);
473 }
474
475 inline char32_t eat_utf8_codepoint(std::string_view& str)
476 {
477 if (const auto cp = string_ops::consume_utf8(str))
478 return cp;
479 throw parse_error(str, "expected UTF-8 codepoint");
480 }
481
482#if 0
483 namespace decade
484 {
485 template <GHPL_FORMAT_TEMPLATE>
486 [[nodiscard]] inline parsing::parse_error parse_error(token_range const& range, GHPL_FORMAT_ARGS) { return parsing::parse_error(to_string_view(range), GHPL_FORMAT_FORWARD); }
487 template <GHPL_FORMAT_TEMPLATE>
488 [[nodiscard]] inline parsing::parse_error parse_error(token_it const& it, GHPL_FORMAT_ARGS) { return parsing::parse_error(it->range, GHPL_FORMAT_FORWARD); }
489
490 struct term
491 {
492 virtual ~term() noexcept = default;
493 token_range source_range;
494
495 term(token_it it) : source_range(it, std::next(it)) {}
496 term(token_range range) : source_range(range) {}
497 };
498
499 struct function_call_expression : public term
500 {
501 std::vector<std::unique_ptr<term>> arguments;
502 std::string name;
503 static std::string make_name(std::span<token_it const> name_parts, bool infix)
504 {
505 std::string name;
506 if (infix)
507 name += ':';
508 for (token_it it : name_parts)
509 {
510 name += it->range;
511 name += ':';
512 }
513 return name;
514 }
515 function_call_expression(token_range range, std::span<token_it const> name_, std::vector<std::unique_ptr<term>> arguments_, bool infix)
516 : term(range)
517 , name(make_name(name_, infix))
518 , arguments(std::move(arguments_))
519 {
520 }
521 };
522
523 struct identifier_expression : public term
524 {
525 std::string identifier;
526
527 identifier_expression(token_it it) : term(it), identifier(it->range) {}
528 };
529
530 struct literal_expression : public term
531 {
532 token literal;
533
534 literal_expression(token_it it) : term(it), literal(*it) {}
535 };
536
537 inline std::unique_ptr<term> parse_expression(token_range& tokens)
538 {
539 std::unique_ptr<term> result;
540
541 auto start = tokens.begin();
542
543 std::vector<std::unique_ptr<term>> constituents;
544 while (tokens && tokens.front().type >= token::word)
545 {
546 if (tokens.front().type == token::word)
547 {
548 constituents.push_back(std::make_unique<identifier_expression>(tokens.begin()));
549 tokens.advance(1);
550 }
551 else if (tokens.front().type == token::number || tokens.front().type == token::string)
552 {
553 constituents.push_back(std::make_unique<literal_expression>(tokens.begin()));
554 tokens.advance(1);
555 }
556 else if (tokens.front().type == token::start_sub_expression)
557 {
558 tokens.advance(1);
559 constituents.push_back(parse_expression(tokens));
560
561 if (tokens.front().type != token::end_sub_expression)
562 throw parse_error(tokens.begin(), "unexpected end of line");
563 tokens.advance(1);
564 }
565 else
566 throw parse_error(tokens.begin(), "expected expression part");
567 }
568 if (constituents.empty())
569 throw parse_error(tokens.begin(), "empty expression encountered");
570
571 const auto constitutent_count = constituents.size();
572 if (constitutent_count == 1)
573 return std::move(constituents[0]);
574
575 const bool infix = (constitutent_count % 2) == 1;
576
577 std::vector<token_it> function_name;
578 std::vector<std::unique_ptr<term>> arguments;
579 if (infix)
580 arguments.push_back(std::exchange(constituents[0], {}));
582 for (size_t i = infix; i < constitutent_count; i += 2)
583 {
584 auto& function_identifier = constituents[i];
585 if (auto identifier = dynamic_cast<identifier_expression*>(function_identifier.get()))
586 {
587 function_name.push_back(identifier->source_range.begin());
588 arguments.push_back(std::exchange(constituents[i + 1], {}));
589 }
590 else
591 throw parse_error(function_identifier->source_range, "expected function name part");
592 }
593
594 return std::make_unique<function_call_expression>(std::ranges::subrange(start, tokens.begin()), std::move(function_name), std::move(arguments), infix);
595 }
596
597 inline std::unique_ptr<term> parse_expression(std::string_view& str)
598 {
599 const auto tokens = lex(str);
600 using tokenit = std::ranges::iterator_t<decltype(tokens)>;
601 token_range range = tokens;
602 return parse_expression(range);
603 }
604
605 }
606#endif
607}
ghassanpl::parsing::consume_c_identifier_with
std::string_view consume_c_identifier_with(std::string_view &str, std::string_view additional_chars)
Will consume a C-style identifier that may contain any additional characters from additional_chars: '...
Definition parsing.h:132
ghassanpl::parsing::eat_float
double eat_float(std::string_view &str)
Eats a C-style floating-point value.
Definition parsing.h:462
ghassanpl::parsing::find_line_and_column
std::pair< size_t, size_t > find_line_and_column(std::string_view of_string, std::string_view in_document) noexcept
Given a multiline in_document, will return the line and column numbers (starting at 1) of the start o...
Definition parsing.h:99
ghassanpl::parsing::consume_c_identifier
std::string_view consume_c_identifier(std::string_view &str)
Will consume a C-style identifier: [A-Za-z_][A-Za-z0-9_]*.
Definition parsing.h:121
ghassanpl::parsing::find_line_number
size_t find_line_number(std::string_view of_string, std::string_view in_document) noexcept
Given a multiline in_document, will return the line number (starting at 1) of the line which contains...
Definition parsing.h:88
ghassanpl::string_ops::string_contains
constexpr bool string_contains(std::string_view str, char c)
A pre-C++23 version of str.contains(c).
Definition string_ops.h:579
ghassanpl::string_ops::consume_until
std::string_view consume_until(std::string_view &str, FUNC &&pred)
Consumes characters from the beginning of str until one matches pred(str[0]), exclusive.
Definition string_ops.h:838
ghassanpl::string_ops::consume
char consume(std::string_view &str)
Consumes and returns the first character in the str, or \0 if no more characters.
Definition string_ops.h:702
ghassanpl::string_ops::from_chars
constexpr auto from_chars(std::string_view str, T &value, const int base=10) noexcept
A version of std::from_chars that takes a std::string_view as the first argument.
Definition string_ops.h:1631
ghassanpl::string_ops::consume_while
std::string_view consume_while(std::string_view &str, FUNC &&pred)
Consumes characters from the beginning of str while they match pred(str[0]).
Definition string_ops.h:805
ghassanpl::string_ops::consume_n
std::string_view consume_n(std::string_view &str, size_t n)
Consumes at most n characters from the beginning of str.
Definition string_ops.h:919
ghassanpl::string_ops::append_utf8
constexpr size_t append_utf8(string8 auto &buffer, char32_t cp)
Appends octets to buffer by encoding cp into UTF-8.
Definition unicode.h:889
ghassanpl::string_ops::consume_utf8
constexpr char32_t consume_utf8(string_view8 auto &str)
Consumes (see consume()) a UTF-8 codepoint from str.
Definition unicode.h:744
ghassanpl::string_ops
Definition string_ops.h:34
ghassanpl::parsing::parse_error
An exception type that will be thrown when eating/parsing fails.
Definition parsing.h:68
ghassanpl::parsing::parse_error::Where
std::string_view Where
Starts at (or near) the point where the parsing failed.
Definition parsing.h:69