diff --git a/furc/include/furc/ast/fwd.hpp b/furc/include/furc/ast/fwd.hpp index a99a64b..ef16e0c 100644 --- a/furc/include/furc/ast/fwd.hpp +++ b/furc/include/furc/ast/fwd.hpp @@ -159,7 +159,7 @@ struct body { }; /** - * @brief Alias for handle to body. + * @brief Alias for body result. * @see body */ using body_h = handle; diff --git a/furc/include/furc/front/lexer.hpp b/furc/include/furc/front/lexer.hpp index b976f5f..b05bb73 100644 --- a/furc/include/furc/front/lexer.hpp +++ b/furc/include/furc/front/lexer.hpp @@ -43,7 +43,7 @@ public: * * @return The token handle. */ - token_handle<> next_token(); + token_r next_token(); /** * @brief Checks whether the cursor is at the EOF. diff --git a/furc/include/furc/front/parser.hpp b/furc/include/furc/front/parser.hpp index 2f9f160..a751603 100644 --- a/furc/include/furc/front/parser.hpp +++ b/furc/include/furc/front/parser.hpp @@ -67,15 +67,17 @@ private: ast::body_h parse_body(); private: - token_handle<> next_token(); - const token_handle<>& peek_token(); - token_handle<> eat_token(token_t type); + static ast::node_handle error_handle(const token_r& result); private: - std::string m_filename; - std::string m_content; - lexer m_lexer; - furlang::arena m_arena; - std::vector> m_peekBuffer; + token_r next_token(); + const token_r& peek_token(); + token_r eat_token(token_t type); +private: + std::string m_filename; + std::string m_content; + lexer m_lexer; + furlang::arena m_arena; + std::vector m_peekBuffer; }; } // namespace front diff --git a/furc/include/furc/front/token.hpp b/furc/include/furc/front/token.hpp index 16663b4..8ebd1ea 100644 --- a/furc/include/furc/front/token.hpp +++ b/furc/include/furc/front/token.hpp @@ -1,9 +1,11 @@ #ifndef FURC_FRONT_TOKEN_HPP #define FURC_FRONT_TOKEN_HPP -#include "furc/handle.hpp" +#include "furc/diag.hpp" +#include "furlang/result.hpp" #include +#include #include #include @@ -173,7 +175,9 @@ using integer_token = std::uint64_t; /**< Integer token. */ * @brief Token. */ struct token { - token_t type = token_t::None; /**< Token type. */ + location location; + token_t type = token_t::None; /**< Token type. */ + /** * @brief Token's value. */ @@ -241,35 +245,39 @@ struct token { /** * @brief Construct a new null token. * + * @param location Token's location. * @param type Token's type. */ - token(token_t type) - : type(type) {} + token(struct location location, token_t type) + : location(location), type(type) {} /** * @brief Construct a new string token. * + * @param location Token's location. * @param type Token's type. * @param value String value. */ - token(token_t type, std::string_view value) - : type(type), value(value) {} + token(struct location location, token_t type, std::string_view value) + : location(location), type(type), value(value) {} /** * @brief Construct a new keyword token. * + * @param location Token's location. * @param keyword Keyword value. */ - token(keyword_token keyword) - : type(token_t::Keyword), value(keyword) {} + token(struct location location, keyword_token keyword) + : location(location), type(token_t::Keyword), value(keyword) {} /** * @brief Construct a new integer token. * + * @param location Token's location. * @param integer Integer value. */ - token(integer_token integer) - : type(token_t::Integer), value(integer) {} + token(struct location location, integer_token integer) + : location(location), type(token_t::Integer), value(integer) {} /** * @brief Returns pointer to this token's value. @@ -313,8 +321,35 @@ static inline std::ostream& operator<<(std::ostream& os, const token& token) { } } -template -using token_handle = handle; /**< Alias to handle of token. */ +/** + * @brief Token error type + */ +enum class token_error_t { + EndOfFile, /**< End of file */ + UnexpectedEof, /**< Unexpected end of file */ + UnexpectedCharacter, /**< Unexpected character */ + UnexpectedToken, /**< Unexpected character */ + IntegerOverflow, /**< Integer overflow */ +}; + +/** + * @brief Token error + * + * For token_r alias. + */ +struct token_error { + location location; /**< Error location. */ + token_error_t type; /**< Error type. */ + std::string message; /**< Error message. */ + + bool operator==(const token_error& rhs) const { + return location == rhs.location && type == rhs.type && message == rhs.message; + } + + bool operator!=(const token_error& rhs) const { return !this->operator==(rhs); } +}; + +using token_r = furlang::result; /**< Alias to a token result. */ } // namespace front } // namespace furc diff --git a/furc/include/furc/handle.hpp b/furc/include/furc/handle.hpp index 04ae96f..85cfb4e 100644 --- a/furc/include/furc/handle.hpp +++ b/furc/include/furc/handle.hpp @@ -171,7 +171,7 @@ public: return *this; } public: - location location() const { return m_data->location; } + location location() const { return m_data->location; } // NOLINT(bugprone-unchecked-optional-access) bool present() const { return m_data.has_value() && m_data->value != nullptr; } bool has_error() const { return m_data.has_value() && m_data->value == nullptr; } diff --git a/furc/src/front/lexer.cpp b/furc/src/front/lexer.cpp index b0bae2b..366a2c3 100644 --- a/furc/src/front/lexer.cpp +++ b/furc/src/front/lexer.cpp @@ -13,7 +13,7 @@ using namespace std::string_literals; lexer::lexer(std::string_view filename, std::string_view content) : m_filename(filename), m_content(content) {} -token_handle<> lexer::next_token() { +token_r lexer::next_token() { skip_spaces(); while (m_cursor + 2 <= m_content.size() && m_content[m_cursor] == '/') { if (m_content[m_cursor + 1] == '/') { @@ -35,7 +35,8 @@ token_handle<> lexer::next_token() { } if (m_cursor + 2 >= m_content.size()) { next(); - return { current_location(), "unexpected end of file before enclosing `*/`" }; + return token_r( + token_error{ current_location(), token_error_t::UnexpectedEof, "before enclosing `*/`" }); } m_cursor += 2; } else { @@ -51,12 +52,14 @@ token_handle<> lexer::next_token() { std::size_t begin = ++m_cursor; while (m_cursor < m_content.size() && m_content[m_cursor] != '"') ++m_cursor; - if (m_cursor >= m_content.size()) return { current_location(), "unexpected end of file before enclosing '\"'" }; + if (m_cursor >= m_content.size()) { + return token_r(token_error{ current_location(), token_error_t::UnexpectedEof, "before enclosing '\"'" }); + } ++m_cursor; return { location, token_t::String, m_content.substr(begin, m_cursor - begin - 1) }; } - case std::char_traits::eof(): return { location, token_t::None }; + case std::char_traits::eof(): return token_r(token_error{ current_location(), token_error_t::EndOfFile }); default: { if (std::isdigit(get()) != 0) { integer_token integer = 0; @@ -70,7 +73,9 @@ token_handle<> lexer::next_token() { if (integer > upperBound || integer == upperBound && (integer - upperBound + digit) > (max % 10)) { while (std::isdigit(get()) != 0) ++m_cursor; - return { location, "integer "s.append(m_content.substr(start, m_cursor - start)) + " is too big" }; + return token_r(token_error{ location, + token_error_t::IntegerOverflow, + std::string(m_content.substr(start, m_cursor - start)) }); } integer *= 10; integer += digit; @@ -150,7 +155,8 @@ token_handle<> lexer::next_token() { return { location, type }; } - return { location, "unexpected character '"s.append(m_content.substr(m_cursor, length)) + "'" }; + return token_r( + token_error{ location, token_error_t::UnexpectedCharacter, std::string(m_content.substr(m_cursor, 1)) }); } } } diff --git a/furc/src/front/parser.cpp b/furc/src/front/parser.cpp index 4859830..af6a6eb 100644 --- a/furc/src/front/parser.cpp +++ b/furc/src/front/parser.cpp @@ -33,7 +33,7 @@ parser::parser(std::string_view filename) ast::program_node_h parser::parse() & { auto program = m_arena.allocate_shared(); - while (peek_token().present() && peek_token()->type != token_t::None && !m_lexer.empty()) { + while (peek_token().has_value()) { program->push(std::move(parse_declaration())); } @@ -47,30 +47,30 @@ ast::declaration_node_h parser::parse_declaration() { auto first = next_token(); switch ((*first)->keyword) { case keyword_token::Func: { - token_handle<> name = eat_token(token_t::Identifier); - if (name.has_error()) return { name.location(), name.error() }; + auto name = eat_token(token_t::Identifier); + if (name.has_error()) return error_handle(name); auto tok = eat_token(token_t::LParen); - if (tok.has_error()) return tok; + if (tok.has_error()) return error_handle(tok); tok = eat_token(token_t::RParen); - if (tok.has_error()) return tok; + if (tok.has_error()) return error_handle(tok); const auto& peek = peek_token(); - if (peek.has_error()) return peek; + if (peek.has_error()) return error_handle(peek); switch (peek->type) { case token_t::LBrace: { ast::body_h body = parse_body(); if (body.has_error()) return body; - return ast::function_definition_node_h{ first.location(), m_arena, *name, std::move(body) }; + return ast::function_definition_node_h{ first->location, m_arena, *name, std::move(body) }; } case token_t::Semicolon: { m_peekBuffer.clear(); - return ast::function_declaration_node_h{ first.location(), m_arena, *name }; + return ast::function_declaration_node_h{ first->location, m_arena, *name }; } - default: return { tok.location(), "unexpected token "s + tok->type }; + default: return { tok->location, "unexpected token "s + tok->type }; } } - default: return { first.location(), "unexpected keyword "s + (*first)->keyword }; + default: return { first->location, "unexpected keyword "s + (*first)->keyword }; } } case token_t::None: @@ -85,15 +85,15 @@ ast::declaration_node_h parser::parse_declaration() { case token_t::Semicolon: case token_t::Colon: default: { - return { first.location(), "unexpected token "s + first->type }; + return { first->location, "unexpected token "s + first->type }; } } } ast::statement_node_h parser::parse_statement() { const auto& tok = peek_token(); - if (tok.has_error()) return tok; - auto location = tok.location(); + if (tok.has_error()) return error_handle(tok); + auto location = tok->location; switch (tok->type) { case token_t::Keyword: { switch (tok->value.keyword) { @@ -101,28 +101,28 @@ ast::statement_node_h parser::parse_statement() { auto tok = next_token(); if (peek_token()->type == token_t::Semicolon) { next_token(); - return ast::return_statement_node_h{ tok.location(), m_arena }; + return ast::return_statement_node_h{ tok->location, m_arena }; } auto value = parse_expression(); auto err = eat_token(token_t::Semicolon); - if (err.has_error()) return err; - return ast::return_statement_node_h{ tok.location(), m_arena, std::move(value) }; + if (err.has_error()) return error_handle(err); + return ast::return_statement_node_h{ tok->location, m_arena, std::move(value) }; } case keyword_token::If: { auto tok = next_token(); auto err = eat_token(token_t::LParen); - if (err.has_error()) return err; + if (err.has_error()) return error_handle(err); auto cond = parse_expression(); err = eat_token(token_t::RParen); - if (err.has_error()) return err; + if (err.has_error()) return error_handle(err); auto then = parse_statement(); if (then.has_error()) return then; - if (peek_token().present() && peek_token()->type == token_t::Keyword && + if (peek_token().has_value() && peek_token()->type == token_t::Keyword && peek_token()->value.keyword == keyword_token::Else) { next_token(); @@ -149,12 +149,12 @@ ast::statement_node_h parser::parse_statement() { auto expression = parse_expression(); if (expression.present()) { auto semi = eat_token(token_t::Semicolon); - if (semi.has_error()) return semi; + if (semi.has_error()) return error_handle(semi); return std::move(expression); } auto token = next_token(); - return { token.location(), "unexpected token "s + token->type + ", expected statement, declaration or expression" }; + return { token->location, "unexpected token "s + token->type + ", expected statement, declaration or expression" }; } ast::expression_node_h parser::parse_expression(std::uint32_t precedence) { @@ -166,20 +166,20 @@ ast::literal_node_h parser::parse_literal() { switch (tok->type) { case token_t::String: { auto tok = next_token(); - return ast::string_literal_node_h{ tok.location(), + return ast::string_literal_node_h{ tok->location, m_arena, - handle{ tok.location(), (*tok)->string } }; + handle{ tok->location, (*tok)->string } }; } case token_t::Integer: { auto tok = next_token(); - return ast::integer_literal_node_h{ tok.location(), + return ast::integer_literal_node_h{ tok->location, m_arena, - handle{ tok.location(), (*tok)->integer } }; + handle{ tok->location, (*tok)->integer } }; } default: break; } - return { tok.location(), "unexpected token "s + tok->type + ", expected literal" }; + return { tok->location, "unexpected token "s + tok->type + ", expected literal" }; } ast::expression_node_h parser::parse_expression_primary() { @@ -187,21 +187,21 @@ ast::expression_node_h parser::parse_expression_primary() { switch (tok->type) { case token_t::Identifier: { auto tok = next_token(); - return ast::var_read_expression_node_h{ tok.location(), + return ast::var_read_expression_node_h{ tok->location, m_arena, - handle{ tok.location(), (*tok)->string } }; + handle{ tok->location, (*tok)->string } }; } case token_t::LParen: { auto tok = next_token(); auto node = parse_expression(); auto err = eat_token(token_t::RParen); - if (err.has_error()) return err; + if (err.has_error()) return error_handle(err); return node; } default: { auto literal = parse_literal(); if (literal.present()) return std::move(literal); - return { tok.location(), "unexpected token"s + tok->type + ", expected expression or literal" }; + return { tok->location, "unexpected token"s + tok->type + ", expected expression or literal" }; } } } @@ -235,7 +235,7 @@ ast::expression_node_h parser::parse_expression_unary(std::uint32_t precedence) expression = parse_expression_unary(current.precedence + 1); } - result = { token.location(), m_arena, current.type, std::move(expression) }; + result = { token->location, m_arena, current.type, std::move(expression) }; } if (!result.present()) return parse_expression_primary(); @@ -319,7 +319,7 @@ ast::expression_node_h parser::parse_expression_rhs(ast::expression_node_h&& ini }; ast::expression_node_h lhs = std::move(init); - while (peek_token().present()) { + while (peek_token().has_value()) { auto it = s_rhsops.find(peek_token()->type); if (it == s_rhsops.end()) return lhs; @@ -346,17 +346,17 @@ ast::expression_node_h parser::parse_expression_rhs(ast::expression_node_h&& ini switch (current.type) { case rhsop_info_t::Unaryop: - lhs = ast::unaryop_expression_node_h{ opToken.location(), m_arena, current.unary, std::move(lhs) }; + lhs = ast::unaryop_expression_node_h{ opToken->location, m_arena, current.unary, std::move(lhs) }; break; case rhsop_info_t::Binop: - lhs = ast::binop_expression_node_h{ opToken.location(), + lhs = ast::binop_expression_node_h{ opToken->location, m_arena, current.binary, std::move(lhs), std::move(rhs) }; break; case rhsop_info_t::Assignment: - lhs = ast::var_assign_expression_node_h{ opToken.location(), + lhs = ast::var_assign_expression_node_h{ opToken->location, m_arena, current.assignment, std::move(lhs), @@ -370,44 +370,50 @@ ast::expression_node_h parser::parse_expression_rhs(ast::expression_node_h&& ini ast::body_h parser::parse_body() { ast::body body; - token_handle<> begin = eat_token(token_t::LBrace); - if (begin.has_error()) return begin; - body.begin = begin.location(); + auto begin = eat_token(token_t::LBrace); + if (begin.has_error()) return error_handle(begin); + body.begin = begin->location; while (!peek_token().has_error() && peek_token()->type != token_t::None && peek_token()->type != token_t::RBrace) { body.statements.push_back(parse_statement()); } - token_handle<> end = eat_token(token_t::RBrace); - if (end.has_error()) return end; - body.end = end.location(); + auto end = eat_token(token_t::RBrace); + if (end.has_error()) return error_handle(end); + body.end = end->location; - return { begin.location(), body }; + return { begin->location, body }; } -token_handle<> parser::next_token() { +ast::node_handle parser::error_handle(const token_r& result) { + return { result.error().location, std::string(result.error().message) }; +} + +token_r parser::next_token() { if (!m_peekBuffer.empty()) { - token_handle<> token = std::move(m_peekBuffer.back()); + auto token = std::move(m_peekBuffer.back()); m_peekBuffer.pop_back(); return token; } return m_lexer.next_token(); } -const token_handle<>& parser::peek_token() { +const token_r& parser::peek_token() { if (m_peekBuffer.empty()) { - token_handle<> token = m_lexer.next_token(); + auto token = m_lexer.next_token(); return m_peekBuffer.emplace_back(std::move(token)); } return m_peekBuffer.front(); } -token_handle<> parser::eat_token(token_t type) { - token_handle<> token = next_token(); - if (!token.present()) return token; +token_r parser::eat_token(token_t type) { + auto token = next_token(); + if (token.has_error()) return token; if (token->type != type) { - if (token->type == token_t::None) return { token.location(), "unexpected end of file, expected " + type }; - return { token.location(), "unexpected token "s + token->type + ", expected " + type }; + if (token->type == token_t::None) + return token_r(token_error{ token->location, token_error_t::UnexpectedToken, ", expected " + type }); + return token_r( + token_error{ token->location, token_error_t::UnexpectedToken, ""s + token->type + ", expected " + type }); } return token; } diff --git a/furc/test/lexer.cpp b/furc/test/lexer.cpp index 4c056d9..17b7c70 100644 --- a/furc/test/lexer.cpp +++ b/furc/test/lexer.cpp @@ -2,8 +2,6 @@ #include "furc/front/lexer.hpp" -#include "furlang/result.hpp" - #include "gtest/gtest.h" #include @@ -13,7 +11,6 @@ using namespace furc::front; using namespace std::string_view_literals; using namespace std::string_literals; -using token_r = furlang::result; using lexer_case = std::pair>; class LexerTestingFixture : public testing::TestWithParam {}; @@ -26,16 +23,16 @@ TEST_P(LexerTestingFixture, LexerTest) { while (it != expected.end()) { const auto& expected = *it++; - auto token = std::move(lexer.next_token()); - if (expected.has_error()) { - EXPECT_TRUE(token.has_error()); - EXPECT_EQ(token.error(), expected.error()); - } else { - EXPECT_EQ(token, expected.value()); - } + EXPECT_EQ(lexer.next_token(), expected); } - // EOF - EXPECT_EQ(lexer.next_token(), token{}); + + auto eof = std::move(lexer.next_token()); + ASSERT_TRUE(eof.has_error()); + ASSERT_EQ(eof.error().type, token_error_t::EndOfFile); +} + +furc::location loc(size_t col, size_t line) { + return furc::location{ "", line, col }; } INSTANTIATE_TEST_SUITE_P(EmptyTests, @@ -45,31 +42,36 @@ INSTANTIATE_TEST_SUITE_P(EmptyTests, INSTANTIATE_TEST_SUITE_P(Comments, LexerTestingFixture, testing::Values(lexer_case("(/** skibidi **/func{//)\n}", - { { token_t::LParen }, { keyword_token::Func }, { token_t::LBrace }, { token_t::RBrace } }))); + { { loc(0, 0), token_t::LParen }, + { loc(16, 0), keyword_token::Func }, + { loc(20, 0), token_t::LBrace }, + { loc(0, 1), token_t::RBrace } }))); INSTANTIATE_TEST_SUITE_P(Integers, LexerTestingFixture, - testing::Values(lexer_case("67 6\n7", { { 67 }, { 6 }, { 7 } }), - lexer_case("18446744073709551615 18446744073709551616", - { { 18446744073709551615ULL }, token_r(std::string("integer 18446744073709551616 is too big")) }))); + testing::Values(lexer_case("67 6\n7", { { loc(0, 0), 67 }, { loc(3, 0), 6 }, { loc(0, 1), 7 } }), + lexer_case("18446744073709551615\n18446744073709551616", + { { loc(0, 0), 18446744073709551615ULL }, + token_r( + token_error{ loc(0, 1), token_error_t::IntegerOverflow, std::string("18446744073709551616") }) }))); INSTANTIATE_TEST_SUITE_P(Tokens, LexerTestingFixture, testing::Values(lexer_case("()\n\t\t{\n}[\"shto-to\"]; :,.main return func", - { { token_t::LParen }, - { token_t::RParen }, - { token_t::LBrace }, - { token_t::RBrace }, - { token_t::LBracket }, - { token_t::String, "shto-to"sv }, - { token_t::RBracket }, - { token_t::Semicolon }, - { token_t::Colon }, - { token_t::Comma }, - { token_t::Dot }, - { token_t::Identifier, "main"sv }, - { keyword_token::Return }, - { keyword_token::Func } }))); + { { loc(0, 0), token_t::LParen }, + { loc(1, 0), token_t::RParen }, + { loc(2, 1), token_t::LBrace }, + { loc(0, 2), token_t::RBrace }, + { loc(1, 2), token_t::LBracket }, + { loc(2, 2), token_t::String, "shto-to"sv }, + { loc(10, 2), token_t::RBracket }, + { loc(11, 2), token_t::Semicolon }, + { loc(15, 2), token_t::Colon }, + { loc(16, 2), token_t::Comma }, + { loc(17, 2), token_t::Dot }, + { loc(18, 2), token_t::Identifier, "main"sv }, + { loc(23, 2), keyword_token::Return }, + { loc(30, 2), keyword_token::Func } }))); } // namespace diff --git a/furlang/include/furlang/result.hpp b/furlang/include/furlang/result.hpp index 126149d..e45ccac 100644 --- a/furlang/include/furlang/result.hpp +++ b/furlang/include/furlang/result.hpp @@ -205,7 +205,7 @@ public: * @param rhs Value to compare against. * @return true if the values are equal. */ - bool operator==(const value_type& rhs) const { return !m_error && !rhs.m_error && m_value.result == rhs; } + bool operator==(const value_type& rhs) const { return !m_error && m_value.result == rhs; } /** * @brief Compares a result with a value for inequality.