From cbb2efc187ade2e3d7c820d77450b1b1c8249971 Mon Sep 17 00:00:00 2001 From: CHatingPython Date: Mon, 25 May 2026 17:17:18 +0200 Subject: [PATCH] Improve AST nodes Signed-off-by: CHatingPython --- furc/include/furc/ast/declaration.hpp | 41 +++++----------- furc/include/furc/ast/expression.hpp | 26 ++++++++++ furc/include/furc/ast/literal.hpp | 65 +++++++++++++++++++++++++ furc/include/furc/ast/node.hpp | 18 +++++-- furc/include/furc/ast/program.hpp | 12 +++-- furc/include/furc/ast/statement.hpp | 36 +++----------- furc/include/furc/front/parser.hpp | 4 ++ furc/include/furc/front/token.hpp | 9 +++- furc/src/front/lexer.cpp | 9 ++++ furc/src/front/parser.cpp | 68 +++++++++++++++++++-------- furc/test/lexer.cpp | 3 +- furc/test/parser.cpp | 4 +- 12 files changed, 202 insertions(+), 93 deletions(-) create mode 100644 furc/include/furc/ast/expression.hpp create mode 100644 furc/include/furc/ast/literal.hpp diff --git a/furc/include/furc/ast/declaration.hpp b/furc/include/furc/ast/declaration.hpp index 1f4b780..9cdcdb4 100644 --- a/furc/include/furc/ast/declaration.hpp +++ b/furc/include/furc/ast/declaration.hpp @@ -17,24 +17,13 @@ enum class declaration_node_t { Variable, }; -static inline std::ostream& operator<<(std::ostream& os, declaration_node_t type) { - switch (type) { - case declaration_node_t::FunctionDeclaration: return os << "function declaration"; - case declaration_node_t::FunctionDefinition: return os << "function definition"; - case declaration_node_t::Variable: return os << "variable"; - } -} - -class declaration_node : public abstract_node { +class declaration_node : public statement_node { public: - virtual declaration_node_t type() const = 0; -public: - virtual std::ostream& print(std::ostream& os) const = 0; + node_t category() const override { return node_t::Declaration; } - friend std::ostream& operator<<(std::ostream& os, const declaration_node& node) { - os << node.type(); - return node.print(os); - } + statement_node_t statement_type() const override { return statement_node_t::Declaration; } + + virtual declaration_node_t declaration_type() const = 0; }; struct function_body { @@ -49,11 +38,11 @@ public: function_declarartion_node(front::token name) : m_name(name) {} public: - declaration_node_t type() const override { return declaration_node_t::FunctionDeclaration; } + declaration_node_t declaration_type() const override { return declaration_node_t::FunctionDeclaration; } front::token name() const { return m_name; } public: - std::ostream& print(std::ostream& os) const override { return os << ": " << m_name; } + std::ostream& print(std::ostream& os) const override { return os << "function " << m_name.value << " declaration"; } protected: front::token m_name; }; @@ -66,37 +55,33 @@ public: ~function_definition_node() override = default; function_definition_node(function_definition_node&& other) noexcept - : function_declarartion_node(std::move(other)), m_name(other.m_name), m_body(std::move(other.m_body)) {} + : function_declarartion_node(std::move(other)), m_body(std::move(other.m_body)) {} function_definition_node(const function_definition_node&) = delete; function_definition_node& operator=(function_definition_node&& other) noexcept { if (this == &other) return *this; function_declarartion_node::operator=(std::move(other)); - m_name = other.m_name; m_body = std::move(other.m_body); return *this; } function_definition_node& operator=(const function_definition_node&) = delete; public: - declaration_node_t type() const override { return declaration_node_t::FunctionDefinition; } + declaration_node_t declaration_type() const override { return declaration_node_t::FunctionDefinition; } const function_body_handle& body() const { return m_body; } public: std::ostream& print(std::ostream& os) const override { - os << ": " << m_name.value; + os << "function " << m_name.value << " definition:"; if (m_body.present()) { - os << '\n' << m_body->begin << ": begin:"; - for (const auto& entry : m_body->statements) { + for (const auto& entry : m_body->statements) os << '\n' << entry; - } - os << '\n' << m_body->end << ": end"; + return os << '\n' << m_body->end << ": " << m_name.value << " end"; } - return os; + return os << (std::string)m_body; // error } private: - front::token m_name; function_body_handle m_body; }; diff --git a/furc/include/furc/ast/expression.hpp b/furc/include/furc/ast/expression.hpp new file mode 100644 index 0000000..48ef95a --- /dev/null +++ b/furc/include/furc/ast/expression.hpp @@ -0,0 +1,26 @@ +#ifndef FURC_AST_EXPRESSION_HPP +#define FURC_AST_EXPRESSION_HPP + +#include "furc/ast/node.hpp" +#include "furc/ast/statement.hpp" + +namespace furc { +namespace ast { + +enum class expression_node_t { + Literal +}; + +class expression_node : public statement_node { +public: + node_t category() const override { return node_t::Expression; } + + statement_node_t statement_type() const override { return statement_node_t::Expression; } + + virtual expression_node_t expression_type() const = 0; +}; + +} // namespace ast +} // namespace furc + +#endif // FURC_AST_EXPRESSION_HPP \ No newline at end of file diff --git a/furc/include/furc/ast/literal.hpp b/furc/include/furc/ast/literal.hpp new file mode 100644 index 0000000..b803228 --- /dev/null +++ b/furc/include/furc/ast/literal.hpp @@ -0,0 +1,65 @@ +#ifndef FURC_AST_LITERAL_HPP +#define FURC_AST_LITERAL_HPP + +#include "furc/ast/expression.hpp" +#include "furc/ast/node.hpp" + +#include + +namespace furc { +namespace ast { + +enum class literal_node_t { + String, + Integer +}; + +static inline std::ostream& operator<<(std::ostream& os, literal_node_t type) { + switch (type) { + case literal_node_t::String: return os << "string"; + case literal_node_t::Integer: return os << "integer"; + } +} + +class literal_node : public expression_node { +public: + node_t category() const override { return node_t::Literal; } + + expression_node_t expression_type() const override { return expression_node_t::Literal; } + + virtual literal_node_t literal_type() const = 0; +}; + +class string_literal_node : public literal_node { +public: + string_literal_node(handle&& value) + : m_value(std::move(value)) {} +public: + literal_node_t literal_type() const override { return literal_node_t::String; } + + const handle& value() const { return m_value; } +public: + std::ostream& print(std::ostream& os) const override { + if (m_value.error()) return os << (std::string)m_value; + return os << '"' << *m_value << '"'; + } +private: + handle m_value; +}; + +class integer_literal_node : public literal_node { +public: + integer_literal_node(handle&& value) + : m_value(std::move(value)) {} +public: + literal_node_t literal_type() const override { return literal_node_t::Integer; } + + const handle& value() const { return m_value; } +private: + handle m_value; +}; + +} // namespace ast +} // namespace furc + +#endif // FURC_AST_LITERAL_HPP \ No newline at end of file diff --git a/furc/include/furc/ast/node.hpp b/furc/include/furc/ast/node.hpp index 2420108..fa06801 100644 --- a/furc/include/furc/ast/node.hpp +++ b/furc/include/furc/ast/node.hpp @@ -16,6 +16,16 @@ enum class node_t { Program, }; +static inline std::ostream& operator<<(std::ostream& os, node_t type) { + switch (type) { + case node_t::Literal: return os << "literal"; + case node_t::Expression: return os << "expression"; + case node_t::Statement: return os << "statement"; + case node_t::Declaration: return os << "declaration"; + case node_t::Program: return os << "program"; + } +} + class node { public: node() = default; @@ -27,12 +37,10 @@ public: node& operator=(const node&) = delete; public: virtual node_t category() const = 0; -}; -template -class abstract_node : public node { -public: - node_t category() const override { return Category; } + virtual std::ostream& print(std::ostream& os) const = 0; + + friend std::ostream& operator<<(std::ostream& os, const node& node) { return node.print(os); } }; template diff --git a/furc/include/furc/ast/program.hpp b/furc/include/furc/ast/program.hpp index 64a5a70..c808978 100644 --- a/furc/include/furc/ast/program.hpp +++ b/furc/include/furc/ast/program.hpp @@ -10,17 +10,19 @@ namespace furc { namespace ast { -class program_node : public abstract_node { +class program_node : public node { public: - program_node() {} + program_node() = default; + + node_t category() const override { return node_t::Program; } public: void push(node_handle&& declaration) { m_declarations.push_back(std::move(declaration)); } const std::vector>& declarations() const { return m_declarations; } public: - friend std::ostream& operator<<(std::ostream& os, const program_node& node) { - os << "program"; - for (const auto& handle : node.m_declarations) { + std::ostream& print(std::ostream& os) const override { + os << "program:"; + for (const auto& handle : m_declarations) { os << '\n' << handle; } return os; diff --git a/furc/include/furc/ast/statement.hpp b/furc/include/furc/ast/statement.hpp index 3674b11..1f4889f 100644 --- a/furc/include/furc/ast/statement.hpp +++ b/furc/include/furc/ast/statement.hpp @@ -9,49 +9,25 @@ namespace furc { namespace ast { enum class statement_node_t { + Expression, Declaration, Return, }; -static inline std::ostream& operator<<(std::ostream& os, statement_node_t type) { - switch (type) { - case statement_node_t::Declaration: return os << "declaration"; - case statement_node_t::Return: return os << "return"; - } -} - -class statement_node : public abstract_node { +class statement_node : public node { public: - virtual statement_node_t type() const = 0; -public: - virtual std::ostream& print(std::ostream& os) const = 0; + node_t category() const override { return node_t::Statement; } - friend std::ostream& operator<<(std::ostream& os, const statement_node& node) { - os << node.type() << " statement"; - return node.print(os); - } -}; - -class declaration_node; -class declaration_statement_node : public statement_node { -public: - declaration_statement_node(node_handle&& declaration) - : m_declaration(std::move(declaration)) {} -public: - statement_node_t type() const override { return statement_node_t::Declaration; } - - std::ostream& print(std::ostream& os) const override { return os; } -private: - node_handle m_declaration; + virtual statement_node_t statement_type() const = 0; }; class return_statement_node : public statement_node { public: return_statement_node() = default; public: - statement_node_t type() const override { return statement_node_t::Return; } + statement_node_t statement_type() const override { return statement_node_t::Return; } - std::ostream& print(std::ostream& os) const override { return os; } + std::ostream& print(std::ostream& os) const override { return os << "return statement"; } }; } // namespace ast diff --git a/furc/include/furc/front/parser.hpp b/furc/include/furc/front/parser.hpp index 59f1971..a7084cb 100644 --- a/furc/include/furc/front/parser.hpp +++ b/furc/include/furc/front/parser.hpp @@ -2,6 +2,8 @@ #define FURC_FRONT_PARSER_HPP #include "furc/ast/declaration.hpp" +#include "furc/ast/expression.hpp" +#include "furc/ast/literal.hpp" #include "furc/ast/node.hpp" #include "furc/ast/program.hpp" #include "furc/front/lexer.hpp" @@ -28,6 +30,8 @@ public: private: ast::node_handle parse_declaration(); ast::node_handle parse_statement(); + ast::node_handle parse_expression(); + ast::node_handle parse_literal(); ast::function_body_handle parse_body(); private: diff --git a/furc/include/furc/front/token.hpp b/furc/include/furc/front/token.hpp index 8251c97..2cc6b7c 100644 --- a/furc/include/furc/front/token.hpp +++ b/furc/include/furc/front/token.hpp @@ -14,6 +14,7 @@ enum class token_t { None, Identifier, Keyword, + String, Integer, Lparen, Rparen, @@ -32,6 +33,7 @@ static inline std::ostream& operator<<(std::ostream& os, token_t type) { case token_t::None: return os << "none"; case token_t::Identifier: return os << "identifier"; case token_t::Keyword: return os << "keyword"; + case token_t::String: return os << "string"; case token_t::Integer: return os << "integer"; case token_t::Lparen: return os << "'('"; case token_t::Rparen: return os << "')'"; @@ -51,6 +53,7 @@ static inline std::string operator+(const std::string& str, token_t type) { case token_t::None: return str + "none"; case token_t::Identifier: return str + "identifier"; case token_t::Keyword: return str + "keyword"; + case token_t::String: return str + "string"; case token_t::Integer: return str + "integer"; case token_t::Lparen: return str + "'('"; case token_t::Rparen: return str + "')'"; @@ -103,9 +106,10 @@ struct token { bool operator==(const token& rhs) const { if (type != rhs.type) return false; switch (type) { - case token_t::Identifier: return value == rhs.value; - case token_t::Keyword: return keyword == rhs.keyword; + case token_t::Identifier: + case token_t::String: case token_t::Integer: return value == rhs.value; + case token_t::Keyword: return keyword == rhs.keyword; case token_t::None: case token_t::Lparen: case token_t::Rparen: @@ -124,6 +128,7 @@ struct token { static inline std::ostream& operator<<(std::ostream& os, const token& token) { switch (token.type) { case token_t::Identifier: + case token_t::String: case token_t::Integer: return os << token.value; case token_t::Keyword: return os << token.keyword; default: return os << token.type; diff --git a/furc/src/front/lexer.cpp b/furc/src/front/lexer.cpp index ae0ed43..fd8c800 100644 --- a/furc/src/front/lexer.cpp +++ b/furc/src/front/lexer.cpp @@ -54,6 +54,15 @@ token_handle<> lexer::next_token() { case ':': return { location, token_t::Colon, m_content.substr(m_cursor++, 1) }; case ',': return { location, token_t::Comma, m_content.substr(m_cursor++, 1) }; case '.': return { location, token_t::Dot, m_content.substr(m_cursor++, 1) }; + case '"': { + std::size_t begin = ++m_cursor; + while (m_cursor < m_content.size() && m_content[m_cursor] != '"') + ++m_cursor; + if (m_cursor >= m_content.size()) return { current_location(), "unexpected end of file before enclosing '\"'" }; + ++m_cursor; + + return { location, token_t::String, m_content.substr(begin, m_cursor - begin - 1) }; + } case std::char_traits::eof(): return { location, token_t::None }; default: { if (std::isdigit(ch) != 0) {} diff --git a/furc/src/front/parser.cpp b/furc/src/front/parser.cpp index e019088..e187e66 100644 --- a/furc/src/front/parser.cpp +++ b/furc/src/front/parser.cpp @@ -36,9 +36,10 @@ ast::node_handle parser::parse() & { } ast::node_handle parser::parse_declaration() { - token_handle<> first = next_token(); + const auto& first = peek_token(); switch (first->type) { case token_t::Keyword: { + auto first = next_token(); switch (first->keyword) { case keyword_token::Func: { token_handle<> name = eat_token(token_t::Identifier); @@ -64,22 +65,9 @@ ast::node_handle parser::parse_declaration() { m_peekBuffer.clear(); return ast::node_handle{ first.location(), m_arena, name }; } - case token_t::None: - case token_t::Identifier: - case token_t::Keyword: - case token_t::Integer: - case token_t::Lparen: - case token_t::Rparen: - case token_t::Rbrace: - case token_t::Lbracket: - case token_t::Rbracket: - case token_t::Colon: - case token_t::Comma: - case token_t::Dot: return { tok.location(), "unexpected token "s + tok->type }; + default: return { tok.location(), "unexpected token "s + tok->type }; } } - case keyword_token::Return: - case keyword_token::None: default: return { first.location(), "unexpected keyword "s + first->keyword }; } } @@ -115,9 +103,46 @@ ast::node_handle parser::parse_statement() { } auto declaration = parse_declaration(); - if (declaration.error()) - return { declaration.location(), "unexpected token "s + tok->type + ", expected statement" }; - return ast::node_handle{ declaration.location(), m_arena, std::move(declaration) }; + if (declaration.present()) return std::move(declaration); + auto expression = parse_expression(); + if (expression.present()) { + auto semi = eat_token(token_t::Semicolon); + if (semi.error()) return semi; + return std::move(expression); + } + + auto token = next_token(); + return { token.location(), "unexpected token "s + token->type + ", expected statement, declaration or expression" }; +} + +ast::node_handle parser::parse_expression() { + const auto& tok = peek_token(); + + auto literal = parse_literal(); + if (literal.present()) return std::move(literal); + return { tok.location(), "unexpected token"s + tok->type + ", expected expression or literal" }; +} + +ast::node_handle parser::parse_literal() { + const auto& tok = peek_token(); + switch (tok->type) { + case token_t::String: { + auto tok = next_token(); + return ast::node_handle{ tok.location(), + m_arena, + handle{ tok.location(), tok->value } }; + } + case token_t::Integer: { + auto tok = next_token(); + // return ast::node_handle{ tok.location(), + // m_arena, + // handle{ tok.location(), tok->value } }; + return { tok.location(), "unimplemented parse_literal() for token_t::Integer" }; + } + default: break; + } + + return { tok.location(), "unexpected token "s + tok->type + ", expected literal" }; } ast::function_body_handle parser::parse_body() { @@ -127,7 +152,7 @@ ast::function_body_handle parser::parse_body() { if (begin.error()) return begin; body.begin = begin.location(); - while (!peek_token().error() && peek_token()->type != token_t::Rbrace) { + while (!peek_token().error() && peek_token()->type != token_t::None && peek_token()->type != token_t::Rbrace) { body.statements.push_back(parse_statement()); } @@ -158,7 +183,10 @@ const token_handle<>& parser::peek_token() { token_handle<> parser::eat_token(token_t type) { token_handle<> token = next_token(); if (!token.present()) return token; - if (token->type != type) return { token.location(), "unexpected token "s + token->type + ", expected " + type }; + if (token->type != type) { + if (token->type == token_t::None) return { token.location(), "unexpected end of file, expected " + type }; + return { token.location(), "unexpected token "s + token->type + ", expected " + type }; + } return token; } diff --git a/furc/test/lexer.cpp b/furc/test/lexer.cpp index 41cddb8..3843cd0 100644 --- a/furc/test/lexer.cpp +++ b/furc/test/lexer.cpp @@ -11,12 +11,13 @@ using namespace std::string_view_literals; #define EXPECT_EMPTY_TOKEN(lexer) EXPECT_EQ((lexer).next_token(), (token{})); TEST(Lexer, Tokens) { - lexer lexer("", "()\n\t\t{\n}[]; :,.main return func"); + lexer lexer("", "()\n\t\t{\n}[\"shto-to\"]; :,.main return func"); EXPECT_TOKEN(lexer, (token{ token_t::Lparen, "("sv })); EXPECT_TOKEN(lexer, (token{ token_t::Rparen, ")"sv })); EXPECT_TOKEN(lexer, (token{ token_t::Lbrace, "{"sv })); EXPECT_TOKEN(lexer, (token{ token_t::Rbrace, "}"sv })); EXPECT_TOKEN(lexer, (token{ token_t::Lbracket, "["sv })); + EXPECT_TOKEN(lexer, (token{ token_t::String, "shto-to"sv })); EXPECT_TOKEN(lexer, (token{ token_t::Rbracket, "]"sv })); EXPECT_TOKEN(lexer, (token{ token_t::Semicolon, ";"sv })); EXPECT_TOKEN(lexer, (token{ token_t::Colon, ":"sv })); diff --git a/furc/test/parser.cpp b/furc/test/parser.cpp index ad2e259..35b2e7f 100644 --- a/furc/test/parser.cpp +++ b/furc/test/parser.cpp @@ -16,7 +16,7 @@ TEST(Parser, EmptyFunctions) { { auto first = program->declarations()[0]; EXPECT_TRUE(first.present()); - EXPECT_EQ(first->type(), declaration_node_t::FunctionDefinition); + EXPECT_EQ(first->declaration_type(), declaration_node_t::FunctionDefinition); node_handle funcDef = first; EXPECT_EQ(funcDef->name().value, "main"); EXPECT_EQ(funcDef->body()->statements.size(), 0); @@ -24,7 +24,7 @@ TEST(Parser, EmptyFunctions) { { auto second = program->declarations()[1]; EXPECT_TRUE(second.present()); - EXPECT_EQ(second->type(), declaration_node_t::FunctionDeclaration); + EXPECT_EQ(second->declaration_type(), declaration_node_t::FunctionDeclaration); node_handle funcDecl = second; EXPECT_EQ(funcDecl->name().value, "foo"); }