Improve AST nodes

Signed-off-by: CHatingPython <chatingpython@gmail.com>

furc/include/furc/ast/declaration.hpp

@@ -17,24 +17,13 @@ enum class declaration_node_t {
     Variable,
 };
 
-static inline std::ostream& operator<<(std::ostream& os, declaration_node_t type) {
-    switch (type) {
-    case declaration_node_t::FunctionDeclaration: return os << "function declaration";
-    case declaration_node_t::FunctionDefinition: return os << "function definition";
-    case declaration_node_t::Variable: return os << "variable";
-    }
-}
-
-class declaration_node : public abstract_node<node_t::Declaration> {
+class declaration_node : public statement_node {
 public:
-    virtual declaration_node_t type() const = 0;
-public:
-    virtual std::ostream& print(std::ostream& os) const = 0;
+    node_t category() const override { return node_t::Declaration; }
 
-    friend std::ostream& operator<<(std::ostream& os, const declaration_node& node) {
-        os << node.type();
-        return node.print(os);
-    }
+    statement_node_t statement_type() const override { return statement_node_t::Declaration; }
+
+    virtual declaration_node_t declaration_type() const = 0;
 };
 
 struct function_body {
@@ -49,11 +38,11 @@ public:
     function_declarartion_node(front::token name)
       : m_name(name) {}
 public:
-    declaration_node_t type() const override { return declaration_node_t::FunctionDeclaration; }
+    declaration_node_t declaration_type() const override { return declaration_node_t::FunctionDeclaration; }
 
     front::token name() const { return m_name; }
 public:
-    std::ostream& print(std::ostream& os) const override { return os << ": " << m_name; }
+    std::ostream& print(std::ostream& os) const override { return os << "function " << m_name.value << " declaration"; }
 protected:
     front::token m_name;
 };
@@ -66,37 +55,33 @@ public:
     ~function_definition_node() override = default;
 
     function_definition_node(function_definition_node&& other) noexcept
-      : function_declarartion_node(std::move(other)), m_name(other.m_name), m_body(std::move(other.m_body)) {}
+      : function_declarartion_node(std::move(other)), m_body(std::move(other.m_body)) {}
 
     function_definition_node(const function_definition_node&) = delete;
 
     function_definition_node& operator=(function_definition_node&& other) noexcept {
         if (this == &other) return *this;
         function_declarartion_node::operator=(std::move(other));
-        m_name = other.m_name;
         m_body = std::move(other.m_body);
         return *this;
     }
 
     function_definition_node& operator=(const function_definition_node&) = delete;
 public:
-    declaration_node_t type() const override { return declaration_node_t::FunctionDefinition; }
+    declaration_node_t declaration_type() const override { return declaration_node_t::FunctionDefinition; }
 
     const function_body_handle& body() const { return m_body; }
 public:
     std::ostream& print(std::ostream& os) const override {
-        os << ": " << m_name.value;
+        os << "function " << m_name.value << " definition:";
         if (m_body.present()) {
-            os << '\n' << m_body->begin << ": begin:";
-            for (const auto& entry : m_body->statements) {
+            for (const auto& entry : m_body->statements)
                 os << '\n' << entry;
-            }
-            os << '\n' << m_body->end << ": end";
+            return os << '\n' << m_body->end << ": " << m_name.value << " end";
         }
-        return os;
+        return os << (std::string)m_body; // error
     }
 private:
-    front::token         m_name;
     function_body_handle m_body;
 };
 

furc/include/furc/ast/expression.hpp

@@ -0,0 +1,26 @@
+#ifndef FURC_AST_EXPRESSION_HPP
+#define FURC_AST_EXPRESSION_HPP
+
+#include "furc/ast/node.hpp"
+#include "furc/ast/statement.hpp"
+
+namespace furc {
+namespace ast {
+
+enum class expression_node_t {
+    Literal
+};
+
+class expression_node : public statement_node {
+public:
+    node_t category() const override { return node_t::Expression; }
+
+    statement_node_t statement_type() const override { return statement_node_t::Expression; }
+
+    virtual expression_node_t expression_type() const = 0;
+};
+
+} // namespace ast
+} // namespace furc
+
+#endif // FURC_AST_EXPRESSION_HPP

furc/include/furc/ast/literal.hpp

@@ -0,0 +1,65 @@
+#ifndef FURC_AST_LITERAL_HPP
+#define FURC_AST_LITERAL_HPP
+
+#include "furc/ast/expression.hpp"
+#include "furc/ast/node.hpp"
+
+#include <ostream>
+
+namespace furc {
+namespace ast {
+
+enum class literal_node_t {
+    String,
+    Integer
+};
+
+static inline std::ostream& operator<<(std::ostream& os, literal_node_t type) {
+    switch (type) {
+    case literal_node_t::String: return os << "string";
+    case literal_node_t::Integer: return os << "integer";
+    }
+}
+
+class literal_node : public expression_node {
+public:
+    node_t category() const override { return node_t::Literal; }
+
+    expression_node_t expression_type() const override { return expression_node_t::Literal; }
+
+    virtual literal_node_t literal_type() const = 0;
+};
+
+class string_literal_node : public literal_node {
+public:
+    string_literal_node(handle<std::string_view>&& value)
+      : m_value(std::move(value)) {}
+public:
+    literal_node_t literal_type() const override { return literal_node_t::String; }
+
+    const handle<std::string_view>& value() const { return m_value; }
+public:
+    std::ostream& print(std::ostream& os) const override {
+        if (m_value.error()) return os << (std::string)m_value;
+        return os << '"' << *m_value << '"';
+    }
+private:
+    handle<std::string_view> m_value;
+};
+
+class integer_literal_node : public literal_node {
+public:
+    integer_literal_node(handle<std::uint64_t>&& value)
+      : m_value(std::move(value)) {}
+public:
+    literal_node_t literal_type() const override { return literal_node_t::Integer; }
+
+    const handle<std::uint64_t>& value() const { return m_value; }
+private:
+    handle<std::uint64_t> m_value;
+};
+
+} // namespace ast
+} // namespace furc
+
+#endif // FURC_AST_LITERAL_HPP

furc/include/furc/ast/node.hpp

@@ -16,6 +16,16 @@ enum class node_t {
     Program,
 };
 
+static inline std::ostream& operator<<(std::ostream& os, node_t type) {
+    switch (type) {
+    case node_t::Literal: return os << "literal";
+    case node_t::Expression: return os << "expression";
+    case node_t::Statement: return os << "statement";
+    case node_t::Declaration: return os << "declaration";
+    case node_t::Program: return os << "program";
+    }
+}
+
 class node {
 public:
     node()          = default;
@@ -27,12 +37,10 @@ public:
     node& operator=(const node&) = delete;
 public:
     virtual node_t category() const = 0;
-};
 
-template <node_t Category>
-class abstract_node : public node {
-public:
-    node_t category() const override { return Category; }
+    virtual std::ostream& print(std::ostream& os) const = 0;
+
+    friend std::ostream& operator<<(std::ostream& os, const node& node) { return node.print(os); }
 };
 
 template <typename T, typename Error = std::string>

furc/include/furc/ast/program.hpp

@@ -10,17 +10,19 @@
 namespace furc {
 namespace ast {
 
-class program_node : public abstract_node<node_t::Program> {
+class program_node : public node {
 public:
-    program_node() {}
+    program_node() = default;
+
+    node_t category() const override { return node_t::Program; }
 public:
     void push(node_handle<declaration_node>&& declaration) { m_declarations.push_back(std::move(declaration)); }
 
     const std::vector<node_handle<declaration_node>>& declarations() const { return m_declarations; }
 public:
-    friend std::ostream& operator<<(std::ostream& os, const program_node& node) {
-        os << "program";
-        for (const auto& handle : node.m_declarations) {
+    std::ostream& print(std::ostream& os) const override {
+        os << "program:";
+        for (const auto& handle : m_declarations) {
             os << '\n' << handle;
         }
         return os;

furc/include/furc/ast/statement.hpp

@@ -9,49 +9,25 @@ namespace furc {
 namespace ast {
 
 enum class statement_node_t {
+    Expression,
     Declaration,
     Return,
 };
 
-static inline std::ostream& operator<<(std::ostream& os, statement_node_t type) {
-    switch (type) {
-    case statement_node_t::Declaration: return os << "declaration";
-    case statement_node_t::Return: return os << "return";
-    }
-}
-
-class statement_node : public abstract_node<node_t::Statement> {
+class statement_node : public node {
 public:
-    virtual statement_node_t type() const = 0;
-public:
-    virtual std::ostream& print(std::ostream& os) const = 0;
+    node_t category() const override { return node_t::Statement; }
 
-    friend std::ostream& operator<<(std::ostream& os, const statement_node& node) {
-        os << node.type() << " statement";
-        return node.print(os);
-    }
-};
-
-class declaration_node;
-class declaration_statement_node : public statement_node {
-public:
-    declaration_statement_node(node_handle<declaration_node>&& declaration)
-      : m_declaration(std::move(declaration)) {}
-public:
-    statement_node_t type() const override { return statement_node_t::Declaration; }
-
-    std::ostream& print(std::ostream& os) const override { return os; }
-private:
-    node_handle<declaration_node> m_declaration;
+    virtual statement_node_t statement_type() const = 0;
 };
 
 class return_statement_node : public statement_node {
 public:
     return_statement_node() = default;
 public:
-    statement_node_t type() const override { return statement_node_t::Return; }
+    statement_node_t statement_type() const override { return statement_node_t::Return; }
 
-    std::ostream& print(std::ostream& os) const override { return os; }
+    std::ostream& print(std::ostream& os) const override { return os << "return statement"; }
 };
 
 } // namespace ast

furc/include/furc/front/parser.hpp

@@ -2,6 +2,8 @@
 #define FURC_FRONT_PARSER_HPP
 
 #include "furc/ast/declaration.hpp"
+#include "furc/ast/expression.hpp"
+#include "furc/ast/literal.hpp"
 #include "furc/ast/node.hpp"
 #include "furc/ast/program.hpp"
 #include "furc/front/lexer.hpp"
@@ -28,6 +30,8 @@ public:
 private:
     ast::node_handle<ast::declaration_node> parse_declaration();
     ast::node_handle<ast::statement_node>   parse_statement();
+    ast::node_handle<ast::expression_node>  parse_expression();
+    ast::node_handle<ast::literal_node>     parse_literal();
 
     ast::function_body_handle parse_body();
 private:

furc/include/furc/front/token.hpp

@@ -14,6 +14,7 @@ enum class token_t {
     None,
     Identifier,
     Keyword,
+    String,
     Integer,
     Lparen,
     Rparen,
@@ -32,6 +33,7 @@ static inline std::ostream& operator<<(std::ostream& os, token_t type) {
     case token_t::None: return os << "none";
     case token_t::Identifier: return os << "identifier";
     case token_t::Keyword: return os << "keyword";
+    case token_t::String: return os << "string";
     case token_t::Integer: return os << "integer";
     case token_t::Lparen: return os << "'('";
     case token_t::Rparen: return os << "')'";
@@ -51,6 +53,7 @@ static inline std::string operator+(const std::string& str, token_t type) {
     case token_t::None: return str + "none";
     case token_t::Identifier: return str + "identifier";
     case token_t::Keyword: return str + "keyword";
+    case token_t::String: return str + "string";
     case token_t::Integer: return str + "integer";
     case token_t::Lparen: return str + "'('";
     case token_t::Rparen: return str + "')'";
@@ -103,9 +106,10 @@ struct token {
     bool operator==(const token& rhs) const {
         if (type != rhs.type) return false;
         switch (type) {
-        case token_t::Identifier: return value == rhs.value;
-        case token_t::Keyword: return keyword == rhs.keyword;
+        case token_t::Identifier:
+        case token_t::String:
         case token_t::Integer: return value == rhs.value;
+        case token_t::Keyword: return keyword == rhs.keyword;
         case token_t::None:
         case token_t::Lparen:
         case token_t::Rparen:
@@ -124,6 +128,7 @@ struct token {
 static inline std::ostream& operator<<(std::ostream& os, const token& token) {
     switch (token.type) {
     case token_t::Identifier:
+    case token_t::String:
     case token_t::Integer: return os << token.value;
     case token_t::Keyword: return os << token.keyword;
     default: return os << token.type;

furc/src/front/lexer.cpp

@@ -54,6 +54,15 @@ token_handle<> lexer::next_token() {
     case ':': return { location, token_t::Colon, m_content.substr(m_cursor++, 1) };
     case ',': return { location, token_t::Comma, m_content.substr(m_cursor++, 1) };
     case '.': return { location, token_t::Dot, m_content.substr(m_cursor++, 1) };
+    case '"': {
+        std::size_t begin = ++m_cursor;
+        while (m_cursor < m_content.size() && m_content[m_cursor] != '"')
+            ++m_cursor;
+        if (m_cursor >= m_content.size()) return { current_location(), "unexpected end of file before enclosing '\"'" };
+        ++m_cursor;
+
+        return { location, token_t::String, m_content.substr(begin, m_cursor - begin - 1) };
+    }
     case std::char_traits<char>::eof(): return { location, token_t::None };
     default: {
         if (std::isdigit(ch) != 0) {}

furc/src/front/parser.cpp

@@ -36,9 +36,10 @@ ast::node_handle<ast::program_node> parser::parse() & {
 }
 
 ast::node_handle<ast::declaration_node> parser::parse_declaration() {
-    token_handle<> first = next_token();
+    const auto& first = peek_token();
     switch (first->type) {
     case token_t::Keyword: {
+        auto first = next_token();
         switch (first->keyword) {
         case keyword_token::Func: {
             token_handle<> name = eat_token(token_t::Identifier);
@@ -64,22 +65,9 @@ ast::node_handle<ast::declaration_node> parser::parse_declaration() {
                 m_peekBuffer.clear();
                 return ast::node_handle<ast::function_declarartion_node>{ first.location(), m_arena, name };
             }
-            case token_t::None:
-            case token_t::Identifier:
-            case token_t::Keyword:
-            case token_t::Integer:
-            case token_t::Lparen:
-            case token_t::Rparen:
-            case token_t::Rbrace:
-            case token_t::Lbracket:
-            case token_t::Rbracket:
-            case token_t::Colon:
-            case token_t::Comma:
-            case token_t::Dot: return { tok.location(), "unexpected token "s + tok->type };
+            default: return { tok.location(), "unexpected token "s + tok->type };
             }
         }
-        case keyword_token::Return:
-        case keyword_token::None:
         default: return { first.location(), "unexpected keyword "s + first->keyword };
         }
     }
@@ -115,9 +103,46 @@ ast::node_handle<ast::statement_node> parser::parse_statement() {
     }
 
     auto declaration = parse_declaration();
-    if (declaration.error())
-        return { declaration.location(), "unexpected token "s + tok->type + ", expected statement" };
-    return ast::node_handle<ast::declaration_statement_node>{ declaration.location(), m_arena, std::move(declaration) };
+    if (declaration.present()) return std::move(declaration);
+    auto expression = parse_expression();
+    if (expression.present()) {
+        auto semi = eat_token(token_t::Semicolon);
+        if (semi.error()) return semi;
+        return std::move(expression);
+    }
+
+    auto token = next_token();
+    return { token.location(), "unexpected token "s + token->type + ", expected statement, declaration or expression" };
+}
+
+ast::node_handle<ast::expression_node> parser::parse_expression() {
+    const auto& tok = peek_token();
+
+    auto literal = parse_literal();
+    if (literal.present()) return std::move(literal);
+    return { tok.location(), "unexpected token"s + tok->type + ", expected expression or literal" };
+}
+
+ast::node_handle<ast::literal_node> parser::parse_literal() {
+    const auto& tok = peek_token();
+    switch (tok->type) {
+    case token_t::String: {
+        auto tok = next_token();
+        return ast::node_handle<ast::string_literal_node>{ tok.location(),
+            m_arena,
+            handle<std::string_view>{ tok.location(), tok->value } };
+    }
+    case token_t::Integer: {
+        auto tok = next_token();
+        // return ast::node_handle<ast::integer_literal_node>{ tok.location(),
+        //     m_arena,
+        //     handle<std::uint64_t>{ tok.location(), tok->value } };
+        return { tok.location(), "unimplemented parse_literal() for token_t::Integer" };
+    }
+    default: break;
+    }
+
+    return { tok.location(), "unexpected token "s + tok->type + ", expected literal" };
 }
 
 ast::function_body_handle parser::parse_body() {
@@ -127,7 +152,7 @@ ast::function_body_handle parser::parse_body() {
     if (begin.error()) return begin;
     body.begin = begin.location();
 
-    while (!peek_token().error() && peek_token()->type != token_t::Rbrace) {
+    while (!peek_token().error() && peek_token()->type != token_t::None && peek_token()->type != token_t::Rbrace) {
         body.statements.push_back(parse_statement());
     }
 
@@ -158,7 +183,10 @@ const token_handle<>& parser::peek_token() {
 token_handle<> parser::eat_token(token_t type) {
     token_handle<> token = next_token();
     if (!token.present()) return token;
-    if (token->type != type) return { token.location(), "unexpected token "s + token->type + ", expected " + type };
+    if (token->type != type) {
+        if (token->type == token_t::None) return { token.location(), "unexpected end of file, expected " + type };
+        return { token.location(), "unexpected token "s + token->type + ", expected " + type };
+    }
     return token;
 }
 

furc/test/lexer.cpp

@@ -11,12 +11,13 @@ using namespace std::string_view_literals;
 #define EXPECT_EMPTY_TOKEN(lexer) EXPECT_EQ((lexer).next_token(), (token{}));
 
 TEST(Lexer, Tokens) {
-    lexer lexer("<TEMP>", "()\n\t\t{\n}[];    :,.main return func");
+    lexer lexer("<TEMP>", "()\n\t\t{\n}[\"shto-to\"];    :,.main return func");
     EXPECT_TOKEN(lexer, (token{ token_t::Lparen, "("sv }));
     EXPECT_TOKEN(lexer, (token{ token_t::Rparen, ")"sv }));
     EXPECT_TOKEN(lexer, (token{ token_t::Lbrace, "{"sv }));
     EXPECT_TOKEN(lexer, (token{ token_t::Rbrace, "}"sv }));
     EXPECT_TOKEN(lexer, (token{ token_t::Lbracket, "["sv }));
+    EXPECT_TOKEN(lexer, (token{ token_t::String, "shto-to"sv }));
     EXPECT_TOKEN(lexer, (token{ token_t::Rbracket, "]"sv }));
     EXPECT_TOKEN(lexer, (token{ token_t::Semicolon, ";"sv }));
     EXPECT_TOKEN(lexer, (token{ token_t::Colon, ":"sv }));

furc/test/parser.cpp

@@ -16,7 +16,7 @@ TEST(Parser, EmptyFunctions) {
     {
         auto first = program->declarations()[0];
         EXPECT_TRUE(first.present());
-        EXPECT_EQ(first->type(), declaration_node_t::FunctionDefinition);
+        EXPECT_EQ(first->declaration_type(), declaration_node_t::FunctionDefinition);
         node_handle<function_definition_node> funcDef = first;
         EXPECT_EQ(funcDef->name().value, "main");
         EXPECT_EQ(funcDef->body()->statements.size(), 0);
@@ -24,7 +24,7 @@ TEST(Parser, EmptyFunctions) {
     {
         auto second = program->declarations()[1];
         EXPECT_TRUE(second.present());
-        EXPECT_EQ(second->type(), declaration_node_t::FunctionDeclaration);
+        EXPECT_EQ(second->declaration_type(), declaration_node_t::FunctionDeclaration);
         node_handle<function_declarartion_node> funcDecl = second;
         EXPECT_EQ(funcDecl->name().value, "foo");
     }