From ad84bc50bd6288d2b80160155c0b961deb0aaa9b Mon Sep 17 00:00:00 2001 From: CHatingPython Date: Tue, 26 May 2026 08:52:35 +0200 Subject: [PATCH] Add integers to lexer Signed-off-by: CHatingPython --- furc/include/furc/ast/declaration.hpp | 11 +++--- furc/include/furc/ast/literal.hpp | 23 ++++++------ furc/include/furc/front/token.hpp | 50 ++++++++++++++++++++------- furc/include/furc/handle.hpp | 15 ++++---- furc/src/front/lexer.cpp | 23 +++++++++++- furc/src/front/parser.cpp | 35 +++++++++---------- furc/test/lexer.cpp | 20 +++++++++-- furc/test/parser.cpp | 4 +-- 8 files changed, 121 insertions(+), 60 deletions(-) diff --git a/furc/include/furc/ast/declaration.hpp b/furc/include/furc/ast/declaration.hpp index 9cdcdb4..1522e8c 100644 --- a/furc/include/furc/ast/declaration.hpp +++ b/furc/include/furc/ast/declaration.hpp @@ -42,7 +42,9 @@ public: front::token name() const { return m_name; } public: - std::ostream& print(std::ostream& os) const override { return os << "function " << m_name.value << " declaration"; } + std::ostream& print(std::ostream& os) const override { + return os << "function " << m_name->string << " declaration"; + } protected: front::token m_name; }; @@ -73,13 +75,14 @@ public: const function_body_handle& body() const { return m_body; } public: std::ostream& print(std::ostream& os) const override { - os << "function " << m_name.value << " definition:"; + function_declarartion_node::print(os); + os << ':'; if (m_body.present()) { for (const auto& entry : m_body->statements) os << '\n' << entry; - return os << '\n' << m_body->end << ": " << m_name.value << " end"; + return os << '\n' << m_body->end << ": " << m_name->string << " end"; } - return os << (std::string)m_body; // error + return os << m_body.error(); // error } private: function_body_handle m_body; diff --git a/furc/include/furc/ast/literal.hpp b/furc/include/furc/ast/literal.hpp index b803228..741c798 100644 --- a/furc/include/furc/ast/literal.hpp +++ b/furc/include/furc/ast/literal.hpp @@ -3,6 +3,7 @@ #include "furc/ast/expression.hpp" #include "furc/ast/node.hpp" +#include "furc/front/token.hpp" #include @@ -14,13 +15,6 @@ enum class literal_node_t { Integer }; -static inline std::ostream& operator<<(std::ostream& os, literal_node_t type) { - switch (type) { - case literal_node_t::String: return os << "string"; - case literal_node_t::Integer: return os << "integer"; - } -} - class literal_node : public expression_node { public: node_t category() const override { return node_t::Literal; } @@ -40,8 +34,8 @@ public: const handle& value() const { return m_value; } public: std::ostream& print(std::ostream& os) const override { - if (m_value.error()) return os << (std::string)m_value; - return os << '"' << *m_value << '"'; + if (m_value.has_error()) return os << (std::string)m_value; + return os << "string literal (" << *m_value << ")"; } private: handle m_value; @@ -49,14 +43,19 @@ private: class integer_literal_node : public literal_node { public: - integer_literal_node(handle&& value) + integer_literal_node(handle&& value) : m_value(std::move(value)) {} public: literal_node_t literal_type() const override { return literal_node_t::Integer; } - const handle& value() const { return m_value; } + const handle& value() const { return m_value; } +public: + std::ostream& print(std::ostream& os) const override { + if (m_value.has_error()) return os << m_value.error(); + return os << "integer literal (" << *m_value << ")"; + } private: - handle m_value; + handle m_value; }; } // namespace ast diff --git a/furc/include/furc/front/token.hpp b/furc/include/furc/front/token.hpp index 2cc6b7c..510c774 100644 --- a/furc/include/furc/front/token.hpp +++ b/furc/include/furc/front/token.hpp @@ -13,8 +13,8 @@ namespace front { enum class token_t { None, Identifier, - Keyword, String, + Keyword, Integer, Lparen, Rparen, @@ -32,8 +32,8 @@ static inline std::ostream& operator<<(std::ostream& os, token_t type) { switch (type) { case token_t::None: return os << "none"; case token_t::Identifier: return os << "identifier"; - case token_t::Keyword: return os << "keyword"; case token_t::String: return os << "string"; + case token_t::Keyword: return os << "keyword"; case token_t::Integer: return os << "integer"; case token_t::Lparen: return os << "'('"; case token_t::Rparen: return os << "')'"; @@ -52,8 +52,8 @@ static inline std::string operator+(const std::string& str, token_t type) { switch (type) { case token_t::None: return str + "none"; case token_t::Identifier: return str + "identifier"; - case token_t::Keyword: return str + "keyword"; case token_t::String: return str + "string"; + case token_t::Keyword: return str + "keyword"; case token_t::Integer: return str + "integer"; case token_t::Lparen: return str + "'('"; case token_t::Rparen: return str + "')'"; @@ -90,10 +90,28 @@ static inline std::string operator+(const std::string& str, keyword_token keywor } } +using integer_token = std::uint64_t; + struct token { - token_t type = token_t::None; - keyword_token keyword = keyword_token::None; - std::string_view value; + token_t type = token_t::None; + union value { + std::nullptr_t null; + std::string_view string; + keyword_token keyword; + integer_token integer; + + value(std::nullptr_t value = nullptr) + : null(value) {} + + value(std::string_view value) + : string(value) {} + + value(keyword_token value) + : keyword(value) {} + + value(integer_token value) + : integer(value) {} + } value; token() = default; @@ -101,15 +119,21 @@ struct token { : type(type), value(value) {} token(keyword_token keyword, std::string_view value = {}) - : type(token_t::Keyword), keyword(keyword), value(value) {} + : type(token_t::Keyword), value(keyword) {} + + token(integer_token integer) + : type(token_t::Integer), value(integer) {} + + union value* operator->() { return &value; } + const union value* operator->() const { return &value; } bool operator==(const token& rhs) const { if (type != rhs.type) return false; switch (type) { case token_t::Identifier: - case token_t::String: - case token_t::Integer: return value == rhs.value; - case token_t::Keyword: return keyword == rhs.keyword; + case token_t::String: return value.string == rhs.value.string; + case token_t::Keyword: return value.keyword == rhs.value.keyword; + case token_t::Integer: return value.integer == rhs.value.integer; case token_t::None: case token_t::Lparen: case token_t::Rparen: @@ -128,9 +152,9 @@ struct token { static inline std::ostream& operator<<(std::ostream& os, const token& token) { switch (token.type) { case token_t::Identifier: - case token_t::String: - case token_t::Integer: return os << token.value; - case token_t::Keyword: return os << token.keyword; + case token_t::String: return os << token.value.string; + case token_t::Keyword: return os << token.value.keyword; + case token_t::Integer: return os << token.value.integer; default: return os << token.type; } } diff --git a/furc/include/furc/handle.hpp b/furc/include/furc/handle.hpp index 9a87e1a..d24e8d7 100644 --- a/furc/include/furc/handle.hpp +++ b/furc/include/furc/handle.hpp @@ -40,7 +40,7 @@ public: template handle(const handle& error) - : m_location(error.location()), m_error(error) {} + : m_location(error.location()), m_error(error.error()) {} ~handle() = default; @@ -65,17 +65,18 @@ public: location location() const { return m_location; } bool present() const { return m_value.has_value(); } - bool error() const { return !m_value.has_value(); } + bool has_error() const { return !m_value.has_value(); } value_type move() { - value_type value = *m_value; + value_type value = std::move(*m_value); m_value.reset(); return value; } + Error error() const { return m_error; } + operator reference() { return *m_value; } operator const_reference() const { return *m_value; } - operator Error() const { return m_error; } reference operator*() { return *m_value; } const_reference operator*() const { return *m_value; } @@ -133,7 +134,7 @@ public: template handle(const handle& error) - : m_location(error.location()), m_error(error) {} + : m_location(error.location()), m_error(error.error()) {} ~handle() = default; @@ -172,13 +173,13 @@ public: location location() const { return m_location; } bool present() const { return m_value != nullptr; } - bool error() const { return m_value == nullptr; } + bool has_error() const { return m_value == nullptr; } std::shared_ptr shared() { return m_value; } + Error error() const { return m_error; } operator reference() { return *m_value; } operator const_reference() const { return *m_value; } - operator Error() const { return m_error; } reference operator*() { return *m_value; } const_reference operator*() const { return *m_value; } diff --git a/furc/src/front/lexer.cpp b/furc/src/front/lexer.cpp index fd8c800..1b7eecf 100644 --- a/furc/src/front/lexer.cpp +++ b/furc/src/front/lexer.cpp @@ -1,6 +1,7 @@ #include "furc/front/lexer.hpp" #include +#include #include #include @@ -65,7 +66,27 @@ token_handle<> lexer::next_token() { } case std::char_traits::eof(): return { location, token_t::None }; default: { - if (std::isdigit(ch) != 0) {} + if (std::isdigit(ch) != 0) { + integer_token integer = 0; + integer_token max = std::numeric_limits::max(); + integer_token upperBound = max / 10; + + std::size_t start = m_cursor; + while (std::isdigit(get()) != 0) { + integer_token digit = get() - '0'; + + if (integer > upperBound || integer == upperBound && (integer - upperBound + digit) > (max % 10)) { + while (std::isdigit(get()) != 0) + ++m_cursor; + return { location, "integer "s.append(m_content.substr(start, m_cursor - start)) + " is too big" }; + } + integer *= 10; + integer += digit; + ++m_cursor; + } + + return { location, integer }; + } if (std::isalnum(ch) != 0 || ch == '_') { std::size_t start = m_cursor++; diff --git a/furc/src/front/parser.cpp b/furc/src/front/parser.cpp index e187e66..4228e76 100644 --- a/furc/src/front/parser.cpp +++ b/furc/src/front/parser.cpp @@ -40,22 +40,22 @@ ast::node_handle parser::parse_declaration() { switch (first->type) { case token_t::Keyword: { auto first = next_token(); - switch (first->keyword) { + switch ((*first)->keyword) { case keyword_token::Func: { token_handle<> name = eat_token(token_t::Identifier); - if (name.error()) return { name.location(), name }; + if (name.has_error()) return { name.location(), name.error() }; auto tok = eat_token(token_t::Lparen); - if (tok.error()) return tok; + if (tok.has_error()) return tok; tok = eat_token(token_t::Rparen); - if (tok.error()) return tok; + if (tok.has_error()) return tok; const auto& peek = peek_token(); - if (peek.error()) return peek; + if (peek.has_error()) return peek; switch (peek->type) { case token_t::Lbrace: { ast::function_body_handle body = parse_body(); - if (body.error()) return body; + if (body.has_error()) return body; return ast::node_handle{ first.location(), m_arena, name, @@ -68,7 +68,7 @@ ast::node_handle parser::parse_declaration() { default: return { tok.location(), "unexpected token "s + tok->type }; } } - default: return { first.location(), "unexpected keyword "s + first->keyword }; + default: return { first.location(), "unexpected keyword "s + (*first)->keyword }; } } case token_t::None: @@ -90,12 +90,12 @@ ast::node_handle parser::parse_declaration() { ast::node_handle parser::parse_statement() { const auto& tok = peek_token(); - if (tok.error()) return tok; + if (tok.has_error()) return tok; switch (tok->type) { case token_t::Keyword: { next_token(); auto err = eat_token(token_t::Semicolon); - if (err.error()) return err; + if (err.has_error()) return err; return ast::node_handle{ tok.location(), m_arena }; } @@ -107,7 +107,7 @@ ast::node_handle parser::parse_statement() { auto expression = parse_expression(); if (expression.present()) { auto semi = eat_token(token_t::Semicolon); - if (semi.error()) return semi; + if (semi.has_error()) return semi; return std::move(expression); } @@ -130,14 +130,13 @@ ast::node_handle parser::parse_literal() { auto tok = next_token(); return ast::node_handle{ tok.location(), m_arena, - handle{ tok.location(), tok->value } }; + handle{ tok.location(), (*tok)->string } }; } case token_t::Integer: { auto tok = next_token(); - // return ast::node_handle{ tok.location(), - // m_arena, - // handle{ tok.location(), tok->value } }; - return { tok.location(), "unimplemented parse_literal() for token_t::Integer" }; + return ast::node_handle{ tok.location(), + m_arena, + handle{ tok.location(), (*tok)->integer } }; } default: break; } @@ -149,15 +148,15 @@ ast::function_body_handle parser::parse_body() { ast::function_body body; token_handle<> begin = eat_token(token_t::Lbrace); - if (begin.error()) return begin; + if (begin.has_error()) return begin; body.begin = begin.location(); - while (!peek_token().error() && peek_token()->type != token_t::None && peek_token()->type != token_t::Rbrace) { + while (!peek_token().has_error() && peek_token()->type != token_t::None && peek_token()->type != token_t::Rbrace) { body.statements.push_back(parse_statement()); } token_handle<> end = eat_token(token_t::Rbrace); - if (end.error()) return end; + if (end.has_error()) return end; body.end = end.location(); return { begin.location(), body }; diff --git a/furc/test/lexer.cpp b/furc/test/lexer.cpp index 3843cd0..867ae11 100644 --- a/furc/test/lexer.cpp +++ b/furc/test/lexer.cpp @@ -8,7 +8,13 @@ using namespace furc::front; using namespace std::string_view_literals; #define EXPECT_TOKEN(lexer, t) EXPECT_EQ((lexer).next_token(), (t)); -#define EXPECT_EMPTY_TOKEN(lexer) EXPECT_EQ((lexer).next_token(), (token{})); +#define EXPECT_EOF(lexer) EXPECT_EQ((lexer).next_token(), (token{})); +#define EXPECT_ERROR(lexer, err) \ + do { \ + auto handle = (lexer).next_token(); \ + EXPECT_TRUE(handle.has_error()); \ + EXPECT_EQ(handle.error(), (err)); \ + } while (0) TEST(Lexer, Tokens) { lexer lexer("", "()\n\t\t{\n}[\"shto-to\"]; :,.main return func"); @@ -26,7 +32,7 @@ TEST(Lexer, Tokens) { EXPECT_TOKEN(lexer, (token{ token_t::Identifier, "main"sv })); EXPECT_TOKEN(lexer, (token{ keyword_token::Return })); EXPECT_TOKEN(lexer, (token{ keyword_token::Func })); - EXPECT_EMPTY_TOKEN(lexer); + EXPECT_EOF(lexer); } TEST(Lexer, Comments) { @@ -35,7 +41,15 @@ TEST(Lexer, Comments) { EXPECT_TOKEN(lexer, (token{ keyword_token::Func })); EXPECT_TOKEN(lexer, (token{ token_t::Lbrace, "{"sv })); EXPECT_TOKEN(lexer, (token{ token_t::Rbrace, "}"sv })); - EXPECT_EMPTY_TOKEN(lexer); + EXPECT_EOF(lexer); +} + +TEST(Lexer, Integers) { + lexer lexer("", "67 18446744073709551615 18446744073709551616"); + EXPECT_TOKEN(lexer, (token{ 67ULL })); + EXPECT_TOKEN(lexer, (token{ 18446744073709551615ULL })); + EXPECT_ERROR(lexer, "integer 18446744073709551616 is too big"); + EXPECT_EOF(lexer); } } // namespace \ No newline at end of file diff --git a/furc/test/parser.cpp b/furc/test/parser.cpp index 35b2e7f..ab03ec3 100644 --- a/furc/test/parser.cpp +++ b/furc/test/parser.cpp @@ -18,7 +18,7 @@ TEST(Parser, EmptyFunctions) { EXPECT_TRUE(first.present()); EXPECT_EQ(first->declaration_type(), declaration_node_t::FunctionDefinition); node_handle funcDef = first; - EXPECT_EQ(funcDef->name().value, "main"); + EXPECT_EQ(funcDef->name()->string, "main"); EXPECT_EQ(funcDef->body()->statements.size(), 0); } { @@ -26,7 +26,7 @@ TEST(Parser, EmptyFunctions) { EXPECT_TRUE(second.present()); EXPECT_EQ(second->declaration_type(), declaration_node_t::FunctionDeclaration); node_handle funcDecl = second; - EXPECT_EQ(funcDecl->name().value, "foo"); + EXPECT_EQ(funcDecl->name()->string, "foo"); } }