Add integers to lexer

Signed-off-by: CHatingPython <chatingpython@gmail.com>

furc/include/furc/ast/declaration.hpp

@@ -42,7 +42,9 @@ public:
 
     front::token name() const { return m_name; }
 public:
-    std::ostream& print(std::ostream& os) const override { return os << "function " << m_name.value << " declaration"; }
+    std::ostream& print(std::ostream& os) const override {
+        return os << "function " << m_name->string << " declaration";
+    }
 protected:
     front::token m_name;
 };
@@ -73,13 +75,14 @@ public:
     const function_body_handle& body() const { return m_body; }
 public:
     std::ostream& print(std::ostream& os) const override {
-        os << "function " << m_name.value << " definition:";
+        function_declarartion_node::print(os);
+        os << ':';
         if (m_body.present()) {
             for (const auto& entry : m_body->statements)
                 os << '\n' << entry;
-            return os << '\n' << m_body->end << ": " << m_name.value << " end";
+            return os << '\n' << m_body->end << ": " << m_name->string << " end";
         }
-        return os << (std::string)m_body; // error
+        return os << m_body.error(); // error
     }
 private:
     function_body_handle m_body;

furc/include/furc/ast/literal.hpp

@@ -3,6 +3,7 @@
 
 #include "furc/ast/expression.hpp"
 #include "furc/ast/node.hpp"
+#include "furc/front/token.hpp"
 
 #include <ostream>
 
@@ -14,13 +15,6 @@ enum class literal_node_t {
     Integer
 };
 
-static inline std::ostream& operator<<(std::ostream& os, literal_node_t type) {
-    switch (type) {
-    case literal_node_t::String: return os << "string";
-    case literal_node_t::Integer: return os << "integer";
-    }
-}
-
 class literal_node : public expression_node {
 public:
     node_t category() const override { return node_t::Literal; }
@@ -40,8 +34,8 @@ public:
     const handle<std::string_view>& value() const { return m_value; }
 public:
     std::ostream& print(std::ostream& os) const override {
-        if (m_value.error()) return os << (std::string)m_value;
+        if (m_value.has_error()) return os << (std::string)m_value;
-        return os << '"' << *m_value << '"';
+        return os << "string literal (" << *m_value << ")";
     }
 private:
     handle<std::string_view> m_value;
@@ -49,14 +43,19 @@ private:
 
 class integer_literal_node : public literal_node {
 public:
-    integer_literal_node(handle<std::uint64_t>&& value)
+    integer_literal_node(handle<front::integer_token>&& value)
       : m_value(std::move(value)) {}
 public:
     literal_node_t literal_type() const override { return literal_node_t::Integer; }
 
-    const handle<std::uint64_t>& value() const { return m_value; }
+    const handle<front::integer_token>& value() const { return m_value; }
+public:
+    std::ostream& print(std::ostream& os) const override {
+        if (m_value.has_error()) return os << m_value.error();
+        return os << "integer literal (" << *m_value << ")";
+    }
 private:
-    handle<std::uint64_t> m_value;
+    handle<front::integer_token> m_value;
 };
 
 } // namespace ast

furc/include/furc/front/token.hpp

@@ -13,8 +13,8 @@ namespace front {
 enum class token_t {
     None,
     Identifier,
-    Keyword,
     String,
+    Keyword,
     Integer,
     Lparen,
     Rparen,
@@ -32,8 +32,8 @@ static inline std::ostream& operator<<(std::ostream& os, token_t type) {
     switch (type) {
     case token_t::None: return os << "none";
     case token_t::Identifier: return os << "identifier";
-    case token_t::Keyword: return os << "keyword";
     case token_t::String: return os << "string";
+    case token_t::Keyword: return os << "keyword";
     case token_t::Integer: return os << "integer";
     case token_t::Lparen: return os << "'('";
     case token_t::Rparen: return os << "')'";
@@ -52,8 +52,8 @@ static inline std::string operator+(const std::string& str, token_t type) {
     switch (type) {
     case token_t::None: return str + "none";
     case token_t::Identifier: return str + "identifier";
-    case token_t::Keyword: return str + "keyword";
     case token_t::String: return str + "string";
+    case token_t::Keyword: return str + "keyword";
     case token_t::Integer: return str + "integer";
     case token_t::Lparen: return str + "'('";
     case token_t::Rparen: return str + "')'";
@@ -90,10 +90,28 @@ static inline std::string operator+(const std::string& str, keyword_token keywor
     }
 }
 
+using integer_token = std::uint64_t;
+
 struct token {
-    token_t          type    = token_t::None;
+    token_t type = token_t::None;
-    keyword_token    keyword = keyword_token::None;
+    union value {
-    std::string_view value;
+        std::nullptr_t   null;
+        std::string_view string;
+        keyword_token    keyword;
+        integer_token    integer;
+
+        value(std::nullptr_t value = nullptr)
+          : null(value) {}
+
+        value(std::string_view value)
+          : string(value) {}
+
+        value(keyword_token value)
+          : keyword(value) {}
+
+        value(integer_token value)
+          : integer(value) {}
+    } value;
 
     token() = default;
 
@@ -101,15 +119,21 @@ struct token {
       : type(type), value(value) {}
 
     token(keyword_token keyword, std::string_view value = {})
-      : type(token_t::Keyword), keyword(keyword), value(value) {}
+      : type(token_t::Keyword), value(keyword) {}
+
+    token(integer_token integer)
+      : type(token_t::Integer), value(integer) {}
+
+    union value*       operator->() { return &value; }
+    const union value* operator->() const { return &value; }
 
     bool operator==(const token& rhs) const {
         if (type != rhs.type) return false;
         switch (type) {
         case token_t::Identifier:
-        case token_t::String:
+        case token_t::String: return value.string == rhs.value.string;
-        case token_t::Integer: return value == rhs.value;
+        case token_t::Keyword: return value.keyword == rhs.value.keyword;
-        case token_t::Keyword: return keyword == rhs.keyword;
+        case token_t::Integer: return value.integer == rhs.value.integer;
         case token_t::None:
         case token_t::Lparen:
         case token_t::Rparen:
@@ -128,9 +152,9 @@ struct token {
 static inline std::ostream& operator<<(std::ostream& os, const token& token) {
     switch (token.type) {
     case token_t::Identifier:
-    case token_t::String:
+    case token_t::String: return os << token.value.string;
-    case token_t::Integer: return os << token.value;
+    case token_t::Keyword: return os << token.value.keyword;
-    case token_t::Keyword: return os << token.keyword;
+    case token_t::Integer: return os << token.value.integer;
     default: return os << token.type;
     }
 }

furc/include/furc/handle.hpp

@@ -40,7 +40,7 @@ public:
 
     template <typename U>
     handle(const handle<U, Error>& error)
-      : m_location(error.location()), m_error(error) {}
+      : m_location(error.location()), m_error(error.error()) {}
 
     ~handle() = default;
 
@@ -65,17 +65,18 @@ public:
     location location() const { return m_location; }
 
     bool present() const { return m_value.has_value(); }
-    bool error() const { return !m_value.has_value(); }
+    bool has_error() const { return !m_value.has_value(); }
 
     value_type move() {
-        value_type value = *m_value;
+        value_type value = std::move(*m_value);
         m_value.reset();
         return value;
     }
 
+    Error error() const { return m_error; }
+
     operator reference() { return *m_value; }
     operator const_reference() const { return *m_value; }
-    operator Error() const { return m_error; }
 
     reference       operator*() { return *m_value; }
     const_reference operator*() const { return *m_value; }
@@ -133,7 +134,7 @@ public:
 
     template <typename U>
     handle(const handle<U, Error>& error)
-      : m_location(error.location()), m_error(error) {}
+      : m_location(error.location()), m_error(error.error()) {}
 
     ~handle() = default;
 
@@ -172,13 +173,13 @@ public:
     location location() const { return m_location; }
 
     bool present() const { return m_value != nullptr; }
-    bool error() const { return m_value == nullptr; }
+    bool has_error() const { return m_value == nullptr; }
 
     std::shared_ptr<value_type> shared() { return m_value; }
+    Error                       error() const { return m_error; }
 
     operator reference() { return *m_value; }
     operator const_reference() const { return *m_value; }
-    operator Error() const { return m_error; }
 
     reference       operator*() { return *m_value; }
     const_reference operator*() const { return *m_value; }

furc/src/front/lexer.cpp

@@ -1,6 +1,7 @@
 #include "furc/front/lexer.hpp"
 
 #include <cctype>
+#include <limits>
 #include <string>
 #include <unordered_map>
 
@@ -65,7 +66,27 @@ token_handle<> lexer::next_token() {
     }
     case std::char_traits<char>::eof(): return { location, token_t::None };
     default: {
-        if (std::isdigit(ch) != 0) {}
+        if (std::isdigit(ch) != 0) {
+            integer_token integer    = 0;
+            integer_token max        = std::numeric_limits<integer_token>::max();
+            integer_token upperBound = max / 10;
+
+            std::size_t start = m_cursor;
+            while (std::isdigit(get()) != 0) {
+                integer_token digit = get() - '0';
+
+                if (integer > upperBound || integer == upperBound && (integer - upperBound + digit) > (max % 10)) {
+                    while (std::isdigit(get()) != 0)
+                        ++m_cursor;
+                    return { location, "integer "s.append(m_content.substr(start, m_cursor - start)) + " is too big" };
+                }
+                integer *= 10;
+                integer += digit;
+                ++m_cursor;
+            }
+
+            return { location, integer };
+        }
 
         if (std::isalnum(ch) != 0 || ch == '_') {
             std::size_t start = m_cursor++;

furc/src/front/parser.cpp

@@ -40,22 +40,22 @@ ast::node_handle<ast::declaration_node> parser::parse_declaration() {
     switch (first->type) {
     case token_t::Keyword: {
         auto first = next_token();
-        switch (first->keyword) {
+        switch ((*first)->keyword) {
         case keyword_token::Func: {
             token_handle<> name = eat_token(token_t::Identifier);
-            if (name.error()) return { name.location(), name };
+            if (name.has_error()) return { name.location(), name.error() };
 
             auto tok = eat_token(token_t::Lparen);
-            if (tok.error()) return tok;
+            if (tok.has_error()) return tok;
             tok = eat_token(token_t::Rparen);
-            if (tok.error()) return tok;
+            if (tok.has_error()) return tok;
 
             const auto& peek = peek_token();
-            if (peek.error()) return peek;
+            if (peek.has_error()) return peek;
             switch (peek->type) {
             case token_t::Lbrace: {
                 ast::function_body_handle body = parse_body();
-                if (body.error()) return body;
+                if (body.has_error()) return body;
                 return ast::node_handle<ast::function_definition_node>{ first.location(),
                     m_arena,
                     name,
@@ -68,7 +68,7 @@ ast::node_handle<ast::declaration_node> parser::parse_declaration() {
             default: return { tok.location(), "unexpected token "s + tok->type };
             }
         }
-        default: return { first.location(), "unexpected keyword "s + first->keyword };
+        default: return { first.location(), "unexpected keyword "s + (*first)->keyword };
         }
     }
     case token_t::None:
@@ -90,12 +90,12 @@ ast::node_handle<ast::declaration_node> parser::parse_declaration() {
 
 ast::node_handle<ast::statement_node> parser::parse_statement() {
     const auto& tok = peek_token();
-    if (tok.error()) return tok;
+    if (tok.has_error()) return tok;
     switch (tok->type) {
     case token_t::Keyword: {
         next_token();
         auto err = eat_token(token_t::Semicolon);
-        if (err.error()) return err;
+        if (err.has_error()) return err;
 
         return ast::node_handle<ast::return_statement_node>{ tok.location(), m_arena };
     }
@@ -107,7 +107,7 @@ ast::node_handle<ast::statement_node> parser::parse_statement() {
     auto expression = parse_expression();
     if (expression.present()) {
         auto semi = eat_token(token_t::Semicolon);
-        if (semi.error()) return semi;
+        if (semi.has_error()) return semi;
         return std::move(expression);
     }
 
@@ -130,14 +130,13 @@ ast::node_handle<ast::literal_node> parser::parse_literal() {
         auto tok = next_token();
         return ast::node_handle<ast::string_literal_node>{ tok.location(),
             m_arena,
-            handle<std::string_view>{ tok.location(), tok->value } };
+            handle<std::string_view>{ tok.location(), (*tok)->string } };
     }
     case token_t::Integer: {
         auto tok = next_token();
-        // return ast::node_handle<ast::integer_literal_node>{ tok.location(),
+        return ast::node_handle<ast::integer_literal_node>{ tok.location(),
-        //     m_arena,
+            m_arena,
-        //     handle<std::uint64_t>{ tok.location(), tok->value } };
+            handle<integer_token>{ tok.location(), (*tok)->integer } };
-        return { tok.location(), "unimplemented parse_literal() for token_t::Integer" };
     }
     default: break;
     }
@@ -149,15 +148,15 @@ ast::function_body_handle parser::parse_body() {
     ast::function_body body;
 
     token_handle<> begin = eat_token(token_t::Lbrace);
-    if (begin.error()) return begin;
+    if (begin.has_error()) return begin;
     body.begin = begin.location();
 
-    while (!peek_token().error() && peek_token()->type != token_t::None && peek_token()->type != token_t::Rbrace) {
+    while (!peek_token().has_error() && peek_token()->type != token_t::None && peek_token()->type != token_t::Rbrace) {
         body.statements.push_back(parse_statement());
     }
 
     token_handle<> end = eat_token(token_t::Rbrace);
-    if (end.error()) return end;
+    if (end.has_error()) return end;
     body.end = end.location();
 
     return { begin.location(), body };

furc/test/lexer.cpp

@@ -8,7 +8,13 @@ using namespace furc::front;
 using namespace std::string_view_literals;
 
 #define EXPECT_TOKEN(lexer, t) EXPECT_EQ((lexer).next_token(), (t));
-#define EXPECT_EMPTY_TOKEN(lexer) EXPECT_EQ((lexer).next_token(), (token{}));
+#define EXPECT_EOF(lexer) EXPECT_EQ((lexer).next_token(), (token{}));
+#define EXPECT_ERROR(lexer, err)                                                                                       \
+    do {                                                                                                               \
+        auto handle = (lexer).next_token();                                                                            \
+        EXPECT_TRUE(handle.has_error());                                                                               \
+        EXPECT_EQ(handle.error(), (err));                                                                              \
+    } while (0)
 
 TEST(Lexer, Tokens) {
     lexer lexer("<TEMP>", "()\n\t\t{\n}[\"shto-to\"];    :,.main return func");
@@ -26,7 +32,7 @@ TEST(Lexer, Tokens) {
     EXPECT_TOKEN(lexer, (token{ token_t::Identifier, "main"sv }));
     EXPECT_TOKEN(lexer, (token{ keyword_token::Return }));
     EXPECT_TOKEN(lexer, (token{ keyword_token::Func }));
-    EXPECT_EMPTY_TOKEN(lexer);
+    EXPECT_EOF(lexer);
 }
 
 TEST(Lexer, Comments) {
@@ -35,7 +41,15 @@ TEST(Lexer, Comments) {
     EXPECT_TOKEN(lexer, (token{ keyword_token::Func }));
     EXPECT_TOKEN(lexer, (token{ token_t::Lbrace, "{"sv }));
     EXPECT_TOKEN(lexer, (token{ token_t::Rbrace, "}"sv }));
-    EXPECT_EMPTY_TOKEN(lexer);
+    EXPECT_EOF(lexer);
+}
+
+TEST(Lexer, Integers) {
+    lexer lexer("<TEMP>", "67 18446744073709551615 18446744073709551616");
+    EXPECT_TOKEN(lexer, (token{ 67ULL }));
+    EXPECT_TOKEN(lexer, (token{ 18446744073709551615ULL }));
+    EXPECT_ERROR(lexer, "integer 18446744073709551616 is too big");
+    EXPECT_EOF(lexer);
 }
 
 } // namespace

furc/test/parser.cpp

@@ -18,7 +18,7 @@ TEST(Parser, EmptyFunctions) {
         EXPECT_TRUE(first.present());
         EXPECT_EQ(first->declaration_type(), declaration_node_t::FunctionDefinition);
         node_handle<function_definition_node> funcDef = first;
-        EXPECT_EQ(funcDef->name().value, "main");
+        EXPECT_EQ(funcDef->name()->string, "main");
         EXPECT_EQ(funcDef->body()->statements.size(), 0);
     }
     {
@@ -26,7 +26,7 @@ TEST(Parser, EmptyFunctions) {
         EXPECT_TRUE(second.present());
         EXPECT_EQ(second->declaration_type(), declaration_node_t::FunctionDeclaration);
         node_handle<function_declarartion_node> funcDecl = second;
-        EXPECT_EQ(funcDecl->name().value, "foo");
+        EXPECT_EQ(funcDecl->name()->string, "foo");
     }
 }