feat(furc): implement lexer

furc/CMakeLists.txt

@@ -13,4 +13,4 @@ include(GoogleTest)
 file(GLOB_RECURSE FURC_TESTS "test/**.cpp")
 add_executable(furc_tests ${FURC_TESTS})
 target_link_libraries(furc_tests PRIVATE libfurc GTest::gtest_main)
-gtest_discover_tests(furc_tests)
+# gtest_discover_tests(furc_tests)

furc/include/furc/front/lexer.hpp

@@ -0,0 +1,46 @@
+#ifndef FURC_FRONT_LEXER_HPP
+#define FURC_FRONT_LEXER_HPP
+
+#include "furc/front/token.hpp"
+
+#include <cstddef>
+#include <optional>
+#include <string_view>
+
+namespace furc {
+
+class lexer {
+public:
+    lexer(std::string_view filepath, std::string_view content)
+      : m_filepath(filepath), m_content(content) {}
+
+    ~lexer() = default;
+
+    lexer(lexer&&) noexcept            = default;
+    lexer& operator=(lexer&&) noexcept = default;
+
+    lexer(const lexer&)            = delete;
+    lexer& operator=(const lexer&) = delete;
+public:
+    token next_token();
+    token peek_token();
+    token skip_token();
+private:
+    void           next();
+    constexpr char get(std::size_t offset = 0) const;
+    void           skip_spaces();
+
+    constexpr token::location location() const;
+private:
+    std::string_view m_filepath;
+    std::string_view m_content;
+    std::size_t      m_cursor    = 0;
+    std::size_t      m_row       = 0;
+    std::size_t      m_lineStart = 0;
+
+    std::optional<token> m_peekToken;
+};
+
+} // namespace furc
+
+#endif // FURC_FRONT_LEXER_HPP

furc/include/furc/front/token.hpp

@@ -3,6 +3,7 @@
 
 #include <cstddef>
 #include <cstdint>
+#include <ostream>
 #include <string_view>
 
 namespace furc {
@@ -12,6 +13,7 @@ struct token {
         Identifier = 0,
         Integer,
         String,
+        Char,
 
         LParen,    /**< `(` */
         RParen,    /**< `)` */
@@ -87,6 +89,7 @@ struct token {
 
         // Errors:
         UnexpectedCharacter,
+        UnexpectedEOF,
         EndOfFile,
     } type;
     union value {
@@ -110,15 +113,90 @@ struct token {
         value.integer = integer;
     }
 
-    token(location loc, std::string_view string)
+    token(location loc, enum type type, std::string_view string)
-      : loc(loc), type(String) {
+      : loc(loc), type(type) {
         value.string = string;
     }
 
-    token(location loc, char character)
+    token(location loc, enum type type, char character)
-      : loc(loc), type(UnexpectedCharacter) {
+      : loc(loc), type(type) {
         value.character = character;
     }
+
+    friend std::ostream& operator<<(std::ostream& os, const token& token) {
+        switch (token.type) {
+        case token::Identifier: return os << token.value.string;
+        case token::String: return os << '"' << token.value.string << '"';
+        case token::Char: return os << '\'' << token.value.character << '\'';
+        case token::Integer: return os << token.value.integer;
+        case token::LParen: return os << "(";
+        case token::RParen: return os << ")";
+        case token::LBrace: return os << "{";
+        case token::RBrace: return os << "}";
+        case token::LBracket: return os << "[";
+        case token::RBracket: return os << "]";
+        case token::Semicolon: return os << ";";
+        case token::Colon: return os << ":";
+        case token::Comma: return os << ",";
+        case token::Dot: return os << ".";
+        case token::Plus: return os << "+";
+        case token::Minus: return os << "-";
+        case token::Star: return os << "*";
+        case token::Slash: return os << "/";
+        case token::Percent: return os << "%";
+        case token::Ampersand: return os << "&";
+        case token::Pipe: return os << "|";
+        case token::Hat: return os << "^";
+        case token::DblAmpersand: return os << "&&";
+        case token::DblPipe: return os << "||";
+        case token::DblPlus: return os << "++";
+        case token::DblMinus: return os << "--";
+        case token::ExMark: return os << "!";
+        case token::CatEars: return os << "^^";
+        case token::Equals: return os << "=";
+        case token::PlusEquals: return os << "+=";
+        case token::MinusEquals: return os << "-=";
+        case token::StarEquals: return os << "*=";
+        case token::SlashEquals: return os << "/=";
+        case token::PercentEquals: return os << "%=";
+        case token::AmpersandEquals: return os << "&=";
+        case token::PipeEquals: return os << "|=";
+        case token::HatEquals: return os << "^=";
+        case token::DblEquals: return os << "==";
+        case token::ExEquals: return os << "!=";
+        case token::LessThan: return os << "<";
+        case token::LessEquals: return os << "<=";
+        case token::GreaterThan: return os << ">";
+        case token::GreaterEquals: return os << ">=";
+        case token::SlimArrow: return os << "->";
+        case token::FatArrow: return os << "=>";
+        case token::Monkey: return os << "@";
+        case token::Sha256: return os << "#";
+        case token::Func: return os << "func";
+        case token::Return: return os << "return";
+        case token::If: return os << "if";
+        case token::Else: return os << "else";
+        case token::While: return os << "while";
+        case token::Public: return os << "public";
+        case token::Private: return os << "private";
+        case token::Pointerof: return os << "pointerof";
+        case token::Sizeof: return os << "sizeof";
+        case token::Lengthof: return os << "lengthof";
+        case token::S8: return os << "s8";
+        case token::U8: return os << "u8";
+        case token::S16: return os << "s16";
+        case token::U16: return os << "u16";
+        case token::S32: return os << "s32";
+        case token::U32: return os << "u32";
+        case token::S64: return os << "s64";
+        case token::U64: return os << "u64";
+        case token::UnexpectedCharacter: return os << "Unexpected character `" << token.value.character << "`";
+        case token::UnexpectedEOF: return os << "Unexpected End Of File";
+        case token::EndOfFile: return os << "End Of File";
+        }
+
+        return os;
+    }
 };
 
 using token_t = enum token::type;

furc/src/front/lexer.cpp

@@ -0,0 +1,154 @@
+#include "furc/front/lexer.hpp"
+
+#include <cctype>
+#include <string_view>
+#include <unordered_map>
+
+namespace furc {
+
+token lexer::next_token() {
+    if (m_peekToken.has_value()) {
+        auto tok    = m_peekToken.value();
+        m_peekToken = {};
+        return tok;
+    }
+
+    skip_spaces();
+
+    if (m_cursor >= m_content.size()) return { location(), token::EndOfFile };
+
+    auto loc = location();
+
+    if (std::isdigit(get()) != 0) {}
+
+    if (std::isalnum(get()) != 0 || get() == '_') {
+        std::size_t begin = m_cursor;
+        next();
+        while (m_cursor < m_content.size() && (std::isalnum(get()) != 0 || get() == '_'))
+            next();
+        return { loc, token::Identifier, m_content.substr(begin, m_cursor - begin) };
+    }
+
+    if (get() == '"') {
+        next();
+
+        std::size_t begin = m_cursor;
+        while (m_cursor < m_content.size() && get() != '"')
+            next();
+        if (m_cursor >= m_content.size()) return { location(), token::UnexpectedEOF };
+
+        next();
+        return { loc, token::String, m_content.substr(begin, m_cursor - begin - 1) };
+    }
+
+    if (get() == '\'') {
+        next();
+        bool slash = get() == '\\';
+        if (slash) next();
+        auto loc2      = location();
+        char character = get();
+        next();
+        if (get() != '\'') return { location(), token::UnexpectedCharacter, get() };
+        next();
+        if (slash) {
+            switch (character) {
+            case '\\': character = '\\'; break;
+            case 'n': character = '\n'; break;
+            case 'r': character = '\r'; break;
+            case 't': character = '\t'; break;
+            default: return { loc2, token::UnexpectedCharacter, character };
+            }
+        }
+        return { loc, token::Char, character };
+    }
+
+    static std::unordered_map<std::string_view, token_t> s_tokens = {
+        { "(", token::LParen },
+        { ")", token::RParen },
+        { "{", token::LBrace },
+        { "}", token::RBrace },
+        { "[", token::LBracket },
+        { "]", token::RBracket },
+        { ";", token::Semicolon },
+        { ":", token::Colon },
+        { ",", token::Comma },
+        { ".", token::Dot },
+        { "+", token::Plus },
+        { "-", token::Minus },
+        { "*", token::Star },
+        { "/", token::Slash },
+        { "%", token::Percent },
+        { "&", token::Ampersand },
+        { "|", token::Pipe },
+        { "^", token::Hat },
+        { "&&", token::DblAmpersand },
+        { "||", token::DblPipe },
+        { "++", token::DblPlus },
+        { "--", token::DblMinus },
+        { "!", token::ExMark },
+        { "^^", token::CatEars },
+        { "=", token::Equals },
+        { "+=", token::PlusEquals },
+        { "-=", token::MinusEquals },
+        { "*=", token::StarEquals },
+        { "/=", token::SlashEquals },
+        { "%=", token::PercentEquals },
+        { "&=", token::AmpersandEquals },
+        { "|=", token::PipeEquals },
+        { "^=", token::HatEquals },
+        { "==", token::DblEquals },
+        { "!=", token::ExEquals },
+        { "<", token::LessThan },
+        { "<=", token::LessEquals },
+        { ">", token::GreaterThan },
+        { ">=", token::GreaterEquals },
+        { "->", token::SlimArrow },
+        { "=>", token::FatArrow },
+        { "@", token::Monkey },
+        { "#", token::Sha256 },
+    };
+
+    std::size_t begin = m_cursor;
+    std::size_t len   = 1;
+    while (begin + len - 1 < m_content.size() && s_tokens.find(m_content.substr(begin, len)) != s_tokens.end())
+        ++len;
+
+    if (len > 1) {
+        auto type = s_tokens[m_content.substr(begin, len - 1)];
+        m_cursor += len - 1;
+        return { loc, type };
+    }
+
+    return { loc, token::UnexpectedCharacter, get() };
+}
+
+token lexer::peek_token() {
+    if (m_peekToken.has_value()) return m_peekToken.value();
+    auto tok    = next_token();
+    m_peekToken = tok;
+    return tok;
+}
+
+token lexer::skip_token() {
+    m_peekToken = {};
+    return next_token();
+}
+
+void lexer::next() {
+    if (m_cursor < m_content.size()) ++m_cursor;
+}
+
+constexpr char lexer::get(std::size_t offset) const {
+    return m_content[m_cursor + offset];
+}
+
+void lexer::skip_spaces() {
+    while (m_cursor < m_content.size() && std::isspace(get()) != 0)
+        ++m_cursor;
+}
+
+constexpr token::location lexer::location() const {
+    return { m_filepath, m_row, m_cursor - m_lineStart };
+}
+
+} // namespace furc

furc/src/main.cpp

@@ -1,7 +1,18 @@
+#include "furc/front/lexer.hpp"
+
 #include <iostream>
 
 int main(void) {
-    std::cout << "Farewell, stasiu!\n";
+    furc::lexer lexer = { "<AK>", "func main(argc: u64) -> s32 { return '\\\\'; }" };
-
+    while (true) {
-    return 0;
+        furc::token token = lexer.next_token();
+        std::cout << token.loc.filepath << ':' << token.loc.row + 1 << ':' << token.loc.col + 1 << ": " << token
+                  << '\n';
+        switch (token.type) {
+        case furc::token::UnexpectedCharacter:
+        case furc::token::UnexpectedEOF: return 1;
+        case furc::token::EndOfFile: return 0;
+        default: break;
+        }
+    }
 }