From 3faafe371fb97c5c89e9250592eaf681c8c76672 Mon Sep 17 00:00:00 2001 From: CHatingPython Date: Wed, 5 Aug 2026 12:34:58 +0200 Subject: [PATCH] feat(furc): implement lexer --- furc/CMakeLists.txt | 2 +- furc/include/furc/front/lexer.hpp | 46 +++++++++ furc/include/furc/front/token.hpp | 86 ++++++++++++++++- furc/src/front/lexer.cpp | 154 ++++++++++++++++++++++++++++++ furc/src/main.cpp | 17 +++- 5 files changed, 297 insertions(+), 8 deletions(-) create mode 100644 furc/include/furc/front/lexer.hpp create mode 100644 furc/src/front/lexer.cpp diff --git a/furc/CMakeLists.txt b/furc/CMakeLists.txt index cc6876c..527bb5f 100644 --- a/furc/CMakeLists.txt +++ b/furc/CMakeLists.txt @@ -13,4 +13,4 @@ include(GoogleTest) file(GLOB_RECURSE FURC_TESTS "test/**.cpp") add_executable(furc_tests ${FURC_TESTS}) target_link_libraries(furc_tests PRIVATE libfurc GTest::gtest_main) -gtest_discover_tests(furc_tests) +# gtest_discover_tests(furc_tests) diff --git a/furc/include/furc/front/lexer.hpp b/furc/include/furc/front/lexer.hpp new file mode 100644 index 0000000..d625c75 --- /dev/null +++ b/furc/include/furc/front/lexer.hpp @@ -0,0 +1,46 @@ +#ifndef FURC_FRONT_LEXER_HPP +#define FURC_FRONT_LEXER_HPP + +#include "furc/front/token.hpp" + +#include +#include +#include + +namespace furc { + +class lexer { +public: + lexer(std::string_view filepath, std::string_view content) + : m_filepath(filepath), m_content(content) {} + + ~lexer() = default; + + lexer(lexer&&) noexcept = default; + lexer& operator=(lexer&&) noexcept = default; + + lexer(const lexer&) = delete; + lexer& operator=(const lexer&) = delete; +public: + token next_token(); + token peek_token(); + token skip_token(); +private: + void next(); + constexpr char get(std::size_t offset = 0) const; + void skip_spaces(); + + constexpr token::location location() const; +private: + std::string_view m_filepath; + std::string_view m_content; + std::size_t m_cursor = 0; + std::size_t m_row = 0; + std::size_t m_lineStart = 0; + + std::optional m_peekToken; +}; + +} // namespace furc + +#endif // FURC_FRONT_LEXER_HPP diff --git a/furc/include/furc/front/token.hpp b/furc/include/furc/front/token.hpp index f00a9bb..ff896ca 100644 --- a/furc/include/furc/front/token.hpp +++ b/furc/include/furc/front/token.hpp @@ -3,6 +3,7 @@ #include #include +#include #include namespace furc { @@ -12,6 +13,7 @@ struct token { Identifier = 0, Integer, String, + Char, LParen, /**< `(` */ RParen, /**< `)` */ @@ -87,6 +89,7 @@ struct token { // Errors: UnexpectedCharacter, + UnexpectedEOF, EndOfFile, } type; union value { @@ -110,15 +113,90 @@ struct token { value.integer = integer; } - token(location loc, std::string_view string) - : loc(loc), type(String) { + token(location loc, enum type type, std::string_view string) + : loc(loc), type(type) { value.string = string; } - token(location loc, char character) - : loc(loc), type(UnexpectedCharacter) { + token(location loc, enum type type, char character) + : loc(loc), type(type) { value.character = character; } + + friend std::ostream& operator<<(std::ostream& os, const token& token) { + switch (token.type) { + case token::Identifier: return os << token.value.string; + case token::String: return os << '"' << token.value.string << '"'; + case token::Char: return os << '\'' << token.value.character << '\''; + case token::Integer: return os << token.value.integer; + case token::LParen: return os << "("; + case token::RParen: return os << ")"; + case token::LBrace: return os << "{"; + case token::RBrace: return os << "}"; + case token::LBracket: return os << "["; + case token::RBracket: return os << "]"; + case token::Semicolon: return os << ";"; + case token::Colon: return os << ":"; + case token::Comma: return os << ","; + case token::Dot: return os << "."; + case token::Plus: return os << "+"; + case token::Minus: return os << "-"; + case token::Star: return os << "*"; + case token::Slash: return os << "/"; + case token::Percent: return os << "%"; + case token::Ampersand: return os << "&"; + case token::Pipe: return os << "|"; + case token::Hat: return os << "^"; + case token::DblAmpersand: return os << "&&"; + case token::DblPipe: return os << "||"; + case token::DblPlus: return os << "++"; + case token::DblMinus: return os << "--"; + case token::ExMark: return os << "!"; + case token::CatEars: return os << "^^"; + case token::Equals: return os << "="; + case token::PlusEquals: return os << "+="; + case token::MinusEquals: return os << "-="; + case token::StarEquals: return os << "*="; + case token::SlashEquals: return os << "/="; + case token::PercentEquals: return os << "%="; + case token::AmpersandEquals: return os << "&="; + case token::PipeEquals: return os << "|="; + case token::HatEquals: return os << "^="; + case token::DblEquals: return os << "=="; + case token::ExEquals: return os << "!="; + case token::LessThan: return os << "<"; + case token::LessEquals: return os << "<="; + case token::GreaterThan: return os << ">"; + case token::GreaterEquals: return os << ">="; + case token::SlimArrow: return os << "->"; + case token::FatArrow: return os << "=>"; + case token::Monkey: return os << "@"; + case token::Sha256: return os << "#"; + case token::Func: return os << "func"; + case token::Return: return os << "return"; + case token::If: return os << "if"; + case token::Else: return os << "else"; + case token::While: return os << "while"; + case token::Public: return os << "public"; + case token::Private: return os << "private"; + case token::Pointerof: return os << "pointerof"; + case token::Sizeof: return os << "sizeof"; + case token::Lengthof: return os << "lengthof"; + case token::S8: return os << "s8"; + case token::U8: return os << "u8"; + case token::S16: return os << "s16"; + case token::U16: return os << "u16"; + case token::S32: return os << "s32"; + case token::U32: return os << "u32"; + case token::S64: return os << "s64"; + case token::U64: return os << "u64"; + case token::UnexpectedCharacter: return os << "Unexpected character `" << token.value.character << "`"; + case token::UnexpectedEOF: return os << "Unexpected End Of File"; + case token::EndOfFile: return os << "End Of File"; + } + + return os; + } }; using token_t = enum token::type; diff --git a/furc/src/front/lexer.cpp b/furc/src/front/lexer.cpp new file mode 100644 index 0000000..42b48f5 --- /dev/null +++ b/furc/src/front/lexer.cpp @@ -0,0 +1,154 @@ +#include "furc/front/lexer.hpp" + +#include +#include +#include + +namespace furc { + +token lexer::next_token() { + if (m_peekToken.has_value()) { + auto tok = m_peekToken.value(); + m_peekToken = {}; + return tok; + } + + skip_spaces(); + + if (m_cursor >= m_content.size()) return { location(), token::EndOfFile }; + + auto loc = location(); + + if (std::isdigit(get()) != 0) {} + + if (std::isalnum(get()) != 0 || get() == '_') { + std::size_t begin = m_cursor; + next(); + while (m_cursor < m_content.size() && (std::isalnum(get()) != 0 || get() == '_')) + next(); + return { loc, token::Identifier, m_content.substr(begin, m_cursor - begin) }; + } + + if (get() == '"') { + next(); + + std::size_t begin = m_cursor; + while (m_cursor < m_content.size() && get() != '"') + next(); + if (m_cursor >= m_content.size()) return { location(), token::UnexpectedEOF }; + + next(); + return { loc, token::String, m_content.substr(begin, m_cursor - begin - 1) }; + } + + if (get() == '\'') { + next(); + bool slash = get() == '\\'; + if (slash) next(); + auto loc2 = location(); + char character = get(); + next(); + if (get() != '\'') return { location(), token::UnexpectedCharacter, get() }; + next(); + if (slash) { + switch (character) { + case '\\': character = '\\'; break; + case 'n': character = '\n'; break; + case 'r': character = '\r'; break; + case 't': character = '\t'; break; + default: return { loc2, token::UnexpectedCharacter, character }; + } + } + return { loc, token::Char, character }; + } + + static std::unordered_map s_tokens = { + { "(", token::LParen }, + { ")", token::RParen }, + { "{", token::LBrace }, + { "}", token::RBrace }, + { "[", token::LBracket }, + { "]", token::RBracket }, + { ";", token::Semicolon }, + { ":", token::Colon }, + { ",", token::Comma }, + { ".", token::Dot }, + { "+", token::Plus }, + { "-", token::Minus }, + { "*", token::Star }, + { "/", token::Slash }, + { "%", token::Percent }, + { "&", token::Ampersand }, + { "|", token::Pipe }, + { "^", token::Hat }, + { "&&", token::DblAmpersand }, + { "||", token::DblPipe }, + { "++", token::DblPlus }, + { "--", token::DblMinus }, + { "!", token::ExMark }, + { "^^", token::CatEars }, + { "=", token::Equals }, + { "+=", token::PlusEquals }, + { "-=", token::MinusEquals }, + { "*=", token::StarEquals }, + { "/=", token::SlashEquals }, + { "%=", token::PercentEquals }, + { "&=", token::AmpersandEquals }, + { "|=", token::PipeEquals }, + { "^=", token::HatEquals }, + { "==", token::DblEquals }, + { "!=", token::ExEquals }, + { "<", token::LessThan }, + { "<=", token::LessEquals }, + { ">", token::GreaterThan }, + { ">=", token::GreaterEquals }, + { "->", token::SlimArrow }, + { "=>", token::FatArrow }, + { "@", token::Monkey }, + { "#", token::Sha256 }, + }; + + std::size_t begin = m_cursor; + std::size_t len = 1; + while (begin + len - 1 < m_content.size() && s_tokens.find(m_content.substr(begin, len)) != s_tokens.end()) + ++len; + + if (len > 1) { + auto type = s_tokens[m_content.substr(begin, len - 1)]; + m_cursor += len - 1; + return { loc, type }; + } + + return { loc, token::UnexpectedCharacter, get() }; +} + +token lexer::peek_token() { + if (m_peekToken.has_value()) return m_peekToken.value(); + auto tok = next_token(); + m_peekToken = tok; + return tok; +} + +token lexer::skip_token() { + m_peekToken = {}; + return next_token(); +} + +void lexer::next() { + if (m_cursor < m_content.size()) ++m_cursor; +} + +constexpr char lexer::get(std::size_t offset) const { + return m_content[m_cursor + offset]; +} + +void lexer::skip_spaces() { + while (m_cursor < m_content.size() && std::isspace(get()) != 0) + ++m_cursor; +} + +constexpr token::location lexer::location() const { + return { m_filepath, m_row, m_cursor - m_lineStart }; +} + +} // namespace furc diff --git a/furc/src/main.cpp b/furc/src/main.cpp index 254b47a..7919698 100644 --- a/furc/src/main.cpp +++ b/furc/src/main.cpp @@ -1,7 +1,18 @@ +#include "furc/front/lexer.hpp" + #include int main(void) { - std::cout << "Farewell, stasiu!\n"; - - return 0; + furc::lexer lexer = { "", "func main(argc: u64) -> s32 { return '\\\\'; }" }; + while (true) { + furc::token token = lexer.next_token(); + std::cout << token.loc.filepath << ':' << token.loc.row + 1 << ':' << token.loc.col + 1 << ": " << token + << '\n'; + switch (token.type) { + case furc::token::UnexpectedCharacter: + case furc::token::UnexpectedEOF: return 1; + case furc::token::EndOfFile: return 0; + default: break; + } + } }