From 03448d865eb3bb8620dbb3e1689501aa6fdd6bec Mon Sep 17 00:00:00 2001 From: CHatingPython Date: Tue, 14 Jul 2026 00:38:27 +0200 Subject: [PATCH] feat(furas): introduce a basic lexer Closes: #56 --- furas/CMakeLists.txt | 3 +- furas/include/furas/lexer.hpp | 47 ++++++++++++++++++ furas/include/furas/token.hpp | 87 +++++++++++++++++++++++++++++++++ furas/src/lexer.cpp | 91 +++++++++++++++++++++++++++++++++++ furas/src/main.cpp | 15 +++++- 5 files changed, 239 insertions(+), 4 deletions(-) create mode 100644 furas/include/furas/lexer.hpp create mode 100644 furas/include/furas/token.hpp create mode 100644 furas/src/lexer.cpp diff --git a/furas/CMakeLists.txt b/furas/CMakeLists.txt index 04342d6..d94ea3f 100644 --- a/furas/CMakeLists.txt +++ b/furas/CMakeLists.txt @@ -1,6 +1,5 @@ file(GLOB_RECURSE FURAS_SRCS "src/**.cpp") file(GLOB_RECURSE FURAS_HDRS "include/**.hpp") add_executable(furas ${FURAS_SRCS} ${FURAS_HDRS}) -target_include_directories(furas PRIVATE include/) -set_target_properties(furas PROPERTIES PREFIX "") +target_include_directories(furas PUBLIC include/) target_link_libraries(furas PRIVATE furlang libfurvm) diff --git a/furas/include/furas/lexer.hpp b/furas/include/furas/lexer.hpp new file mode 100644 index 0000000..ba3297e --- /dev/null +++ b/furas/include/furas/lexer.hpp @@ -0,0 +1,47 @@ +#ifndef FURAS_LEXER_HPP +#define FURAS_LEXER_HPP + +#include "furas/token.hpp" +#include "furlang/result.hpp" + +#include +#include + +namespace furas { + +struct lexer_location { + std::string_view filename; + std::size_t row, col; +}; + +struct lexer_error { + enum type { + EndOfFile = 0, + UnknownCharacter, + } type; + + lexer_location location; + std::string message; +}; + +using token_r = furlang::result; + +class lexer { +public: + lexer(std::string_view filename, std::string_view content) + : m_filename(filename), m_content(content) {} + + token_r next_token(); +private: + constexpr lexer_location location() const { return { m_filename, m_cursor - m_lineStart, m_column }; } +private: + std::string_view m_filename; + std::string_view m_content; + std::size_t m_cursor = 0; + std::size_t m_lineStart = 0; + std::size_t m_column = 0; +}; + +} // namespace furas + +#endif // FURAS_LEXER_HPP diff --git a/furas/include/furas/token.hpp b/furas/include/furas/token.hpp new file mode 100644 index 0000000..5e464da --- /dev/null +++ b/furas/include/furas/token.hpp @@ -0,0 +1,87 @@ +#ifndef FURAS_TOKEN_HPP +#define FURAS_TOKEN_HPP + +#include +#include + +namespace furas { + +struct token { + enum type { + Identifier = 0, /**< An identifier. */ + Signed, /**< A signed integer. */ + Unsigned, /**< An unsigned integer. */ + + // Markers + Monkey, /**< Constant marker (`@`). */ + Dolar, /**< Type marker(`$`). */ + Sha256, /**< Label marker(`#`). */ + Percent, /**< Variable marker(`%`). The more the better. */ + + Dot, /**< . */ + + // Keywords + Import, /**< `import` keyword for importing functions and types. */ + Public, /**< `public` access specifier. */ + Private, /**< `private` access specifier. */ + + // Instructions + Push, + Array, + Get, + Drop, + Dup, + Clone, + Ref, + Add, + Sub, + Mul, + Div, + Mod, + Eq, + Neq, + Lt, + Gt, + Le, + Ge, + Ptrof, + Sizeof, + Lenof, + Load, + Store, + Call, + Jump, + Jnz, + Ret, + + Count + } type = Count; + union value { + std::nullptr_t null = nullptr; + std::string_view string; + std::int64_t integer; + std::uint64_t uint; + } value; + + token(enum type type) + : type(type) {} + + token(enum type type, std::string_view string) + : type(type) { + value.string = string; + } + + token(std::uint64_t num) + : type(Unsigned) { + value.uint = num; + } + + token(std::int64_t num) + : type(Signed) { + value.integer = num; + } +}; + +} // namespace furas + +#endif // FURAS_TOKEN_HPP diff --git a/furas/src/lexer.cpp b/furas/src/lexer.cpp new file mode 100644 index 0000000..0d04bd5 --- /dev/null +++ b/furas/src/lexer.cpp @@ -0,0 +1,91 @@ +#include "furas/lexer.hpp" + +#include +#include + +namespace furas { + +using namespace std::string_literals; + +token_r lexer::next_token() { + while (m_cursor < m_content.size() && std::isspace(m_content[m_cursor]) != 0) { + if (m_content[m_cursor] == '\n') { + m_lineStart = m_cursor + 1; + ++m_column; + } + ++m_cursor; + } + + // TODO: Add support for single-line comments + // TODO: Add support for multi-line comments + + if (m_cursor >= m_content.size()) return token_r{ lexer_error{ lexer_error::EndOfFile, location() } }; + + // TODO: Add support for negative integers (I am positive thanks to stasiu :v:) + // TODO: Add support for hexadecimal and binary numeric literals + if (std::isdigit(m_content[m_cursor]) != 0) { + std::uint64_t num = 0; + while (m_cursor < m_content.size() && std::isdigit(m_content[m_cursor]) != 0) { + num *= 10; + num += m_content[m_cursor++] - '0'; + } + return { num }; + } + + static std::unordered_map s_tokens = { + { "import", token::Import }, + { "public", token::Public }, + { "private", token::Private }, + + { "push", token::Push }, + { "array", token::Array }, + { "get", token::Get }, + { "drop", token::Drop }, + { "dup", token::Dup }, + { "clone", token::Clone }, + { "ref", token::Ref }, + { "add", token::Add }, + { "sub", token::Sub }, + { "mul", token::Mul }, + { "div", token::Div }, + { "mod", token::Mod }, + { "eq", token::Eq }, + { "neq", token::Neq }, + { "lt", token::Lt }, + { "gt", token::Gt }, + { "le", token::Le }, + { "ge", token::Ge }, + { "ptrof", token::Ptrof }, + { "sizeof", token::Sizeof }, + { "lenof", token::Lenof }, + { "load", token::Load }, + { "store", token::Store }, + { "call", token::Call }, + { "jump", token::Jump }, + { "jnz", token::Jnz }, + { "ret", token::Ret }, + }; + + if (std::isalnum(m_content[m_cursor]) != 0) { + std::size_t begin = m_cursor++; + while (m_cursor < m_content.size() && (std::isalnum(m_content[m_cursor]) != 0 || m_content[m_cursor] == '_')) + ++m_cursor; + std::string_view str = m_content.substr(begin, m_cursor - begin); + if (auto it = s_tokens.find(str); it != s_tokens.end()) return { it->second }; + return { token::Identifier, str }; + } + + switch (m_content[m_cursor]) { + case '@': ++m_cursor; return { token::Monkey }; + case '$': ++m_cursor; return { token::Dolar }; + case '#': ++m_cursor; return { token::Sha256 }; + case '%': ++m_cursor; return { token::Percent }; + case '.': ++m_cursor; return { token::Dot }; + default: + return token_r{ + lexer_error{ lexer_error::UnknownCharacter, location(), "Unknown character '"s + m_content[m_cursor] + "'" } + }; + } +} + +} // namespace furas diff --git a/furas/src/main.cpp b/furas/src/main.cpp index 07844bd..b797a8a 100644 --- a/furas/src/main.cpp +++ b/furas/src/main.cpp @@ -1,7 +1,18 @@ +#include "furas/lexer.hpp" + #include -int main(void) { - std::cout << "Hello, Stasiu!\n"; +int main(void) { // NOLINT + std::string source = "%uwu 10 public private int le"; + furas::lexer lexer("", source); + while (true) { + furas::token_r token = lexer.next_token(); + if (token.has_error()) { + std::cout << token.error().message << '\n'; + break; + } + std::cout << token->type << '\n'; + } return 0; }