KPGPMC/furlang
5
4
Fork 2
Code Issues 8 Pull Requests Actions Packages Projects 2 Releases Wiki Activity

feat(furc): implement lexer

Browse Source
This commit is contained in:
Aleksander Krajewski
2026-08-05 12:34:58 +02:00
parent 4604f5186e
commit 3faafe371f
5 changed files with 297 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files
furc/CMakeLists.txt
+1 -1
View File
@@ -13,4 +13,4 @@ include(GoogleTest)
file(GLOB_RECURSE FURC_TESTS "test/**.cpp")
add_executable(furc_tests ${FURC_TESTS})
target_link_libraries(furc_tests PRIVATE libfurc GTest::gtest_main)
gtest_discover_tests(furc_tests)
# gtest_discover_tests(furc_tests)
furc/include/furc/front/lexer.hpp
+46
View File
@@ -0,0 +1,46 @@
#ifndef FURC_FRONT_LEXER_HPP
#define FURC_FRONT_LEXER_HPP
#include "furc/front/token.hpp"
#include <cstddef>
#include <optional>
#include <string_view>
namespace furc {
class lexer {
public:
lexer(std::string_view filepath, std::string_view content)
: m_filepath(filepath), m_content(content) {}
~lexer() = default;
lexer(lexer&&) noexcept = default;
lexer& operator=(lexer&&) noexcept = default;
lexer(const lexer&) = delete;
lexer& operator=(const lexer&) = delete;
public:
token next_token();
token peek_token();
token skip_token();
private:
void next();
constexpr char get(std::size_t offset = 0) const;
void skip_spaces();
constexpr token::location location() const;
private:
std::string_view m_filepath;
std::string_view m_content;
std::size_t m_cursor = 0;
std::size_t m_row = 0;
std::size_t m_lineStart = 0;
std::optional<token> m_peekToken;
};
} // namespace furc
#endif // FURC_FRONT_LEXER_HPP
furc/include/furc/front/token.hpp
+82 -4
View File
@@ -3,6 +3,7 @@
#include <cstddef>
#include <cstdint>
#include <ostream>
#include <string_view>
namespace furc {
@@ -12,6 +13,7 @@ struct token {
Identifier = 0,
Integer,
String,
Char,
LParen, /**< `(` */
RParen, /**< `)` */
@@ -87,6 +89,7 @@ struct token {
// Errors:
UnexpectedCharacter,
UnexpectedEOF,
EndOfFile,
} type;
union value {
@@ -110,15 +113,90 @@ struct token {
value.integer = integer;
}
token(location loc, std::string_view string)
: loc(loc), type(String) {
token(location loc, enum type type, std::string_view string)
: loc(loc), type(type) {
value.string = string;
}
token(location loc, char character)
: loc(loc), type(UnexpectedCharacter) {
token(location loc, enum type type, char character)
: loc(loc), type(type) {
value.character = character;
}
friend std::ostream& operator<<(std::ostream& os, const token& token) {
switch (token.type) {
case token::Identifier: return os << token.value.string;
case token::String: return os << '"' << token.value.string << '"';
case token::Char: return os << '\'' << token.value.character << '\'';
case token::Integer: return os << token.value.integer;
case token::LParen: return os << "(";
case token::RParen: return os << ")";
case token::LBrace: return os << "{";
case token::RBrace: return os << "}";
case token::LBracket: return os << "[";
case token::RBracket: return os << "]";
case token::Semicolon: return os << ";";
case token::Colon: return os << ":";
case token::Comma: return os << ",";
case token::Dot: return os << ".";
case token::Plus: return os << "+";
case token::Minus: return os << "-";
case token::Star: return os << "*";
case token::Slash: return os << "/";
case token::Percent: return os << "%";
case token::Ampersand: return os << "&";
case token::Pipe: return os << "|";
case token::Hat: return os << "^";
case token::DblAmpersand: return os << "&&";
case token::DblPipe: return os << "||";
case token::DblPlus: return os << "++";
case token::DblMinus: return os << "--";
case token::ExMark: return os << "!";
case token::CatEars: return os << "^^";
case token::Equals: return os << "=";
case token::PlusEquals: return os << "+=";
case token::MinusEquals: return os << "-=";
case token::StarEquals: return os << "*=";
case token::SlashEquals: return os << "/=";
case token::PercentEquals: return os << "%=";
case token::AmpersandEquals: return os << "&=";
case token::PipeEquals: return os << "|=";
case token::HatEquals: return os << "^=";
case token::DblEquals: return os << "==";
case token::ExEquals: return os << "!=";
case token::LessThan: return os << "<";
case token::LessEquals: return os << "<=";
case token::GreaterThan: return os << ">";
case token::GreaterEquals: return os << ">=";
case token::SlimArrow: return os << "->";
case token::FatArrow: return os << "=>";
case token::Monkey: return os << "@";
case token::Sha256: return os << "#";
case token::Func: return os << "func";
case token::Return: return os << "return";
case token::If: return os << "if";
case token::Else: return os << "else";
case token::While: return os << "while";
case token::Public: return os << "public";
case token::Private: return os << "private";
case token::Pointerof: return os << "pointerof";
case token::Sizeof: return os << "sizeof";
case token::Lengthof: return os << "lengthof";
case token::S8: return os << "s8";
case token::U8: return os << "u8";
case token::S16: return os << "s16";
case token::U16: return os << "u16";
case token::S32: return os << "s32";
case token::U32: return os << "u32";
case token::S64: return os << "s64";
case token::U64: return os << "u64";
case token::UnexpectedCharacter: return os << "Unexpected character `" << token.value.character << "`";
case token::UnexpectedEOF: return os << "Unexpected End Of File";
case token::EndOfFile: return os << "End Of File";
}
return os;
}
};
using token_t = enum token::type;
furc/src/front/lexer.cpp
+154
View File
@@ -0,0 +1,154 @@
#include "furc/front/lexer.hpp"
#include <cctype>
#include <string_view>
#include <unordered_map>
namespace furc {
token lexer::next_token() {
if (m_peekToken.has_value()) {
auto tok = m_peekToken.value();
m_peekToken = {};
return tok;
}
skip_spaces();
if (m_cursor >= m_content.size()) return { location(), token::EndOfFile };
auto loc = location();
if (std::isdigit(get()) != 0) {}
if (std::isalnum(get()) != 0 || get() == '_') {
std::size_t begin = m_cursor;
next();
while (m_cursor < m_content.size() && (std::isalnum(get()) != 0 || get() == '_'))
next();
return { loc, token::Identifier, m_content.substr(begin, m_cursor - begin) };
}
if (get() == '"') {
next();
std::size_t begin = m_cursor;
while (m_cursor < m_content.size() && get() != '"')
next();
if (m_cursor >= m_content.size()) return { location(), token::UnexpectedEOF };
next();
return { loc, token::String, m_content.substr(begin, m_cursor - begin - 1) };
}
if (get() == '\'') {
next();
bool slash = get() == '\\';
if (slash) next();
auto loc2 = location();
char character = get();
next();
if (get() != '\'') return { location(), token::UnexpectedCharacter, get() };
next();
if (slash) {
switch (character) {
case '\\': character = '\\'; break;
case 'n': character = '\n'; break;
case 'r': character = '\r'; break;
case 't': character = '\t'; break;
default: return { loc2, token::UnexpectedCharacter, character };
}
}
return { loc, token::Char, character };
}
static std::unordered_map<std::string_view, token_t> s_tokens = {
{ "(", token::LParen },
{ ")", token::RParen },
{ "{", token::LBrace },
{ "}", token::RBrace },
{ "[", token::LBracket },
{ "]", token::RBracket },
{ ";", token::Semicolon },
{ ":", token::Colon },
{ ",", token::Comma },
{ ".", token::Dot },
{ "+", token::Plus },
{ "-", token::Minus },
{ "*", token::Star },
{ "/", token::Slash },
{ "%", token::Percent },
{ "&", token::Ampersand },
{ "|", token::Pipe },
{ "^", token::Hat },
{ "&&", token::DblAmpersand },
{ "||", token::DblPipe },
{ "++", token::DblPlus },
{ "--", token::DblMinus },
{ "!", token::ExMark },
{ "^^", token::CatEars },
{ "=", token::Equals },
{ "+=", token::PlusEquals },
{ "-=", token::MinusEquals },
{ "*=", token::StarEquals },
{ "/=", token::SlashEquals },
{ "%=", token::PercentEquals },
{ "&=", token::AmpersandEquals },
{ "|=", token::PipeEquals },
{ "^=", token::HatEquals },
{ "==", token::DblEquals },
{ "!=", token::ExEquals },
{ "<", token::LessThan },
{ "<=", token::LessEquals },
{ ">", token::GreaterThan },
{ ">=", token::GreaterEquals },
{ "->", token::SlimArrow },
{ "=>", token::FatArrow },
{ "@", token::Monkey },
{ "#", token::Sha256 },
};
std::size_t begin = m_cursor;
std::size_t len = 1;
while (begin + len - 1 < m_content.size() && s_tokens.find(m_content.substr(begin, len)) != s_tokens.end())
++len;
if (len > 1) {
auto type = s_tokens[m_content.substr(begin, len - 1)];
m_cursor += len - 1;
return { loc, type };
}
return { loc, token::UnexpectedCharacter, get() };
}
token lexer::peek_token() {
if (m_peekToken.has_value()) return m_peekToken.value();
auto tok = next_token();
m_peekToken = tok;
return tok;
}
token lexer::skip_token() {
m_peekToken = {};
return next_token();
}
void lexer::next() {
if (m_cursor < m_content.size()) ++m_cursor;
}
constexpr char lexer::get(std::size_t offset) const {
return m_content[m_cursor + offset];
}
void lexer::skip_spaces() {
while (m_cursor < m_content.size() && std::isspace(get()) != 0)
++m_cursor;
}
constexpr token::location lexer::location() const {
return { m_filepath, m_row, m_cursor - m_lineStart };
}
} // namespace furc
furc/src/main.cpp
+14 -3
View File
@@ -1,7 +1,18 @@
#include "furc/front/lexer.hpp"
#include <iostream>
int main(void) {
std::cout << "Farewell, stasiu!\n";
return 0;
furc::lexer lexer = { "<AK>", "func main(argc: u64) -> s32 { return '\\\\'; }" };
while (true) {
furc::token token = lexer.next_token();
std::cout << token.loc.filepath << ':' << token.loc.row + 1 << ':' << token.loc.col + 1 << ": " << token
<< '\n';
switch (token.type) {
case furc::token::UnexpectedCharacter:
case furc::token::UnexpectedEOF: return 1;
case furc::token::EndOfFile: return 0;
default: break;
}
}
}