zlotowek/forklang
1
0
Fork 0
forked from KPGPMC/furlang
Code Pull Requests Activity

refactor(lexer): change token_handle to token result

Browse Source 
Refs: #1
This commit is contained in:
Aleksander Krajewski
2026-06-02 16:47:02 +02:00
parent 3491f9aebb
commit cb7a348540
9 changed files with 162 additions and 111 deletions
Download Patch File Download Diff File Expand all files Collapse all files
furc/test/lexer.cpp
+32 -30
View File
@@ -2,8 +2,6 @@
#include "furc/front/lexer.hpp"
#include "furlang/result.hpp"
#include "gtest/gtest.h"
#include <string>
@@ -13,7 +11,6 @@ using namespace furc::front;
using namespace std::string_view_literals;
using namespace std::string_literals;
using token_r = furlang::result<token, std::string>;
using lexer_case = std::pair<std::string, std::vector<token_r>>;
class LexerTestingFixture : public testing::TestWithParam<lexer_case> {};
@@ -26,16 +23,16 @@ TEST_P(LexerTestingFixture, LexerTest) {
while (it != expected.end()) {
const auto& expected = *it++;
auto token = std::move(lexer.next_token());
if (expected.has_error()) {
EXPECT_TRUE(token.has_error());
EXPECT_EQ(token.error(), expected.error());
} else {
EXPECT_EQ(token, expected.value());
}
EXPECT_EQ(lexer.next_token(), expected);
}
// EOF
EXPECT_EQ(lexer.next_token(), token{});
auto eof = std::move(lexer.next_token());
ASSERT_TRUE(eof.has_error());
ASSERT_EQ(eof.error().type, token_error_t::EndOfFile);
}
furc::location loc(size_t col, size_t line) {
return furc::location{ "<TEMP>", line, col };
}
INSTANTIATE_TEST_SUITE_P(EmptyTests,
@@ -45,31 +42,36 @@ INSTANTIATE_TEST_SUITE_P(EmptyTests,
INSTANTIATE_TEST_SUITE_P(Comments,
LexerTestingFixture,
testing::Values(lexer_case("(/** skibidi **/func{//)\n}",
{ { token_t::LParen }, { keyword_token::Func }, { token_t::LBrace }, { token_t::RBrace } })));
{ { loc(0, 0), token_t::LParen },
{ loc(16, 0), keyword_token::Func },
{ loc(20, 0), token_t::LBrace },
{ loc(0, 1), token_t::RBrace } })));
INSTANTIATE_TEST_SUITE_P(Integers,
LexerTestingFixture,
testing::Values(lexer_case("67 6\n7", { { 67 }, { 6 }, { 7 } }),
lexer_case("18446744073709551615 18446744073709551616",
{ { 18446744073709551615ULL }, token_r(std::string("integer 18446744073709551616 is too big")) })));
testing::Values(lexer_case("67 6\n7", { { loc(0, 0), 67 }, { loc(3, 0), 6 }, { loc(0, 1), 7 } }),
lexer_case("18446744073709551615\n18446744073709551616",
{ { loc(0, 0), 18446744073709551615ULL },
token_r(
token_error{ loc(0, 1), token_error_t::IntegerOverflow, std::string("18446744073709551616") }) })));
INSTANTIATE_TEST_SUITE_P(Tokens,
LexerTestingFixture,
testing::Values(lexer_case("()\n\t\t{\n}[\"shto-to\"]; :,.main return func",
{ { token_t::LParen },
{ token_t::RParen },
{ token_t::LBrace },
{ token_t::RBrace },
{ token_t::LBracket },
{ token_t::String, "shto-to"sv },
{ token_t::RBracket },
{ token_t::Semicolon },
{ token_t::Colon },
{ token_t::Comma },
{ token_t::Dot },
{ token_t::Identifier, "main"sv },
{ keyword_token::Return },
{ keyword_token::Func } })));
{ { loc(0, 0), token_t::LParen },
{ loc(1, 0), token_t::RParen },
{ loc(2, 1), token_t::LBrace },
{ loc(0, 2), token_t::RBrace },
{ loc(1, 2), token_t::LBracket },
{ loc(2, 2), token_t::String, "shto-to"sv },
{ loc(10, 2), token_t::RBracket },
{ loc(11, 2), token_t::Semicolon },
{ loc(15, 2), token_t::Colon },
{ loc(16, 2), token_t::Comma },
{ loc(17, 2), token_t::Dot },
{ loc(18, 2), token_t::Identifier, "main"sv },
{ loc(23, 2), keyword_token::Return },
{ loc(30, 2), keyword_token::Func } })));
} // namespace