diff --git a/furc/include/furc/front/token.hpp b/furc/include/furc/front/token.hpp index 1740702..8251c97 100644 --- a/furc/include/furc/front/token.hpp +++ b/furc/include/furc/front/token.hpp @@ -103,10 +103,10 @@ struct token { bool operator==(const token& rhs) const { if (type != rhs.type) return false; switch (type) { - case token_t::None: case token_t::Identifier: return value == rhs.value; case token_t::Keyword: return keyword == rhs.keyword; case token_t::Integer: return value == rhs.value; + case token_t::None: case token_t::Lparen: case token_t::Rparen: case token_t::Lbrace: diff --git a/furc/src/front/lexer.cpp b/furc/src/front/lexer.cpp index f846964..ae0ed43 100644 --- a/furc/src/front/lexer.cpp +++ b/furc/src/front/lexer.cpp @@ -13,6 +13,32 @@ lexer::lexer(std::string_view filename, std::string_view content) token_handle<> lexer::next_token() { skip_spaces(); + while (m_cursor + 2 <= m_content.size() && m_content[m_cursor] == '/') { + if (m_content[m_cursor + 1] == '/') { + m_cursor += 2; + while (m_content[m_cursor] != '\n') { + next(); + } + } else if (m_content[m_cursor + 1] == '*') { + m_cursor += 2; + while (m_cursor + 2 < m_content.size()) { + if (m_content[m_cursor + 1] == '*') { + next(); + } else if (m_content[m_cursor + 0] != '*' || m_content[m_cursor + 1] != '/') { + next(); + next(); + } else { + break; + } + } + if (m_cursor + 2 >= m_content.size()) { + next(); + return { current_location(), "unexpected end of file before enclosing `*/`" }; + } + m_cursor += 2; + } + skip_spaces(); + } location location = current_location(); diff --git a/furc/test/lexer.cpp b/furc/test/lexer.cpp index f134553..41cddb8 100644 --- a/furc/test/lexer.cpp +++ b/furc/test/lexer.cpp @@ -8,6 +8,7 @@ using namespace furc::front; using namespace std::string_view_literals; #define EXPECT_TOKEN(lexer, t) EXPECT_EQ((lexer).next_token(), (t)); +#define EXPECT_EMPTY_TOKEN(lexer) EXPECT_EQ((lexer).next_token(), (token{})); TEST(Lexer, Tokens) { lexer lexer("", "()\n\t\t{\n}[]; :,.main return func"); @@ -24,6 +25,16 @@ TEST(Lexer, Tokens) { EXPECT_TOKEN(lexer, (token{ token_t::Identifier, "main"sv })); EXPECT_TOKEN(lexer, (token{ keyword_token::Return })); EXPECT_TOKEN(lexer, (token{ keyword_token::Func })); + EXPECT_EMPTY_TOKEN(lexer); +} + +TEST(Lexer, Comments) { + lexer lexer("", "(/** skibidi **/func{//)\n}"); + EXPECT_TOKEN(lexer, (token{ token_t::Lparen, "("sv })); + EXPECT_TOKEN(lexer, (token{ keyword_token::Func })); + EXPECT_TOKEN(lexer, (token{ token_t::Lbrace, "{"sv })); + EXPECT_TOKEN(lexer, (token{ token_t::Rbrace, "}"sv })); + EXPECT_EMPTY_TOKEN(lexer); } } // namespace \ No newline at end of file