Add comments to lexer

Signed-off-by: CHatingPython <chatingpython@gmail.com>

furc/include/furc/front/token.hpp

@@ -103,10 +103,10 @@ struct token {
     bool operator==(const token& rhs) const {
         if (type != rhs.type) return false;
         switch (type) {
-        case token_t::None:
         case token_t::Identifier: return value == rhs.value;
         case token_t::Keyword: return keyword == rhs.keyword;
         case token_t::Integer: return value == rhs.value;
+        case token_t::None:
         case token_t::Lparen:
         case token_t::Rparen:
         case token_t::Lbrace:

furc/src/front/lexer.cpp

@@ -13,6 +13,32 @@ lexer::lexer(std::string_view filename, std::string_view content)
 
 token_handle<> lexer::next_token() {
     skip_spaces();
+    while (m_cursor + 2 <= m_content.size() && m_content[m_cursor] == '/') {
+        if (m_content[m_cursor + 1] == '/') {
+            m_cursor += 2;
+            while (m_content[m_cursor] != '\n') {
+                next();
+            }
+        } else if (m_content[m_cursor + 1] == '*') {
+            m_cursor += 2;
+            while (m_cursor + 2 < m_content.size()) {
+                if (m_content[m_cursor + 1] == '*') {
+                    next();
+                } else if (m_content[m_cursor + 0] != '*' || m_content[m_cursor + 1] != '/') {
+                    next();
+                    next();
+                } else {
+                    break;
+                }
+            }
+            if (m_cursor + 2 >= m_content.size()) {
+                next();
+                return { current_location(), "unexpected end of file before enclosing `*/`" };
+            }
+            m_cursor += 2;
+        }
+        skip_spaces();
+    }
 
     location location = current_location();
 

furc/test/lexer.cpp

@@ -8,6 +8,7 @@ using namespace furc::front;
 using namespace std::string_view_literals;
 
 #define EXPECT_TOKEN(lexer, t) EXPECT_EQ((lexer).next_token(), (t));
+#define EXPECT_EMPTY_TOKEN(lexer) EXPECT_EQ((lexer).next_token(), (token{}));
 
 TEST(Lexer, Tokens) {
     lexer lexer("<TEMP>", "()\n\t\t{\n}[];    :,.main return func");
@@ -24,6 +25,16 @@ TEST(Lexer, Tokens) {
     EXPECT_TOKEN(lexer, (token{ token_t::Identifier, "main"sv }));
     EXPECT_TOKEN(lexer, (token{ keyword_token::Return }));
     EXPECT_TOKEN(lexer, (token{ keyword_token::Func }));
+    EXPECT_EMPTY_TOKEN(lexer);
+}
+
+TEST(Lexer, Comments) {
+    lexer lexer("<TEMP>", "(/** skibidi **/func{//)\n}");
+    EXPECT_TOKEN(lexer, (token{ token_t::Lparen, "("sv }));
+    EXPECT_TOKEN(lexer, (token{ keyword_token::Func }));
+    EXPECT_TOKEN(lexer, (token{ token_t::Lbrace, "{"sv }));
+    EXPECT_TOKEN(lexer, (token{ token_t::Rbrace, "}"sv }));
+    EXPECT_EMPTY_TOKEN(lexer);
 }
 
 } // namespace