refactor(furc/lexer): improve token peeking

furc/include/furc/front/lexer.hpp

@@ -4,7 +4,7 @@
 #include "furc/front/token.hpp"
 
 #include <cstddef>
-#include <optional>
+#include <deque>
 #include <string_view>
 
 namespace furc {
@@ -23,9 +23,10 @@ public:
     lexer& operator=(const lexer&) = delete;
 public:
     token next_token();
-    token peek_token();
+    token peek_token(std::size_t offset = 0);
-    token skip_token();
 private:
+    token get_token();
+
     void           next();
     constexpr char get(std::size_t offset = 0) const;
     void           skip_spaces();
@@ -38,7 +39,7 @@ private:
     std::size_t      m_row       = 0;
     std::size_t      m_lineStart = 0;
 
-    std::optional<token> m_peekToken;
+    std::deque<token> m_peekToken;
 };
 
 } // namespace furc

furc/src/front/lexer.cpp

@@ -8,12 +8,23 @@
 namespace furc {
 
 token lexer::next_token() {
-    if (m_peekToken.has_value()) {
+    if (m_peekToken.empty()) return get_token();
-        auto tok    = m_peekToken.value();
+    auto tok = m_peekToken.front();
-        m_peekToken = {};
+    m_peekToken.pop_front();
-        return tok;
+    return tok;
-    }
+}
 
+token lexer::peek_token(std::size_t offset) {
+    if (m_peekToken.size() <= offset) {
+        while (m_peekToken.size() <= offset) {
+            auto tok = get_token();
+            m_peekToken.push_back(tok);
+        }
+    }
+    return m_peekToken[offset];
+}
+
+token lexer::get_token() {
     skip_spaces();
 
     if (m_cursor >= m_content.size()) return { location(), token::EndOfFile };
@@ -171,18 +182,6 @@ token lexer::next_token() {
     return { loc, token::UnexpectedCharacter, get() };
 }
 
-token lexer::peek_token() {
-    if (m_peekToken.has_value()) return m_peekToken.value();
-    auto tok    = next_token();
-    m_peekToken = tok;
-    return tok;
-}
-
-token lexer::skip_token() {
-    m_peekToken = {};
-    return next_token();
-}
-
 void lexer::next() {
     if (m_cursor < m_content.size()) ++m_cursor;
 }
@@ -192,8 +191,12 @@ constexpr char lexer::get(std::size_t offset) const {
 }
 
 void lexer::skip_spaces() {
-    while (m_cursor < m_content.size() && std::isspace(get()) != 0)
+    while (m_cursor < m_content.size() && std::isspace(get()) != 0) {
-        ++m_cursor;
+        if (m_content[m_cursor++] == '\n') {
+            ++m_row;
+            m_lineStart = m_cursor;
+        }
+    }
 }
 
 constexpr token::location lexer::location() const {