KPGPMC/furlang
5
3
Fork 2
Code Issues 11 Pull Requests Actions Packages Projects 3 Releases Wiki Activity

Basic lexer

Browse Source
This commit is contained in:
CHatingPython
2026-05-24 14:57:57 +02:00
commit 28f6deaba5
12 changed files with 458 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.clang-format
Executable
+67
View File
@@ -0,0 +1,67 @@
BasedOnStyle: LLVM
IndentWidth: 4
TabWidth: 4
UseTab: Never
ContinuationIndentWidth: 4
ColumnLimit: 120
BreakBeforeBraces: Attach
BraceWrapping:
BeforeCatch: true
BeforeElse: false
BeforeWhile: false
BeforeLambdaBody: false
AllowShortFunctionsOnASingleLine: Inline
AllowShortBlocksOnASingleLine: Empty
AllowShortCaseLabelsOnASingleLine: true
AllowShortEnumsOnASingleLine: false
AllowShortIfStatementsOnASingleLine: true
AllowShortLoopsOnASingleLine: false
AllowShortCompoundRequirementOnASingleLine: true
NamespaceIndentation: None
AccessModifierOffset: -4
EmptyLineBeforeAccessModifier: Never
SpaceBeforeParens: ControlStatementsExceptControlMacros
SpaceBeforeInheritanceColon: true
SpacesBeforeTrailingComments: 1
AlignTrailingComments: true
AlignConsecutiveAssignments:
Enabled: true
AlignCompound: true
PadOperators: false
AlignConsecutiveDeclarations: Consecutive
AlignAfterOpenBracket: DontAlign
ConstructorInitializerIndentWidth: 2
PackConstructorInitializers: NextLineOnly
BreakConstructorInitializers: BeforeColon
BinPackParameters: false
BinPackArguments: false
AllowAllArgumentsOnNextLine: false
AlwaysBreakTemplateDeclarations: Yes
BreakAfterReturnType: None
Cpp11BracedListStyle: false
BreakArrays: false
IncludeBlocks: Regroup
SortIncludes: true
ReflowComments: false
KeepEmptyLinesAtTheStartOfBlocks: true
DerivePointerAlignment: false
PointerAlignment: Left
AllowAllParametersOfDeclarationOnNextLine: false
PenaltyReturnTypeOnItsOwnLine: 1000
.clang-tidy
Executable
+93
View File
@@ -0,0 +1,93 @@
---
Checks: >
-*,
bugprone-*,
performance-*,
readability-*,
portability-*,
cppcoreguidelines-*,
-bugprone-assignment-in-if-condition,
-bugprone-easily-swappable-parameters,
-bugprone-multi-level-implicit-pointer-conversion,
-readability-function-cognitive-complexity,
-readability-magic-numbers,
-readability-redundant-access-specifiers,
-readability-use-anyofallof,
-readability-named-parameter,
-performance-enum-size,
-cppcoreguidelines-avoid-magic-numbers,
-cppcoreguidelines-pro-bounds-pointer-arithmetic,
-cppcoreguidelines-pro-type-union-access,
-cppcoreguidelines-pro-type-reinterpret-cast,
-cppcoreguidelines-avoid-do-while,
-cppcoreguidelines-avoid-non-const-global-variables,
-cppcoreguidelines-avoid-c-arrays,
-cppcoreguidelines-pro-bounds-constant-array-index,
-cppcoreguidelines-macro-usage,
-cppcoreguidelines-owning-memory
WarningsAsErrors: "*"
HeaderFilterRegex: ".*"
CheckOptions:
- key: readability-braces-around-statements.ShortStatementLines
value: 3
- key: readability-identifier-length.MinimumParameterNameLength
value: 2
- key: readability-identifier-length.MinimumVariableNameLength
value: 2
# Namespaces
- key: readability-identifier-naming.NamespaceCase
value: lower_case
# Types
- key: readability-identifier-naming.StructCase
value: snake_case
- key: readability-identifier-naming.ClassCase
value: snake_case
- key: readability-identifier-naming.EnumCase
value: snake_case
# - key: readability-identifier-naming.AbstractClassPrefix
# value: I
# Functions
- key: readability-identifier-naming.FunctionCase
value: snake_case
# Variables
- key: readability-identifier-naming.VariableCase
value: camelBack
# Public members
- key: readability-identifier-naming.PublicMemberCase
value: camelBack
# Static members
- key: readability-identifier-naming.StaticVariableCase
value: camelBack
- key: readability-identifier-naming.StaticVariablePrefix
value: s_
- key: readability-identifier-naming.ClassMemberCase
value: camelBack
- key: readability-identifier-naming.ClassMemberPrefix
value: s_
# Static constexpr
- key: readability-identifier-naming.StaticConstexprVariableCase
value: aNy_CaSe
# Constants
- key: readability-identifier-naming.ConstantCase
value: aNy_CaSe
- key: readability-identifier-naming.StaticConstantCase
value: aNy_CaSe
- key: readability-identifier-naming.EnumConstantCase
value: CamelCase
- key: readability-identifier-naming.ScopedEnumConstantCase
value: CamelCase
.clangd
Executable
+25
View File
@@ -0,0 +1,25 @@
CompileFlags:
Add: [-std=c11, -Wall, -Werror, -Wpedantic]
InlayHints:
BlockEnd: true
Designators: false
Enabled: true
ParameterNames: true
DeducedTypes: true
TypeNameLimit: 24
Completion:
HeaderInsertion: IWYU
Hover:
ShowAKA: true
If:
PathMatch: (./.*\.c|./.*\.h)
Diagnostics:
UnusedIncludes: Strict
MissingIncludes: Strict
Documentation:
CommentFormat: Doxygen
.gitignore
+2
View File
@@ -0,0 +1,2 @@
build/
.cache
CMakeLists.txt
+6
View File
@@ -0,0 +1,6 @@
cmake_minimum_required(VERSION 3.14)
project(furlang VERSION 0.0.0 LANGUAGES CXX)
add_subdirectory(furlang)
add_subdirectory(furc)
furc/CMakeLists.txt
+4
View File
@@ -0,0 +1,4 @@
file(GLOB_RECURSE FURC_SRCS "src/**.cpp")
file(GLOB_RECURSE FURC_HDRS "src/**.cpp")
add_executable(furc ${FURC_SRCS} ${FURC_HDRS})
target_include_directories(furc PRIVATE include/)
furc/include/furc/diag.hpp
+21
View File
@@ -0,0 +1,21 @@
#ifndef FURC_DIAG_HPP
#define FURC_DIAG_HPP
#include <ostream>
#include <string_view>
namespace furc {
struct location {
std::string_view filename;
std::size_t line;
std::size_t column;
};
static inline std::ostream& operator<<(std::ostream& os, const location& location) {
return os << location.filename << ':' << location.line + 1 << ':' << location.column + 1;
}
} // namespace furc
#endif // FURC_DIAG_HPP
furc/include/furc/front/lexer.hpp
+43
View File
@@ -0,0 +1,43 @@
#ifndef FURC_FRONT_LEXER_HPP
#define FURC_FRONT_LEXER_HPP
#include "furc/front/token.hpp"
namespace furc {
namespace front {
class lexer {
public:
lexer(std::string_view filename, std::string_view content);
~lexer() = default;
lexer(lexer&&) = default;
lexer(const lexer&) = delete;
lexer& operator=(lexer&&) = default;
lexer& operator=(const lexer&) = delete;
public:
token next_token();
private:
void next();
char get(std::size_t offset = 0) const;
void skip_spaces();
location current_location();
private:
template <typename... Args>
token create_token(location location, Args&&... args) {
token tok(std::forward<Args>(args)...);
tok.location = location;
return tok;
}
private:
std::string_view m_filename;
std::string_view m_content;
std::size_t m_cursor = 0;
std::size_t m_row = 0;
std::size_t m_lineStart = 0;
};
} // namespace front
} // namespace furc
#endif // FURC_FRONT_LEXER_HPP
furc/include/furc/front/token.hpp
+106
View File
@@ -0,0 +1,106 @@
#ifndef FURC_FRONT_TOKEN_HPP
#define FURC_FRONT_TOKEN_HPP
#include "furc/diag.hpp"
#include <cassert>
#include <ostream>
#include <string_view>
namespace furc {
namespace front {
enum class token_t {
None,
Error,
Identifier,
Keyword,
Integer,
Lparen,
Rparen,
Lbrace,
Rbrace,
Lbracket,
Rbracket,
Semicolon,
Colon,
};
static inline bool is_token_type_empty(token_t type) {
return type == token_t::None || type == token_t::Error;
}
static inline std::ostream& operator<<(std::ostream& os, token_t type) {
switch (type) {
case token_t::None: return os << "none";
case token_t::Error: return os << "error";
case token_t::Identifier: return os << "identifier";
case token_t::Keyword: return os << "keyword";
case token_t::Integer: return os << "integer";
case token_t::Lparen: return os << "'('";
case token_t::Rparen: return os << "')'";
case token_t::Lbrace: return os << "'{'";
case token_t::Rbrace: return os << "'}'";
case token_t::Lbracket: return os << "'['";
case token_t::Rbracket: return os << "']'";
case token_t::Semicolon: return os << "';'";
case token_t::Colon: return os << "':'";
}
}
enum class error_token {
None,
UnexpectedCharacter,
};
static inline std::ostream& operator<<(std::ostream& os, error_token error) {
switch (error) {
case error_token::None: return os << "none";
case error_token::UnexpectedCharacter: return os << "unexpected character";
}
}
enum class keyword_token {
None,
Func,
Return,
};
struct token {
token_t type = token_t::None;
error_token error = error_token::None;
keyword_token keyword = keyword_token::None;
std::string_view value;
location location;
token() = default;
token(token_t type, std::string_view value = {})
: type(type), value(value) {}
token(error_token error, std::string_view value = {})
: type(token_t::Error), error(error), value(value) {}
token(keyword_token keyword, std::string_view value = {})
: type(token_t::Keyword), keyword(keyword), value(value) {}
};
static inline std::ostream& operator<<(std::ostream& os, const token& token) {
os << token.location << ": " << token.type;
switch (token.type) {
case token_t::Error: {
os << ": " << token.error;
if (!token.value.empty()) os << " \"" << token.value << '"';
return os;
}
case token_t::Identifier:
case token_t::Keyword:
case token_t::Integer: return os << ": " << token.value;
default: return os;
}
}
} // namespace front
} // namespace furc
#endif // FURC_FRONT_TOKEN_HPP
furc/src/back/lexer.cpp
+76
View File
@@ -0,0 +1,76 @@
#include "furc/front/lexer.hpp"
#include <cctype>
#include <unordered_map>
namespace furc::front {
lexer::lexer(std::string_view filename, std::string_view content)
: m_filename(filename), m_content(content) {}
token lexer::next_token() {
skip_spaces();
location location = current_location();
char ch = get();
switch (ch) {
case '(': return create_token(location, token_t::Lparen, m_content.substr(m_cursor++, 1));
case ')': return create_token(location, token_t::Rparen, m_content.substr(m_cursor++, 1));
case '{': return create_token(location, token_t::Lbrace, m_content.substr(m_cursor++, 1));
case '}': return create_token(location, token_t::Rbrace, m_content.substr(m_cursor++, 1));
case '[': return create_token(location, token_t::Lbracket, m_content.substr(m_cursor++, 1));
case ']': return create_token(location, token_t::Rbracket, m_content.substr(m_cursor++, 1));
case ';': return create_token(location, token_t::Semicolon, m_content.substr(m_cursor++, 1));
case ':': return create_token(location, token_t::Colon, m_content.substr(m_cursor++, 1));
case std::char_traits<char>::eof(): return create_token(location, token_t::None);
default: {
if (std::isdigit(ch) != 0) {}
if (std::isalnum(ch) != 0 || ch == '_') {
std::size_t start = m_cursor++;
while (std::isalnum(get()) != 0 || get() == '_')
next();
std::string_view value = m_content.substr(start, m_cursor - start);
static std::unordered_map<std::string_view, keyword_token> s_keywords = {
{ "func", keyword_token::Func },
{ "return", keyword_token::Return },
};
if (auto it = s_keywords.find(value); it != s_keywords.end())
return create_token(location, it->second, value);
return create_token(location, token_t::Identifier, value);
}
return create_token(location, error_token::UnexpectedCharacter, m_content.substr(m_cursor, 1));
}
}
}
void lexer::next() {
if (m_cursor >= m_content.size()) return;
char ch = get();
++m_cursor;
if (ch == '\n') {
++m_row;
m_lineStart = m_cursor;
}
}
char lexer::get(std::size_t offset) const {
if (m_cursor + offset < m_content.size()) return m_content[m_cursor + offset];
return std::char_traits<char>::eof();
}
void lexer::skip_spaces() {
while (std::isspace(get()) != 0)
next();
}
location lexer::current_location() {
return { m_filename, m_row, m_cursor - m_lineStart };
}
} // namespace furc::front
furc/src/main.cpp
+15
View File
@@ -0,0 +1,15 @@
#include "furc/front/lexer.hpp"
#include <iostream>
int main(void) {
furc::front::lexer lexer("<TEMP>", "func main() {\n return 0;\n}");
furc::front::token token = {};
while (!furc::front::is_token_type_empty((token = lexer.next_token()).type)) {
std::cout << token << '\n';
}
if (token.type == furc::front::token_t::Error) std::cout << token << '\n';
return 0;
}
furlang/CMakeLists.txt
View File