From 28f6deaba54e89e156f17f4c1bb18eff0da5635e Mon Sep 17 00:00:00 2001 From: CHatingPython Date: Sun, 24 May 2026 14:57:57 +0200 Subject: [PATCH] Basic lexer --- .clang-format | 67 +++++++++++++++++++ .clang-tidy | 93 ++++++++++++++++++++++++++ .clangd | 25 +++++++ .gitignore | 2 + CMakeLists.txt | 6 ++ furc/CMakeLists.txt | 4 ++ furc/include/furc/diag.hpp | 21 ++++++ furc/include/furc/front/lexer.hpp | 43 ++++++++++++ furc/include/furc/front/token.hpp | 106 ++++++++++++++++++++++++++++++ furc/src/back/lexer.cpp | 76 +++++++++++++++++++++ furc/src/main.cpp | 15 +++++ furlang/CMakeLists.txt | 0 12 files changed, 458 insertions(+) create mode 100755 .clang-format create mode 100755 .clang-tidy create mode 100755 .clangd create mode 100644 .gitignore create mode 100644 CMakeLists.txt create mode 100644 furc/CMakeLists.txt create mode 100644 furc/include/furc/diag.hpp create mode 100644 furc/include/furc/front/lexer.hpp create mode 100644 furc/include/furc/front/token.hpp create mode 100644 furc/src/back/lexer.cpp create mode 100644 furc/src/main.cpp create mode 100644 furlang/CMakeLists.txt diff --git a/.clang-format b/.clang-format new file mode 100755 index 0000000..1271d76 --- /dev/null +++ b/.clang-format @@ -0,0 +1,67 @@ +BasedOnStyle: LLVM + +IndentWidth: 4 +TabWidth: 4 +UseTab: Never +ContinuationIndentWidth: 4 + +ColumnLimit: 120 + +BreakBeforeBraces: Attach +BraceWrapping: + BeforeCatch: true + BeforeElse: false + BeforeWhile: false + BeforeLambdaBody: false + +AllowShortFunctionsOnASingleLine: Inline +AllowShortBlocksOnASingleLine: Empty +AllowShortCaseLabelsOnASingleLine: true +AllowShortEnumsOnASingleLine: false +AllowShortIfStatementsOnASingleLine: true +AllowShortLoopsOnASingleLine: false +AllowShortCompoundRequirementOnASingleLine: true + +NamespaceIndentation: None +AccessModifierOffset: -4 +EmptyLineBeforeAccessModifier: Never + +SpaceBeforeParens: ControlStatementsExceptControlMacros +SpaceBeforeInheritanceColon: true +SpacesBeforeTrailingComments: 1 + +AlignTrailingComments: true + +AlignConsecutiveAssignments: + Enabled: true + AlignCompound: true + PadOperators: false + +AlignConsecutiveDeclarations: Consecutive +AlignAfterOpenBracket: DontAlign + +ConstructorInitializerIndentWidth: 2 +PackConstructorInitializers: NextLineOnly +BreakConstructorInitializers: BeforeColon + +BinPackParameters: false +BinPackArguments: false +AllowAllArgumentsOnNextLine: false + +AlwaysBreakTemplateDeclarations: Yes +BreakAfterReturnType: None + +Cpp11BracedListStyle: false +BreakArrays: false + +IncludeBlocks: Regroup +SortIncludes: true + +ReflowComments: false +KeepEmptyLinesAtTheStartOfBlocks: true + +DerivePointerAlignment: false +PointerAlignment: Left +AllowAllParametersOfDeclarationOnNextLine: false + +PenaltyReturnTypeOnItsOwnLine: 1000 \ No newline at end of file diff --git a/.clang-tidy b/.clang-tidy new file mode 100755 index 0000000..bbe8587 --- /dev/null +++ b/.clang-tidy @@ -0,0 +1,93 @@ +--- +Checks: > + -*, + + bugprone-*, + performance-*, + readability-*, + portability-*, + cppcoreguidelines-*, + + -bugprone-assignment-in-if-condition, + -bugprone-easily-swappable-parameters, + -bugprone-multi-level-implicit-pointer-conversion, + -readability-function-cognitive-complexity, + -readability-magic-numbers, + -readability-redundant-access-specifiers, + -readability-use-anyofallof, + -readability-named-parameter, + -performance-enum-size, + -cppcoreguidelines-avoid-magic-numbers, + -cppcoreguidelines-pro-bounds-pointer-arithmetic, + -cppcoreguidelines-pro-type-union-access, + -cppcoreguidelines-pro-type-reinterpret-cast, + -cppcoreguidelines-avoid-do-while, + -cppcoreguidelines-avoid-non-const-global-variables, + -cppcoreguidelines-avoid-c-arrays, + -cppcoreguidelines-pro-bounds-constant-array-index, + -cppcoreguidelines-macro-usage, + -cppcoreguidelines-owning-memory + +WarningsAsErrors: "*" + +HeaderFilterRegex: ".*" + +CheckOptions: + - key: readability-braces-around-statements.ShortStatementLines + value: 3 + + - key: readability-identifier-length.MinimumParameterNameLength + value: 2 + + - key: readability-identifier-length.MinimumVariableNameLength + value: 2 + + # Namespaces + - key: readability-identifier-naming.NamespaceCase + value: lower_case + + # Types + - key: readability-identifier-naming.StructCase + value: snake_case + - key: readability-identifier-naming.ClassCase + value: snake_case + - key: readability-identifier-naming.EnumCase + value: snake_case + # - key: readability-identifier-naming.AbstractClassPrefix + # value: I + + # Functions + - key: readability-identifier-naming.FunctionCase + value: snake_case + + # Variables + - key: readability-identifier-naming.VariableCase + value: camelBack + + # Public members + - key: readability-identifier-naming.PublicMemberCase + value: camelBack + + # Static members + - key: readability-identifier-naming.StaticVariableCase + value: camelBack + - key: readability-identifier-naming.StaticVariablePrefix + value: s_ + - key: readability-identifier-naming.ClassMemberCase + value: camelBack + - key: readability-identifier-naming.ClassMemberPrefix + value: s_ + + # Static constexpr + - key: readability-identifier-naming.StaticConstexprVariableCase + value: aNy_CaSe + + # Constants + - key: readability-identifier-naming.ConstantCase + value: aNy_CaSe + - key: readability-identifier-naming.StaticConstantCase + value: aNy_CaSe + - key: readability-identifier-naming.EnumConstantCase + value: CamelCase + - key: readability-identifier-naming.ScopedEnumConstantCase + value: CamelCase \ No newline at end of file diff --git a/.clangd b/.clangd new file mode 100755 index 0000000..356abbc --- /dev/null +++ b/.clangd @@ -0,0 +1,25 @@ +CompileFlags: + Add: [-std=c11, -Wall, -Werror, -Wpedantic] + +InlayHints: + BlockEnd: true + Designators: false + Enabled: true + ParameterNames: true + DeducedTypes: true + TypeNameLimit: 24 + +Completion: + HeaderInsertion: IWYU + +Hover: + ShowAKA: true + +If: + PathMatch: (./.*\.c|./.*\.h) +Diagnostics: + UnusedIncludes: Strict + MissingIncludes: Strict + +Documentation: + CommentFormat: Doxygen \ No newline at end of file diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..9994ccd --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +build/ +.cache \ No newline at end of file diff --git a/CMakeLists.txt b/CMakeLists.txt new file mode 100644 index 0000000..68a378f --- /dev/null +++ b/CMakeLists.txt @@ -0,0 +1,6 @@ +cmake_minimum_required(VERSION 3.14) + +project(furlang VERSION 0.0.0 LANGUAGES CXX) + +add_subdirectory(furlang) +add_subdirectory(furc) \ No newline at end of file diff --git a/furc/CMakeLists.txt b/furc/CMakeLists.txt new file mode 100644 index 0000000..35c8ae3 --- /dev/null +++ b/furc/CMakeLists.txt @@ -0,0 +1,4 @@ +file(GLOB_RECURSE FURC_SRCS "src/**.cpp") +file(GLOB_RECURSE FURC_HDRS "src/**.cpp") +add_executable(furc ${FURC_SRCS} ${FURC_HDRS}) +target_include_directories(furc PRIVATE include/) \ No newline at end of file diff --git a/furc/include/furc/diag.hpp b/furc/include/furc/diag.hpp new file mode 100644 index 0000000..f3de0e7 --- /dev/null +++ b/furc/include/furc/diag.hpp @@ -0,0 +1,21 @@ +#ifndef FURC_DIAG_HPP +#define FURC_DIAG_HPP + +#include +#include + +namespace furc { + +struct location { + std::string_view filename; + std::size_t line; + std::size_t column; +}; + +static inline std::ostream& operator<<(std::ostream& os, const location& location) { + return os << location.filename << ':' << location.line + 1 << ':' << location.column + 1; +} + +} // namespace furc + +#endif // FURC_DIAG_HPP \ No newline at end of file diff --git a/furc/include/furc/front/lexer.hpp b/furc/include/furc/front/lexer.hpp new file mode 100644 index 0000000..1522869 --- /dev/null +++ b/furc/include/furc/front/lexer.hpp @@ -0,0 +1,43 @@ +#ifndef FURC_FRONT_LEXER_HPP +#define FURC_FRONT_LEXER_HPP + +#include "furc/front/token.hpp" + +namespace furc { +namespace front { + +class lexer { +public: + lexer(std::string_view filename, std::string_view content); + ~lexer() = default; + + lexer(lexer&&) = default; + lexer(const lexer&) = delete; + lexer& operator=(lexer&&) = default; + lexer& operator=(const lexer&) = delete; +public: + token next_token(); +private: + void next(); + char get(std::size_t offset = 0) const; + void skip_spaces(); + location current_location(); +private: + template + token create_token(location location, Args&&... args) { + token tok(std::forward(args)...); + tok.location = location; + return tok; + } +private: + std::string_view m_filename; + std::string_view m_content; + std::size_t m_cursor = 0; + std::size_t m_row = 0; + std::size_t m_lineStart = 0; +}; + +} // namespace front +} // namespace furc + +#endif // FURC_FRONT_LEXER_HPP \ No newline at end of file diff --git a/furc/include/furc/front/token.hpp b/furc/include/furc/front/token.hpp new file mode 100644 index 0000000..79ea03d --- /dev/null +++ b/furc/include/furc/front/token.hpp @@ -0,0 +1,106 @@ +#ifndef FURC_FRONT_TOKEN_HPP +#define FURC_FRONT_TOKEN_HPP + +#include "furc/diag.hpp" + +#include +#include +#include + +namespace furc { +namespace front { + +enum class token_t { + None, + Error, + Identifier, + Keyword, + Integer, + Lparen, + Rparen, + Lbrace, + Rbrace, + Lbracket, + Rbracket, + Semicolon, + Colon, +}; + +static inline bool is_token_type_empty(token_t type) { + return type == token_t::None || type == token_t::Error; +} + +static inline std::ostream& operator<<(std::ostream& os, token_t type) { + switch (type) { + case token_t::None: return os << "none"; + case token_t::Error: return os << "error"; + case token_t::Identifier: return os << "identifier"; + case token_t::Keyword: return os << "keyword"; + case token_t::Integer: return os << "integer"; + case token_t::Lparen: return os << "'('"; + case token_t::Rparen: return os << "')'"; + case token_t::Lbrace: return os << "'{'"; + case token_t::Rbrace: return os << "'}'"; + case token_t::Lbracket: return os << "'['"; + case token_t::Rbracket: return os << "']'"; + case token_t::Semicolon: return os << "';'"; + case token_t::Colon: return os << "':'"; + } +} + +enum class error_token { + None, + UnexpectedCharacter, +}; + +static inline std::ostream& operator<<(std::ostream& os, error_token error) { + switch (error) { + case error_token::None: return os << "none"; + case error_token::UnexpectedCharacter: return os << "unexpected character"; + } +} + +enum class keyword_token { + None, + Func, + Return, +}; + +struct token { + token_t type = token_t::None; + error_token error = error_token::None; + keyword_token keyword = keyword_token::None; + std::string_view value; + location location; + + token() = default; + + token(token_t type, std::string_view value = {}) + : type(type), value(value) {} + + token(error_token error, std::string_view value = {}) + : type(token_t::Error), error(error), value(value) {} + + token(keyword_token keyword, std::string_view value = {}) + : type(token_t::Keyword), keyword(keyword), value(value) {} +}; + +static inline std::ostream& operator<<(std::ostream& os, const token& token) { + os << token.location << ": " << token.type; + switch (token.type) { + case token_t::Error: { + os << ": " << token.error; + if (!token.value.empty()) os << " \"" << token.value << '"'; + return os; + } + case token_t::Identifier: + case token_t::Keyword: + case token_t::Integer: return os << ": " << token.value; + default: return os; + } +} + +} // namespace front +} // namespace furc + +#endif // FURC_FRONT_TOKEN_HPP \ No newline at end of file diff --git a/furc/src/back/lexer.cpp b/furc/src/back/lexer.cpp new file mode 100644 index 0000000..cfad2b4 --- /dev/null +++ b/furc/src/back/lexer.cpp @@ -0,0 +1,76 @@ +#include "furc/front/lexer.hpp" + +#include +#include + +namespace furc::front { + +lexer::lexer(std::string_view filename, std::string_view content) + : m_filename(filename), m_content(content) {} + +token lexer::next_token() { + skip_spaces(); + + location location = current_location(); + + char ch = get(); + switch (ch) { + case '(': return create_token(location, token_t::Lparen, m_content.substr(m_cursor++, 1)); + case ')': return create_token(location, token_t::Rparen, m_content.substr(m_cursor++, 1)); + case '{': return create_token(location, token_t::Lbrace, m_content.substr(m_cursor++, 1)); + case '}': return create_token(location, token_t::Rbrace, m_content.substr(m_cursor++, 1)); + case '[': return create_token(location, token_t::Lbracket, m_content.substr(m_cursor++, 1)); + case ']': return create_token(location, token_t::Rbracket, m_content.substr(m_cursor++, 1)); + case ';': return create_token(location, token_t::Semicolon, m_content.substr(m_cursor++, 1)); + case ':': return create_token(location, token_t::Colon, m_content.substr(m_cursor++, 1)); + case std::char_traits::eof(): return create_token(location, token_t::None); + default: { + if (std::isdigit(ch) != 0) {} + + if (std::isalnum(ch) != 0 || ch == '_') { + std::size_t start = m_cursor++; + while (std::isalnum(get()) != 0 || get() == '_') + next(); + + std::string_view value = m_content.substr(start, m_cursor - start); + + static std::unordered_map s_keywords = { + { "func", keyword_token::Func }, + { "return", keyword_token::Return }, + }; + + if (auto it = s_keywords.find(value); it != s_keywords.end()) + return create_token(location, it->second, value); + return create_token(location, token_t::Identifier, value); + } + + return create_token(location, error_token::UnexpectedCharacter, m_content.substr(m_cursor, 1)); + } + } +} + +void lexer::next() { + if (m_cursor >= m_content.size()) return; + char ch = get(); + ++m_cursor; + if (ch == '\n') { + ++m_row; + m_lineStart = m_cursor; + } +} + +char lexer::get(std::size_t offset) const { + if (m_cursor + offset < m_content.size()) return m_content[m_cursor + offset]; + return std::char_traits::eof(); +} + +void lexer::skip_spaces() { + while (std::isspace(get()) != 0) + next(); +} + +location lexer::current_location() { + return { m_filename, m_row, m_cursor - m_lineStart }; +} + +} // namespace furc::front \ No newline at end of file diff --git a/furc/src/main.cpp b/furc/src/main.cpp new file mode 100644 index 0000000..00d1ffc --- /dev/null +++ b/furc/src/main.cpp @@ -0,0 +1,15 @@ +#include "furc/front/lexer.hpp" + +#include + +int main(void) { + furc::front::lexer lexer("", "func main() {\n return 0;\n}"); + + furc::front::token token = {}; + while (!furc::front::is_token_type_empty((token = lexer.next_token()).type)) { + std::cout << token << '\n'; + } + if (token.type == furc::front::token_t::Error) std::cout << token << '\n'; + + return 0; +} \ No newline at end of file diff --git a/furlang/CMakeLists.txt b/furlang/CMakeLists.txt new file mode 100644 index 0000000..e69de29