Skip to content

Instantly share code, notes, and snippets.

@unrays
Last active September 15, 2026 14:20
Show Gist options
  • Select an option

  • Save unrays/3e118fece96ad5bc0bc341a7b4389dc1 to your computer and use it in GitHub Desktop.

Select an option

Save unrays/3e118fece96ad5bc0bc341a7b4389dc1 to your computer and use it in GitHub Desktop.
COMPILE-TIME KEYWORD-TO-TOKEN CATEGORIZATION SYSTEM - This code comes from an unfinished project of mine. I found certain parts quite interesting, so I extracted a few of them and turned them into gists. This code is for educational purposes and is not functional, as it requires the rest of the architecture to work.
// Copyright (c) July 2026 Félix-Olivier Dumas. All rights reserved.
// Licensed under the terms described in the LICENSE file
template<typename entry_tuple>
struct TokenKeywordCategorizerConfigurationSchema {
using first_t = std::decay_t<std::tuple_element_t<0, entry_tuple>>;
using second_t = std::decay_t<std::tuple_element_t<1, entry_tuple>>;
static constexpr bool valid =
requires {
requires std::tuple_size_v<entry_tuple> == 2;
requires is_keyword_matching_policy<first_t>;
requires std::is_same_v<std::decay_t<decltype(second_t::value)>, TokenKind>;
};
};
template<typename entry_tuple>
struct TokenKeywordCategorizerConfigurationModel final {
using predicate = std::tuple_element_t<0, entry_tuple>;
static constexpr TokenKind corresponding = std::tuple_element_t<1, entry_tuple>::value;
};
template<typename... Entries>
using TokenKeywordCategorizerConfiguration = ConfigurationSystem<
TokenKeywordCategorizerConfigurationSchema,
TokenKeywordCategorizerConfigurationModel,
Entries...
>;
template <typename T>
constexpr bool is_token_keyword_categorizer_config_v = false;
template <typename... Entries>
constexpr bool is_token_keyword_categorizer_config_v<TokenKeywordCategorizerConfiguration<Entries...>> = true;
template <typename T>
concept is_token_keyword_categorizer_configuration = is_token_keyword_categorizer_config_v<T>;
// ca se répète souvent, y'aurais peut-être moyen de factoriser tout cela de facon a rendre la chose plus lisible
export template<is_token_keyword_categorizer_configuration Configuration>
struct TokenKeywordCategorizer final {
protected:
using configuration_tuple_t = typename Configuration::type;
static constexpr bool is_nothrow_config = []<typename... Ts>(std::type_identity<std::tuple<Ts...>>) {
return (noexcept(Ts::predicate::matches(std::string_view{})) && ...);
}(std::type_identity<configuration_tuple_t>{});
public:
[[nodiscard]] static constexpr TokenKind transform(std::string_view sv) noexcept(is_nothrow_config) {
return [&]<typename... Ts>(std::type_identity<std::tuple<Ts...>>) {
TokenKind result = TokenKind::Unknown;
(((Ts::predicate::matches(sv)) && (result = Ts::corresponding, true)), ...);
return result;
}(std::type_identity<configuration_tuple_t>{});
}
};
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment