tokenizer.hpp

Go to the documentation of this file.

 
#pragma once
 
#include <functional>
#include <map>
#include <string>
#include <string_view>
 
#include "tcalc/common.hpp"
#include "tcalc/error.hpp"
#include "tcalc/token.hpp"
 
namespace tcalc::token {
 
class TCALC_PUBLIC Tokenizer
{
public:
  using KeywordComp =
    std::greater<std::string_view>; 
  inline static const std::map<std::string_view, TokenType, KeywordComp>
    KEYWORDS = {
      { "def", TokenType::DEF },         { "let", TokenType::LET },
      { "if", TokenType::IF },           { "then", TokenType::THEN },
      { "else", TokenType::ELSE },       { "import", TokenType::IMPORT },
      { "==", TokenType::EQUAL },        { "!=", TokenType::NOTEQUAL },
      { ">=", TokenType::GREATEREQUAL }, { "<=", TokenType::LESSEQUAL },
      { "&&", TokenType::AND },          { "||", TokenType::OR },
      { "+", TokenType::PLUS },          { "-", TokenType::MINUS },
      { "*", TokenType::MULTIPLY },      { "/", TokenType::DIVIDE },
      { "(", TokenType::LPAREN },        { ")", TokenType::RPAREN },
      { ",", TokenType::COMMA },         { ";", TokenType::SEMICOLON },
      { "=", TokenType::ASSIGN },        { ">", TokenType::GREATER },
      { "<", TokenType::LESS },          { "!", TokenType::NOT },
    }; 
  constexpr static char QUOTE = '\''; 
private:
  std::string_view _input;
  std::string_view::const_iterator _pos;
 
public:
  explicit Tokenizer(std::string_view input)
    : _input{ input }
    , _pos{ _input.begin() }
  {
  }
 
  error::Result<Token> next();
 
  [[nodiscard]] TCALC_INLINE auto pos() const noexcept { return _pos; }
 
  [[nodiscard]] TCALC_INLINE auto spos() const noexcept
  {
    return std::distance(_input.begin(), _pos);
  }
 
private:
  template<typename Pred>
  auto _next_with(TokenType type, const Pred& pred)
  {
    const auto* start = _pos;
    while (_pos != _input.end() && pred(*_pos)) {
      ++_pos;
    }
    return Token{ type, std::string{ start, _pos } };
  }
 
  template<typename Pred>
  void _skip_with(const Pred& pred)
  {
    while (_pos != _input.end() && pred(*_pos)) {
      ++_pos;
    }
  }
 
  error::Result<Token> _parse_number();
 
  error::Result<Token> _parse_quoted_identifier();
 
  bool _is_keyword(std::string_view keyword);
 
  static bool _is_identifier_char(char c);
 
  static bool _is_first_identifier_char(char c);
 
  static bool _is_skippable_char(char c);
};
 
}