Toshihiro Shimizu 890ddd
Toshihiro Shimizu 890ddd
Toshihiro Shimizu 890ddd
#include "ttokenizer.h"
Toshihiro Shimizu 890ddd
#include <qstring></qstring>
Toshihiro Shimizu 890ddd
Shinya Kitaoka 120a6e
namespace TSyntax {
Toshihiro Shimizu 890ddd
Shinya Kitaoka 120a6e
int Token::getIntValue() const {
Shinya Kitaoka 120a6e
  return QString::fromStdString(getText()).toInt();
Toshihiro Shimizu 890ddd
}
Toshihiro Shimizu 890ddd
Shinya Kitaoka 120a6e
double Token::getDoubleValue() const {
Shinya Kitaoka 120a6e
  return QString::fromStdString(getText()).toDouble();
Toshihiro Shimizu 890ddd
}
Toshihiro Shimizu 890ddd
Toshihiro Shimizu 890ddd
//===================================================================
Toshihiro Shimizu 890ddd
Shinya Kitaoka 120a6e
Tokenizer::Tokenizer() : m_buffer(), m_index(0) {}
Toshihiro Shimizu 890ddd
Toshihiro Shimizu 890ddd
//-------------------------------------------------------------------
Toshihiro Shimizu 890ddd
Shinya Kitaoka 120a6e
Tokenizer::Tokenizer(std::string buffer) : m_buffer(), m_index(0) {
Shinya Kitaoka 120a6e
  setBuffer(buffer);
Toshihiro Shimizu 890ddd
}
Toshihiro Shimizu 890ddd
Toshihiro Shimizu 890ddd
//===================================================================
Toshihiro Shimizu 890ddd
Shinya Kitaoka 120a6e
Tokenizer::~Tokenizer() {}
Toshihiro Shimizu 890ddd
Toshihiro Shimizu 890ddd
//-------------------------------------------------------------------
Toshihiro Shimizu 890ddd
Shinya Kitaoka 120a6e
void Tokenizer::setBuffer(std::string buffer) {
Shinya Kitaoka 120a6e
  m_buffer = buffer + '\0';
Shinya Kitaoka 120a6e
  m_index  = 0;
Shinya Kitaoka 120a6e
  m_tokens.clear();
Shinya Kitaoka 120a6e
Shinya Kitaoka 120a6e
  bool stringBlock = false;
Shinya Kitaoka 120a6e
Shinya Kitaoka 120a6e
  int i         = 0;
Shinya Kitaoka 120a6e
  const char *s = &m_buffer[0];
Shinya Kitaoka 120a6e
  for (;;) {
Shinya Kitaoka 120a6e
    while (isascii(s[i]) && isspace(s[i])) i++;
Shinya Kitaoka 120a6e
Shinya Kitaoka 120a6e
    int j = i;
Shinya Kitaoka 120a6e
Shinya Kitaoka 120a6e
    if (s[i] == '\0') {
Shinya Kitaoka 120a6e
      m_tokens.push_back(Token("", Token::Eos, j));
Shinya Kitaoka 120a6e
      break;
Shinya Kitaoka 120a6e
    }
Shinya Kitaoka 120a6e
Shinya Kitaoka 120a6e
    if (s[i] == '"') {
Shinya Kitaoka 120a6e
      stringBlock = !stringBlock;
Shinya Kitaoka 120a6e
      m_tokens.push_back(Token("\"", Token::Punct, j));
Shinya Kitaoka 120a6e
Shinya Kitaoka 120a6e
      ++i;
Shinya Kitaoka 120a6e
      continue;
Shinya Kitaoka 120a6e
    }
Shinya Kitaoka 120a6e
Shinya Kitaoka 120a6e
    std::string token;
Shinya Kitaoka 120a6e
Shinya Kitaoka 120a6e
    if (stringBlock) {
Shinya Kitaoka 120a6e
      // string block - read mercilessly until either another '"' or EOS
Shinya Kitaoka 120a6e
      token = std::string(1, s[i++]);
Shinya Kitaoka 120a6e
Shinya Kitaoka 120a6e
      while (s[i] != '"' && s[i] != '\0') token.append(1, s[i++]);
Shinya Kitaoka 120a6e
Shinya Kitaoka 120a6e
      m_tokens.push_back(Token(token, Token::Ident, j));
Shinya Kitaoka 120a6e
    } else if (isascii(s[i]) && isalpha(s[i]) || s[i] == '_') {
Shinya Kitaoka 120a6e
      // ident
Shinya Kitaoka 120a6e
      token = std::string(1, s[i++]);
Shinya Kitaoka 120a6e
Shinya Kitaoka 120a6e
      while (isascii(s[i]) && (isalpha(s[i]) || s[i] == '_' || isdigit(s[i])))
Shinya Kitaoka 120a6e
        token.append(1, s[i++]);
Shinya Kitaoka 120a6e
Shinya Kitaoka 120a6e
      m_tokens.push_back(Token(token, Token::Ident, j));
Shinya Kitaoka 120a6e
    } else if (isascii(s[i]) && isdigit(s[i]) || s[i] == '.') {
Shinya Kitaoka 120a6e
      // number
Shinya Kitaoka 120a6e
      while (isascii(s[i]) && isdigit(s[i])) token.append(1, s[i++]);
Shinya Kitaoka 120a6e
Shinya Kitaoka 120a6e
      if (s[i] == '.') {
Shinya Kitaoka 120a6e
        token.append(1, s[i++]);
Shinya Kitaoka 120a6e
Shinya Kitaoka 120a6e
        while (isascii(s[i]) && isdigit(s[i])) token.append(1, s[i++]);
Shinya Kitaoka 120a6e
Shinya Kitaoka 120a6e
        if ((s[i] == 'e' || s[i] == 'E') &&
Shinya Kitaoka 120a6e
            (isascii(s[i + 1]) && isdigit(s[i + 1]) ||
Shinya Kitaoka 120a6e
             (s[i + 1] == '-' || s[i + 1] == '+') && isascii(s[i + 2]) &&
Shinya Kitaoka 120a6e
                 isdigit(s[i + 2]))) {
Shinya Kitaoka 120a6e
          token.append(1, s[i++]);
Shinya Kitaoka 120a6e
Shinya Kitaoka 120a6e
          if (s[i] == '-' || s[i] == '+') token.append(1, s[i++]);
Shinya Kitaoka 120a6e
Shinya Kitaoka 120a6e
          while (isascii(s[i]) && isdigit(s[i])) token.append(1, s[i++]);
Shinya Kitaoka 120a6e
        }
Shinya Kitaoka 120a6e
      }
Shinya Kitaoka 120a6e
      m_tokens.push_back(Token(token, Token::Number, j));
Shinya Kitaoka 120a6e
    } else {
Shinya Kitaoka 120a6e
      // punct.
Shinya Kitaoka 120a6e
      if (s[i + 1] != '\0') {
Shinya Kitaoka 120a6e
        token = std::string(s + i, 2);
Shinya Kitaoka 120a6e
Shinya Kitaoka 120a6e
        const std::string ss[] = {"==", "!=", ">=", "<=", "||", "&&"};
Shinya Kitaoka 120a6e
Shinya Kitaoka 120a6e
        const int m = tArrayCount(ss);
Shinya Kitaoka 120a6e
        if (std::find(ss, ss + m, token) != ss + m)
Shinya Kitaoka 120a6e
          i += 2;
Shinya Kitaoka 120a6e
        else
Shinya Kitaoka 120a6e
          token = std::string(1, s[i++]);
Shinya Kitaoka 120a6e
      } else
Shinya Kitaoka 120a6e
        token = std::string(1, s[i++]);
Shinya Kitaoka 120a6e
Shinya Kitaoka 120a6e
      m_tokens.push_back(Token(token, Token::Punct, j));
Shinya Kitaoka 120a6e
    }
Shinya Kitaoka 120a6e
  }
Toshihiro Shimizu 890ddd
}
Toshihiro Shimizu 890ddd
Toshihiro Shimizu 890ddd
//-------------------------------------------------------------------
Toshihiro Shimizu 890ddd
Shinya Kitaoka 120a6e
int Tokenizer::getTokenCount() const { return m_tokens.size(); }
Toshihiro Shimizu 890ddd
Toshihiro Shimizu 890ddd
//-------------------------------------------------------------------
Toshihiro Shimizu 890ddd
Shinya Kitaoka 120a6e
const Token &Tokenizer::getToken(int index) const {
Shinya Kitaoka 120a6e
  assert(0 <= index && index < getTokenCount());
Shinya Kitaoka 120a6e
  return m_tokens[index];
Toshihiro Shimizu 890ddd
}
Toshihiro Shimizu 890ddd
Toshihiro Shimizu 890ddd
//-------------------------------------------------------------------
Toshihiro Shimizu 890ddd
Shinya Kitaoka 120a6e
void Tokenizer::reset() { m_index = 0; }
Toshihiro Shimizu 890ddd
Toshihiro Shimizu 890ddd
//-------------------------------------------------------------------
Toshihiro Shimizu 890ddd
Shinya Kitaoka 120a6e
const Token &Tokenizer::getToken() { return getToken(m_index); }
Toshihiro Shimizu 890ddd
Toshihiro Shimizu 890ddd
//-------------------------------------------------------------------
Toshihiro Shimizu 890ddd
Shinya Kitaoka 120a6e
Token Tokenizer::nextToken() {
Shinya Kitaoka 120a6e
  Token token = getToken();
Shinya Kitaoka 120a6e
  if (m_index + 1 < getTokenCount()) m_index++;
Shinya Kitaoka 120a6e
  return token;
Toshihiro Shimizu 890ddd
}
Toshihiro Shimizu 890ddd
Toshihiro Shimizu 890ddd
//-------------------------------------------------------------------
Toshihiro Shimizu 890ddd
Shinya Kitaoka 120a6e
bool Tokenizer::eos() const { return m_index + 1 == getTokenCount(); }
Toshihiro Shimizu 890ddd
Toshihiro Shimizu 890ddd
//-------------------------------------------------------------------
Toshihiro Shimizu 890ddd
Shinya Kitaoka 120a6e
Token Tokenizer::getTokenFromPos(int pos) const {
Shinya Kitaoka 120a6e
  int len = m_buffer.length();
Shinya Kitaoka 120a6e
  if (pos < 0 || pos >= len) return Token(pos);
Shinya Kitaoka 120a6e
  int x = 0;
Shinya Kitaoka 120a6e
  for (int i = 0; i < getTokenCount(); i++) {
Shinya Kitaoka 120a6e
    const Token &token = getToken(i);
Shinya Kitaoka 120a6e
    int y              = token.getPos();
Shinya Kitaoka 120a6e
    if (pos < y) {
Shinya Kitaoka 120a6e
      assert(x < y);
Shinya Kitaoka 120a6e
      return Token(x, y - 1);
Shinya Kitaoka 120a6e
    }
Shinya Kitaoka 120a6e
    x = y + (int)token.getText().length();
Shinya Kitaoka 120a6e
    if (pos < x) return token;
Shinya Kitaoka 120a6e
  }
Shinya Kitaoka 120a6e
  assert(x < len);
Shinya Kitaoka 120a6e
  return Token(x, len - 1);
Toshihiro Shimizu 890ddd
}
Toshihiro Shimizu 890ddd
Toshihiro Shimizu 890ddd
//===================================================================
Toshihiro Shimizu 890ddd
Shinya Kitaoka 120a6e
}  // TSyntax