Toshihiro Shimizu 890ddd
Toshihiro Shimizu 890ddd
Toshihiro Shimizu 890ddd
#include "tparser.h"
Toshihiro Shimizu 890ddd
#include "tgrammar.h"
Toshihiro Shimizu 890ddd
#include "ttokenizer.h"
Toshihiro Shimizu 890ddd
#include "tutil.h"
Toshihiro Shimizu 890ddd
Toshihiro Shimizu 890ddd
#include <iostream></iostream>
Toshihiro Shimizu 890ddd
Toshihiro Shimizu 890ddd
namespace TSyntax
Toshihiro Shimizu 890ddd
{
Toshihiro Shimizu 890ddd
Toshihiro Shimizu 890ddd
//-------------------------------------------------------------------
Toshihiro Shimizu 890ddd
// RunningPattern
Toshihiro Shimizu 890ddd
//-------------------------------------------------------------------
Toshihiro Shimizu 890ddd
Toshihiro Shimizu 890ddd
class RunningPattern
Toshihiro Shimizu 890ddd
{
Toshihiro Shimizu 890ddd
public:
Toshihiro Shimizu 890ddd
	std::vector<token> m_tokens;</token>
Toshihiro Shimizu 890ddd
	const Pattern *m_pattern;
Toshihiro Shimizu 890ddd
Toshihiro Shimizu 890ddd
	RunningPattern(const Pattern *pattern) : m_pattern(pattern) {}
Toshihiro Shimizu 890ddd
Toshihiro Shimizu 890ddd
	int getPriority() const { return m_pattern->getPriority(); }
Toshihiro Shimizu 890ddd
	bool isFinished(const Token &token) const { return m_pattern->isFinished(m_tokens, token); }
Toshihiro Shimizu 890ddd
	bool isComplete(const Token &token) const { return m_pattern->isComplete(m_tokens, token); }
Toshihiro Shimizu 890ddd
	bool expressionExpected() const { return m_pattern->expressionExpected(m_tokens); }
Toshihiro Shimizu 890ddd
	bool matchToken(const Token &token) const { return m_pattern->matchToken(m_tokens, token); }
Toshihiro Shimizu 890ddd
	void advance()
Toshihiro Shimizu 890ddd
	{
Toshihiro Shimizu 890ddd
		assert(!isFinished(Token()));
Toshihiro Shimizu 890ddd
		m_tokens.push_back(Token());
Toshihiro Shimizu 890ddd
	}
Toshihiro Shimizu 890ddd
	void advance(const Token &token)
Toshihiro Shimizu 890ddd
	{
Toshihiro Shimizu 890ddd
		assert(!isFinished(token));
Toshihiro Shimizu 890ddd
		m_tokens.push_back(token);
Toshihiro Shimizu 890ddd
	}
Toshihiro Shimizu 890ddd
	TokenType getTokenType(const Token &token) const { return m_pattern->getTokenType(m_tokens, token); }
Toshihiro Shimizu 890ddd
Toshihiro Shimizu 890ddd
	void createNode(Calculator *calc, std::vector<calculatornode *=""> &stack)</calculatornode>
Toshihiro Shimizu 890ddd
	{
Toshihiro Shimizu 890ddd
		m_pattern->createNode(calc, stack, m_tokens);
Toshihiro Shimizu 890ddd
	}
Toshihiro Shimizu 890ddd
};
Toshihiro Shimizu 890ddd
Toshihiro Shimizu 890ddd
//-------------------------------------------------------------------
Toshihiro Shimizu 890ddd
// Parser::Imp
Toshihiro Shimizu 890ddd
//-------------------------------------------------------------------
Toshihiro Shimizu 890ddd
Toshihiro Shimizu 890ddd
class Parser::Imp
Toshihiro Shimizu 890ddd
{
Toshihiro Shimizu 890ddd
public:
Toshihiro Shimizu 890ddd
	const Grammar *m_grammar;
Toshihiro Shimizu 890ddd
	Tokenizer m_tokenizer;
Shinya Kitaoka 3bfa54
	std::string m_errorString;
Toshihiro Shimizu 890ddd
	bool m_isValid;
Toshihiro Shimizu 890ddd
	Calculator *m_calculator;
Toshihiro Shimizu 890ddd
	std::vector<runningpattern> m_patternStack;</runningpattern>
Toshihiro Shimizu 890ddd
	std::vector<calculatornode *=""> m_nodeStack;</calculatornode>
Toshihiro Shimizu 890ddd
	std::vector<syntaxtoken> m_syntaxTokens;</syntaxtoken>
Toshihiro Shimizu 890ddd
	Grammar::Position m_position;
Toshihiro Shimizu 890ddd
	// Pattern *m_lastPattern;
Toshihiro Shimizu 890ddd
Toshihiro Shimizu 890ddd
	Imp(const Grammar *grammar)
Toshihiro Shimizu 890ddd
		: m_grammar(grammar), m_errorString(""), m_isValid(false), m_calculator(0), m_position(Grammar::ExpressionStart)
Toshihiro Shimizu 890ddd
	{
Toshihiro Shimizu 890ddd
	}
Toshihiro Shimizu 890ddd
	~Imp()
Toshihiro Shimizu 890ddd
	{
Toshihiro Shimizu 890ddd
		clearPointerContainer(m_nodeStack);
Toshihiro Shimizu 890ddd
		delete m_calculator;
Toshihiro Shimizu 890ddd
	}
Toshihiro Shimizu 890ddd
Toshihiro Shimizu 890ddd
	void pushSyntaxToken(TokenType type);
Toshihiro Shimizu 890ddd
	bool parseExpression(bool checkOnly);
Toshihiro Shimizu 890ddd
	void flushPatterns(int minPriority, int minIndex, bool checkOnly);
Toshihiro Shimizu 890ddd
	bool checkSyntax(std::vector<syntaxtoken> &tokens);</syntaxtoken>
Toshihiro Shimizu 890ddd
};
Toshihiro Shimizu 890ddd
Toshihiro Shimizu 890ddd
//-------------------------------------------------------------------
Toshihiro Shimizu 890ddd
// Parser
Toshihiro Shimizu 890ddd
//-------------------------------------------------------------------
Toshihiro Shimizu 890ddd
Toshihiro Shimizu 890ddd
Parser::Parser(const Grammar *grammar)
Toshihiro Shimizu 890ddd
	: m_imp(new Imp(grammar))
Toshihiro Shimizu 890ddd
{
Toshihiro Shimizu 890ddd
}
Toshihiro Shimizu 890ddd
Toshihiro Shimizu 890ddd
//-------------------------------------------------------------------
Toshihiro Shimizu 890ddd
Toshihiro Shimizu 890ddd
Parser::~Parser()
Toshihiro Shimizu 890ddd
{
Toshihiro Shimizu 890ddd
}
Toshihiro Shimizu 890ddd
Toshihiro Shimizu 890ddd
//-------------------------------------------------------------------
Toshihiro Shimizu 890ddd
Toshihiro Shimizu 890ddd
void Parser::Imp::flushPatterns(int minPriority, int minIndex, bool checkOnly)
Toshihiro Shimizu 890ddd
{
Toshihiro Shimizu 890ddd
	while ((int)m_patternStack.size() > minIndex && m_patternStack.back().getPriority() >= minPriority) {
Toshihiro Shimizu 890ddd
		if (!checkOnly)
Toshihiro Shimizu 890ddd
			m_patternStack.back().createNode(m_calculator, m_nodeStack);
Toshihiro Shimizu 890ddd
		m_patternStack.pop_back();
Toshihiro Shimizu 890ddd
	}
Toshihiro Shimizu 890ddd
}
Toshihiro Shimizu 890ddd
Toshihiro Shimizu 890ddd
//-------------------------------------------------------------------
Toshihiro Shimizu 890ddd
Toshihiro Shimizu 890ddd
void Parser::Imp::pushSyntaxToken(TokenType type)
Toshihiro Shimizu 890ddd
{
Toshihiro Shimizu 890ddd
	SyntaxToken syntaxToken;
Toshihiro Shimizu 890ddd
	Token token = m_tokenizer.getToken();
Toshihiro Shimizu 890ddd
	syntaxToken.m_pos = token.getPos();
Toshihiro Shimizu 890ddd
	syntaxToken.m_length = token.getPos1() - token.getPos() + 1;
Toshihiro Shimizu 890ddd
	syntaxToken.m_type = type;
Toshihiro Shimizu 890ddd
	m_syntaxTokens.push_back(syntaxToken);
Toshihiro Shimizu 890ddd
}
Toshihiro Shimizu 890ddd
Toshihiro Shimizu 890ddd
//-------------------------------------------------------------------
Toshihiro Shimizu 890ddd
Toshihiro Shimizu 890ddd
bool Parser::Imp::parseExpression(bool checkOnly)
Toshihiro Shimizu 890ddd
{
Toshihiro Shimizu 890ddd
	if (!m_grammar)
Toshihiro Shimizu 890ddd
		return false;
Toshihiro Shimizu 890ddd
	int stackMinIndex = m_patternStack.size();
Toshihiro Shimizu 890ddd
	int count = 0;
Toshihiro Shimizu 890ddd
	Grammar::Position position = Grammar::ExpressionStart;
Toshihiro Shimizu 890ddd
	m_position = position;
Toshihiro Shimizu 890ddd
	// warning: parseExpression() is called recursively. m_position is used after parseExpression()
Toshihiro Shimizu 890ddd
	// for diagnostic and suggestions. position is used in the actual parsing
Toshihiro Shimizu 890ddd
Toshihiro Shimizu 890ddd
	bool expressionExpected = true;
Toshihiro Shimizu 890ddd
	while (!m_tokenizer.eos()) {
Toshihiro Shimizu 890ddd
		// token, position -> pattern
Toshihiro Shimizu 890ddd
		const Pattern *pattern = m_grammar->getPattern(position, m_tokenizer.getToken());
Toshihiro Shimizu 890ddd
		if (!pattern) {
Toshihiro Shimizu 890ddd
			// no pattern found for the next token
Toshihiro Shimizu 890ddd
			if (position == Grammar::ExpressionEnd) {
Toshihiro Shimizu 890ddd
				// already found an expression
Toshihiro Shimizu 890ddd
				flushPatterns(-1, stackMinIndex, checkOnly);
Toshihiro Shimizu 890ddd
				return true;
Toshihiro Shimizu 890ddd
			} else {
Toshihiro Shimizu 890ddd
				// syntax error : expression not found
Toshihiro Shimizu 890ddd
				if (checkOnly)
Toshihiro Shimizu 890ddd
					pushSyntaxToken(UnexpectedToken);
Toshihiro Shimizu 890ddd
Toshihiro Shimizu 890ddd
				m_errorString = "Unexpected token";
Toshihiro Shimizu 890ddd
				return false;
Toshihiro Shimizu 890ddd
			}
Toshihiro Shimizu 890ddd
		}
Toshihiro Shimizu 890ddd
		// pattern found: start scanning it
Toshihiro Shimizu 890ddd
		RunningPattern runningPattern(pattern);
Toshihiro Shimizu 890ddd
		if (position == Grammar::ExpressionEnd) // patterns of the form "<expr> ...."</expr>
Toshihiro Shimizu 890ddd
			runningPattern.advance();
Toshihiro Shimizu 890ddd
Toshihiro Shimizu 890ddd
		// eat the first token
Toshihiro Shimizu 890ddd
		if (checkOnly)
Toshihiro Shimizu 890ddd
			pushSyntaxToken(runningPattern.getTokenType(m_tokenizer.getToken()));
Toshihiro Shimizu 890ddd
		runningPattern.advance(m_tokenizer.getToken());
Toshihiro Shimizu 890ddd
		m_tokenizer.nextToken();
Toshihiro Shimizu 890ddd
		expressionExpected = false;
Toshihiro Shimizu 890ddd
Toshihiro Shimizu 890ddd
		while (!runningPattern.isFinished(m_tokenizer.getToken())) {
Toshihiro Shimizu 890ddd
			if (runningPattern.expressionExpected()) {
Toshihiro Shimizu 890ddd
				// expression expected
Toshihiro Shimizu 890ddd
				runningPattern.advance();
Toshihiro Shimizu 890ddd
				if (runningPattern.isFinished(m_tokenizer.getToken())) {
Toshihiro Shimizu 890ddd
					// pattern ended with an expression
Toshihiro Shimizu 890ddd
					expressionExpected = true;
Toshihiro Shimizu 890ddd
					break;
Toshihiro Shimizu 890ddd
				}
Toshihiro Shimizu 890ddd
				// parse a sub expression
Toshihiro Shimizu 890ddd
				if (!parseExpression(checkOnly)) {
Toshihiro Shimizu 890ddd
					return false;
Toshihiro Shimizu 890ddd
				}
Toshihiro Shimizu 890ddd
			} else {
Toshihiro Shimizu 890ddd
				// "terminal" expected
Toshihiro Shimizu 890ddd
				if (m_tokenizer.eos()) {
Toshihiro Shimizu 890ddd
					// EOS
Toshihiro Shimizu 890ddd
					if (runningPattern.isComplete(Token()))
Toshihiro Shimizu 890ddd
						break;
Toshihiro Shimizu 890ddd
					if (checkOnly)
Toshihiro Shimizu 890ddd
						pushSyntaxToken(Eos);
Toshihiro Shimizu 890ddd
Toshihiro Shimizu 890ddd
					m_errorString = "Uncompleted syntax";
Toshihiro Shimizu 890ddd
					return false;
Toshihiro Shimizu 890ddd
				}
Toshihiro Shimizu 890ddd
				if (!runningPattern.matchToken(m_tokenizer.getToken())) {
Toshihiro Shimizu 890ddd
					// token mismatch
Toshihiro Shimizu 890ddd
					if (runningPattern.isComplete(m_tokenizer.getToken()))
Toshihiro Shimizu 890ddd
						break;
Toshihiro Shimizu 890ddd
					if (checkOnly)
Toshihiro Shimizu 890ddd
						pushSyntaxToken(Mismatch);
Toshihiro Shimizu 890ddd
					m_errorString = "Syntax error";
Toshihiro Shimizu 890ddd
					return false;
Toshihiro Shimizu 890ddd
				}
Toshihiro Shimizu 890ddd
				// token matched
Toshihiro Shimizu 890ddd
				if (checkOnly)
Toshihiro Shimizu 890ddd
					pushSyntaxToken(runningPattern.getTokenType(m_tokenizer.getToken()));
Toshihiro Shimizu 890ddd
				runningPattern.advance(m_tokenizer.getToken());
Toshihiro Shimizu 890ddd
				m_tokenizer.nextToken();
Toshihiro Shimizu 890ddd
			}
Toshihiro Shimizu 890ddd
		}
Toshihiro Shimizu 890ddd
Toshihiro Shimizu 890ddd
		// pattern ended
Toshihiro Shimizu 890ddd
		if (expressionExpected) {
Toshihiro Shimizu 890ddd
			// pattern terminated with an expression
Toshihiro Shimizu 890ddd
			if (position == Grammar::ExpressionEnd) {
Toshihiro Shimizu 890ddd
				// "E op . E"
Toshihiro Shimizu 890ddd
				flushPatterns(runningPattern.getPriority(), stackMinIndex, checkOnly);
Toshihiro Shimizu 890ddd
				m_patternStack.push_back(runningPattern);
Toshihiro Shimizu 890ddd
			} else {
Toshihiro Shimizu 890ddd
				// "op . E"
Toshihiro Shimizu 890ddd
				m_patternStack.push_back(runningPattern);
Toshihiro Shimizu 890ddd
			}
Toshihiro Shimizu 890ddd
			m_position = position = Grammar::ExpressionStart;
Toshihiro Shimizu 890ddd
		} else {
Toshihiro Shimizu 890ddd
			// pattern terminates with a keyword
Toshihiro Shimizu 890ddd
			if (position == Grammar::ExpressionEnd) {
Toshihiro Shimizu 890ddd
				// "E op ."
Toshihiro Shimizu 890ddd
				flushPatterns(runningPattern.getPriority(), stackMinIndex, checkOnly);
Toshihiro Shimizu 890ddd
			} else {
Toshihiro Shimizu 890ddd
				// "leaf .", "f(E) .", "(E) ."
Toshihiro Shimizu 890ddd
			}
Toshihiro Shimizu 890ddd
			if (!checkOnly)
Toshihiro Shimizu 890ddd
				runningPattern.createNode(m_calculator, m_nodeStack);
Toshihiro Shimizu 890ddd
			count++;
Toshihiro Shimizu 890ddd
			m_position = position = Grammar::ExpressionEnd;
Toshihiro Shimizu 890ddd
		}
Toshihiro Shimizu 890ddd
	}
Toshihiro Shimizu 890ddd
	if (count == 0 || expressionExpected) {
Toshihiro Shimizu 890ddd
Toshihiro Shimizu 890ddd
		m_errorString = "Expression expected";
Toshihiro Shimizu 890ddd
		return false;
Toshihiro Shimizu 890ddd
	}
Toshihiro Shimizu 890ddd
	flushPatterns(-1, stackMinIndex, checkOnly);
Toshihiro Shimizu 890ddd
	return true;
Toshihiro Shimizu 890ddd
}
Toshihiro Shimizu 890ddd
Toshihiro Shimizu 890ddd
//-------------------------------------------------------------------
Toshihiro Shimizu 890ddd
Shinya Kitaoka 3bfa54
Calculator *Parser::parse(std::string text)
Toshihiro Shimizu 890ddd
{
Toshihiro Shimizu 890ddd
	m_imp->m_tokenizer.setBuffer(text);
Toshihiro Shimizu 890ddd
	clearPointerContainer(m_imp->m_nodeStack);
Toshihiro Shimizu 890ddd
	m_imp->m_errorString = "";
Toshihiro Shimizu 890ddd
	m_imp->m_isValid = false;
Toshihiro Shimizu 890ddd
	m_imp->m_calculator = new Calculator();
Toshihiro Shimizu 890ddd
	bool ret = m_imp->parseExpression(false);
Toshihiro Shimizu 890ddd
	if (ret && !m_imp->m_nodeStack.empty()) {
Toshihiro Shimizu 890ddd
		m_imp->m_calculator->setRootNode(m_imp->m_nodeStack.back());
Toshihiro Shimizu 890ddd
		m_imp->m_nodeStack.pop_back();
Toshihiro Shimizu 890ddd
		m_imp->m_isValid = true;
Toshihiro Shimizu 890ddd
	} else {
Toshihiro Shimizu 890ddd
		delete m_imp->m_calculator;
Toshihiro Shimizu 890ddd
		m_imp->m_calculator = 0;
Toshihiro Shimizu 890ddd
	}
Toshihiro Shimizu 890ddd
	clearPointerContainer(m_imp->m_nodeStack);
Toshihiro Shimizu 890ddd
	Calculator *calculator = m_imp->m_calculator;
Toshihiro Shimizu 890ddd
	m_imp->m_calculator = 0;
Toshihiro Shimizu 890ddd
	return calculator;
Toshihiro Shimizu 890ddd
}
Toshihiro Shimizu 890ddd
Toshihiro Shimizu 890ddd
//-------------------------------------------------------------------
Toshihiro Shimizu 890ddd
Shinya Kitaoka 3bfa54
Parser::SyntaxStatus Parser::checkSyntax(std::vector<syntaxtoken> &tokens, std::string text)</syntaxtoken>
Toshihiro Shimizu 890ddd
{
Toshihiro Shimizu 890ddd
	m_imp->m_tokenizer.setBuffer(text);
Toshihiro Shimizu 890ddd
	if (m_imp->m_tokenizer.eos())
Toshihiro Shimizu 890ddd
		return Incomplete;
Toshihiro Shimizu 890ddd
	bool ret = m_imp->parseExpression(true);
Toshihiro Shimizu 890ddd
	tokens = m_imp->m_syntaxTokens;
Toshihiro Shimizu 890ddd
	if (ret && m_imp->m_tokenizer.eos())
Toshihiro Shimizu 890ddd
		return Correct;
Toshihiro Shimizu 890ddd
	if (tokens.empty())
Toshihiro Shimizu 890ddd
		return Incomplete;
Toshihiro Shimizu 890ddd
	SyntaxToken &lastToken = tokens.back();
Toshihiro Shimizu 890ddd
	if (lastToken.m_type == Eos)
Toshihiro Shimizu 890ddd
		return Incomplete;
Toshihiro Shimizu 890ddd
Toshihiro Shimizu 890ddd
	int k = lastToken.m_pos + lastToken.m_length;
Toshihiro Shimizu 890ddd
	if (k >= (int)text.length()) {
Toshihiro Shimizu 890ddd
		if (lastToken.m_type < Unknown) {
Toshihiro Shimizu 890ddd
			lastToken.m_type = Unknown;
Toshihiro Shimizu 890ddd
		}
Toshihiro Shimizu 890ddd
		return ExtraIgnored;
Toshihiro Shimizu 890ddd
	}
Toshihiro Shimizu 890ddd
Toshihiro Shimizu 890ddd
	SyntaxToken extraIgnored;
Toshihiro Shimizu 890ddd
	extraIgnored.m_type = Unknown;
Toshihiro Shimizu 890ddd
	extraIgnored.m_pos = k;
Toshihiro Shimizu 890ddd
	extraIgnored.m_length = text.length() - k;
Toshihiro Shimizu 890ddd
	tokens.push_back(extraIgnored);
Toshihiro Shimizu 890ddd
Toshihiro Shimizu 890ddd
	return Error;
Toshihiro Shimizu 890ddd
}
Toshihiro Shimizu 890ddd
Toshihiro Shimizu 890ddd
//-------------------------------------------------------------------
Toshihiro Shimizu 890ddd
Shinya Kitaoka 3bfa54
void Parser::getSuggestions(Grammar::Suggestions &suggestions, std::string text)
Toshihiro Shimizu 890ddd
{
Toshihiro Shimizu 890ddd
	std::vector<syntaxtoken> tokens;</syntaxtoken>
Toshihiro Shimizu 890ddd
	Parser::SyntaxStatus status = checkSyntax(tokens, text);
Toshihiro Shimizu 890ddd
	suggestions.clear();
Toshihiro Shimizu 890ddd
	if (status == Correct || status == Incomplete || status == ExtraIgnored)
Toshihiro Shimizu 890ddd
		m_imp->m_grammar->getSuggestions(suggestions, m_imp->m_position);
Toshihiro Shimizu 890ddd
}
Toshihiro Shimizu 890ddd
Toshihiro Shimizu 890ddd
//-------------------------------------------------------------------
Toshihiro Shimizu 890ddd
Shinya Kitaoka 3bfa54
std::string Parser::getCurrentPatternString(std::string text)
Toshihiro Shimizu 890ddd
{
Toshihiro Shimizu 890ddd
	return "ohime";
Toshihiro Shimizu 890ddd
}
Toshihiro Shimizu 890ddd
Toshihiro Shimizu 890ddd
//-------------------------------------------------------------------
Toshihiro Shimizu 890ddd
Toshihiro Shimizu 890ddd
bool Parser::isValid() const
Toshihiro Shimizu 890ddd
{
Toshihiro Shimizu 890ddd
	return m_imp->m_isValid;
Toshihiro Shimizu 890ddd
}
Toshihiro Shimizu 890ddd
Toshihiro Shimizu 890ddd
//-------------------------------------------------------------------
Toshihiro Shimizu 890ddd
Shinya Kitaoka 3bfa54
std::string Parser::getText() const
Toshihiro Shimizu 890ddd
{
Toshihiro Shimizu 890ddd
	return m_imp->m_tokenizer.getBuffer();
Toshihiro Shimizu 890ddd
}
Toshihiro Shimizu 890ddd
Toshihiro Shimizu 890ddd
//-------------------------------------------------------------------
Toshihiro Shimizu 890ddd
Shinya Kitaoka 3bfa54
std::string Parser::getError() const
Toshihiro Shimizu 890ddd
{
Toshihiro Shimizu 890ddd
	return m_imp->m_errorString;
Toshihiro Shimizu 890ddd
}
Toshihiro Shimizu 890ddd
Toshihiro Shimizu 890ddd
//-------------------------------------------------------------------
Toshihiro Shimizu 890ddd
Toshihiro Shimizu 890ddd
std::pair<int, int=""> Parser::getErrorPos() const</int,>
Toshihiro Shimizu 890ddd
{
Toshihiro Shimizu 890ddd
	if (m_imp->m_errorString == "")
Toshihiro Shimizu 890ddd
		return std::make_pair(0, -1);
Toshihiro Shimizu 890ddd
	Token token = m_imp->m_tokenizer.getToken();
Toshihiro Shimizu 890ddd
	return std::make_pair(token.getPos(), token.getPos1());
Toshihiro Shimizu 890ddd
}
Toshihiro Shimizu 890ddd
Toshihiro Shimizu 890ddd
//-------------------------------------------------------------------
Toshihiro Shimizu 890ddd
Toshihiro Shimizu 890ddd
} // namespace TSyntax