#ifndef TTOKENIZER_INCLUDED
#define TTOKENIZER_INCLUDED

#include "tcommon.h"

#ifdef WIN32
#pragma warning(push)
#pragma warning(disable : 4251)
#endif

#undef DVAPI
#undef DVVAR
#ifdef TNZBASE_EXPORTS
#define DVAPI DV_EXPORT_API
#define DVVAR DV_EXPORT_VAR
#else
#define DVAPI DV_IMPORT_API
#define DVVAR DV_IMPORT_VAR
#endif

namespace TSyntax
{

//-------------------------------------------------------------------

class DVAPI Token
{
public:
	enum Type { None,
				Space,
				Ident,
				Number,
				Punct,
				Eos };

	Token(int pos = 0)
		: m_text(""), m_type(None), m_pos(pos)
	{
	}
	Token(int p0, int p1)
		: m_text(p1 - p0 + 1, ' '), m_type(Space), m_pos(p0)
	{
	}

	Token(string text, Type type, int pos)
		: m_text(text), m_type(type), m_pos(pos)
	{
	}

	Type getType() const { return m_type; }

	string getText() const { return m_text; }
	int getIntValue() const;
	double getDoubleValue() const;

	int getPos() const { return m_pos; }
	int getPos1() const { return m_pos + m_text.length() - 1; }

private:
	string m_text;
	int m_pos;
	Type m_type;
};

//-------------------------------------------------------------------

class DVAPI Tokenizer
{

	string m_buffer;
	std::vector<Token> m_tokens;
	int m_index;

public:
	Tokenizer();
	Tokenizer(string buffer);
	~Tokenizer();

	void setBuffer(string buffer);
	string getBuffer() const { return m_buffer; }

	int getTokenCount() const;
	const Token &getToken(int index) const;

	//! get the token containing the pos-th character in the input string
	Token getTokenFromPos(int pos) const;

	//! reset the token index. (set it to 0)
	void reset();

	//! return the current token (possibly Eos)
	const Token &getToken();

	//! same as getToken(), but post-increment the token index (if !eos())
	Token nextToken();

	//! return true if the current token is the last one
	bool eos() const;

	//! to read all the sequence:
	//! while(!tokenizer.eos()) {token = tokenizer.nextToken();...}
};

//-------------------------------------------------------------------

} // namespace TSyntax

#ifdef WIN32
#pragma warning(pop)
#endif

#endif