00001
00002
00003 #ifndef MIX_TOKENIZER_H_
00004 #define MIX_TOKENIZER_H_
00005
00006
00007 #include "classes.h"
00008 #include "misc.h"
00009 #include "XMLToken.h"
00010
00011 #include <stack>
00012 #include <string>
00013
00014 namespace MiX{
00015
00016 template <class charT,class char_traits,class xml_traits>
00017 class Tokenizer{
00018 public:
00019 MiX_Template_Typedef(charT,char_traits,xml_traits);
00020 typedef Tokenizer<charT,char_traits,xml_traits> this_type;
00021 typedef XMLToken<charT,char_traits,xml_traits> token_type;
00022 private:
00023 const charT* data_;
00024 const charT* start_;
00025 const charT* current_;
00031 charT* buf_;
00032 long len_;
00033
00034 charT null_;
00035 static const charT tokens_[TokenCount];
00036 std::stack<token_type> stack_;
00037 static TokenType isToken(charT);
00038
00039 std::basic_istream<charT, char_traits>* is_;
00040
00041 bool injectStringFromBuffer(const charT* text)
00042 { current_ = start_ = data_ = text; return true; }
00043
00044 bool fillUpBuffer();
00045 public:
00047 Tokenizer() : is_(0),buf_(0),null_(xml_traits::null()) { }
00049 ~Tokenizer(){ if(buf_) delete buf_; }
00051 bool injectString(const charT* text)
00052 { current_ = start_ = data_ = text; return true; }
00054 bool ejectToken(token_type& dest);
00056 void pushToken(const token_type& tok){
00057 stack_.push(tok);
00058 }
00059
00060 bool injectStream(std::basic_istream<charT, char_traits>& is)
00061 { is_ = &is; fillUpBuffer(); return true; }
00062
00063 };
00064
00065 }
00066
00067 #ifndef MIX_TOKENIZER_CPP_
00068 #include "Tokenizer.cpp"
00069 #endif
00070
00071 #endif