src/parser/Tokenizer.h

   1 #ifndef SQUID_PARSER_TOKENIZER_H_
   2 #define SQUID_PARSER_TOKENIZER_H_
   3
   4 #include "CharacterSet.h"
   5 #include "SBuf.h"
   6
   7 namespace Parser {
   8
   9 class Tokenizer {
  10 public:
  11    explicit Tokenizer(const SBuf &inBuf) : buf_(inBuf) {}
  12
  13    bool atEnd() const { return !buf_.length(); }
  14    const SBuf& remaining() const { return buf_; }
  15    void reset(const SBuf &newBuf) { buf_ = newBuf; }
  16
  17    /* The following methods start from the beginning of the input buffer.
  18     * They return true and consume parsed chars if a non-empty token is found.
  19     * Otherwise, they return false without any side-effects. */
  20
  21    /** Basic strtok(3):
  22     *  Skips all leading delimiters (if any),
  23     *  accumulates all characters up to the first delimiter (a token), and
  24     *  skips all trailing delimiters (if any).
  25     *  Want to extract delimiters? Use three prefix() calls instead.
  26     */
  27    bool token(SBuf &returnedToken, const CharacterSet &whitespace);
  28
  29    /// Accumulates all sequential permitted characters (a token).
  30    bool prefix(SBuf &returnedToken, const CharacterSet &tokenChars);
  31
  32    /// Skips all sequential permitted characters (a token).
  33    bool skip(const CharacterSet &tokenChars);
  34
  35    /// Skips a given token.
  36    bool skip(const SBuf &tokenToSkip);
  37
  38    /// Skips a given character (a token).
  39    bool skip(const char tokenChar);
  40
  41 protected:
  42     //obtain the length of the longest prefix in buf_ only made of chars in tokenChars
  43     SBuf::size_type findFirstNotIn(const CharacterSet& tokenChars, SBuf::size_type startAtPos = 0);
  44     SBuf::size_type findFirstIn(const CharacterSet& tokenChars, SBuf::size_type startAtPos = 0);
  45
  46 private:
  47    SBuf buf_; ///< yet unparsed input
  48 };
  49
  50
  51 } /* namespace Parser */
  52 #endif /* SQUID_PARSER_TOKENIZER_H_ */