Add suffix operators to Tokenizer

In some situations we need to tokenize a fixed-length buffer in reverse.

diff --git a/src/parser/Tokenizer.cc b/src/parser/Tokenizer.cc
index c14937f67874968af8a7997e74cb5f6cb47b9bbb..6cf6adfc87bff896653c980d72cf9e4684ca2704 100644 (file)
--- a/src/parser/Tokenizer.cc
+++ b/src/parser/Tokenizer.cc
@@ -82,6 +82,27 @@ Parser::Tokenizer::prefix(SBuf &returnedToken, const CharacterSet &tokenChars, c
     return true;
 }
 
+bool
+Parser::Tokenizer::suffix(SBuf &returnedToken, const CharacterSet &tokenChars, const SBuf::size_type limit)
+{
+    SBuf span = buf_;
+
+    if (limit < buf_.length())
+        span.consume(buf_.length() - limit); // ignore the N prefix characters
+
+    auto i = span.rbegin();
+    SBuf::size_type found = 0;
+    while (i != span.rend() && tokenChars[*i]) {
+        ++i;
+        ++found;
+    }
+    if (!found)
+        return false;
+    returnedToken = buf_;
+    buf_ = returnedToken.consume(buf_.length() - found);
+    return true;
+}
+
 SBuf::size_type
 Parser::Tokenizer::skipAll(const CharacterSet &tokenChars)
 {
@@ -99,6 +120,23 @@ Parser::Tokenizer::skipOne(const CharacterSet &chars)
     return false;
 }
 
+bool
+Parser::Tokenizer::skipSuffix(const SBuf &tokenToSkip)
+{
+    if (buf_.length() < tokenToSkip.length())
+        return false;
+
+    SBuf::size_type offset = 0;
+    if (tokenToSkip.length() < buf_.length())
+        offset = buf_.length() - tokenToSkip.length();
+
+    if (buf_.substr(offset, SBuf::npos).cmp(tokenToSkip) == 0) {
+        buf_ = buf_.substr(0,offset);
+        return true;
+    }
+    return false;
+}
+
 bool
 Parser::Tokenizer::skip(const SBuf &tokenToSkip)
 {

diff --git a/src/parser/Tokenizer.h b/src/parser/Tokenizer.h
index 9cd30d912bc16fe5a58a43981cce92128bf8113f..2fcb91ffe490904fc2cb0afbec5bfc62f791a596 100644 (file)
--- a/src/parser/Tokenizer.h
+++ b/src/parser/Tokenizer.h
@@ -70,6 +70,27 @@ public:
      */
     bool prefix(SBuf &returnedToken, const CharacterSet &tokenChars, SBuf::size_type limit = SBuf::npos);
 
+    /** Extracts all sequential permitted characters up to an optional length limit.
+     * Operates on the trailing end of the buffer.
+     *
+     *  Note that Tokenizer cannot tell whether the buffer will
+     *  gain more data when/if more input becomes available later.
+     *
+     * \retval true one or more characters were found, the sequence (string) is placed in returnedToken
+     * \retval false no characters from the permitted set were found
+     */
+    bool suffix(SBuf &returnedToken, const CharacterSet &tokenChars, SBuf::size_type limit = SBuf::npos);
+
+    /** skips a given suffix character sequence (string)
+     * Operates on the trailing end of the buffer.
+     *
+     *  Note that Tokenizer cannot tell whether the buffer will
+     *  gain more data when/if more input becomes available later.
+     *
+     * \return whether the exact character sequence was found and skipped
+     */
+    bool skipSuffix(const SBuf &tokenToSkip);
+
     /** skips a given character sequence (string)
      *
      * \return whether the exact character sequence was found and skipped