Cleanup Tokenizer::token

* remove incorrect comment

* return false with no side effects if end of buffer is reached inside
  the token, unless \0 is defined as one of the delimiters.

diff --git a/src/parser/Tokenizer.cc b/src/parser/Tokenizer.cc
index 453f36a24841e4af49a0c979ef7e8125b6408e4e..b3bc3ce5a25d021cf61125612a9b190fde245e3d 100644 (file)
--- a/src/parser/Tokenizer.cc
+++ b/src/parser/Tokenizer.cc
@@ -5,12 +5,14 @@ bool
 Parser::Tokenizer::token(SBuf &returnedToken, const CharacterSet &delimiters)
 {
     SBuf savebuf(buf_);
-    SBuf retval;
-    SBuf::size_type tokenLen = 0;
     skip(delimiters);
-    // can't use prefix as we're looking for the first char not in delimiters
-    tokenLen = buf_.findFirstOf(delimiters); // not found = npos => consume to end
-    retval = buf_.consume(tokenLen);
+    SBuf::size_type tokenLen = buf_.findFirstOf(delimiters); // not found = npos => consume to end
+    if (tokenLen == SBuf::npos && !delimiters['\0']) {
+        // no delimiter found, nor is NUL/EOS/npos acceptible as one
+        buf_ = savebuf;
+        return false;
+    }
+    SBuf retval = buf_.consume(tokenLen);
     skip(delimiters);
     returnedToken = retval;
     return true;