Fix Tokenizer::prefix when fetching length limited tokens

diff --git a/src/parser/Tokenizer.cc b/src/parser/Tokenizer.cc
index 3d43c08b80e3429b55ba38fea63fe772a04ebc32..c14937f67874968af8a7997e74cb5f6cb47b9bbb 100644 (file)
--- a/src/parser/Tokenizer.cc
+++ b/src/parser/Tokenizer.cc
@@ -71,11 +71,13 @@ Parser::Tokenizer::token(SBuf &returnedToken, const CharacterSet &delimiters)
 bool
 Parser::Tokenizer::prefix(SBuf &returnedToken, const CharacterSet &tokenChars, const SBuf::size_type limit)
 {
-    const SBuf::size_type prefixLen = buf_.substr(0,limit).findFirstNotOf(tokenChars);
+    SBuf::size_type prefixLen = buf_.substr(0,limit).findFirstNotOf(tokenChars);
     if (prefixLen == 0)
         return false;
     if (prefixLen == SBuf::npos && (atEnd() || limit == 0))
         return false;
+    if (prefixLen == SBuf::npos && limit > 0)
+        prefixLen = limit;
     returnedToken = consume(prefixLen); // cannot be empty after the npos check
     return true;
 }