Document token() behaviour at end-of-buffer

diff --git a/src/parser/Tokenizer.h b/src/parser/Tokenizer.h
index 9436e9f19c108ff9fb8a86f6e0559f0f86134d05..93796fb25b0e8ddfc3f3a183b2c883ebf8b67ded 100644 (file)
--- a/src/parser/Tokenizer.h
+++ b/src/parser/Tokenizer.h
@@ -36,9 +36,14 @@ public:
    /** Basic strtok(3):
     *  Skips all leading delimiters (if any),
     *  accumulates all characters up to the next delimiter (a token), and
-    *  skips all trailing delimiters (if any).
+    *  skips all trailing delimiters.
     *
     *  Want to extract delimiters? Use prefix() instead.
+    *
+    * At least one terminating delimiter is required. \0 may be passed
+    * as a delimiter to treat end of buffer content as the end of token.
+    *
+    * \return false if no terminal delimiter is found.
     */
    bool token(SBuf &returnedToken, const CharacterSet &delimiters);