Implemented more Tokenizer methods and unit tests

diff --git a/src/parser/Tokenizer.cc b/src/parser/Tokenizer.cc
index e0805621db1514b59fd038e2fe2a1ad4549ec268..716b204a5b1004eb90ebaa671f2d8a6faa5a7ca2 100644 (file)
--- a/src/parser/Tokenizer.cc
+++ b/src/parser/Tokenizer.cc
@@ -3,6 +3,34 @@
 
 namespace Parser {
 
+SBuf::size_type
+Tokenizer::findPrefixLen(const CharacterSet& tokenChars)
+{
+    SBuf::size_type prefixLen = 0;
+    const SBuf::size_type len = buf_.length();
+    while (prefixLen < len) {
+        if (!tokenChars[buf_[prefixLen]])
+            break;
+        ++prefixLen;
+    }
+    return prefixLen;
+}
+
+SBuf::size_type
+Tokenizer::findFirstOf(const CharacterSet& tokenChars)
+{
+    SBuf::size_type s = 0;
+    const SBuf::size_type len = buf_.length();
+    bool found = false;
+    while (s < len) {
+        if (tokenChars[buf_[prefixLen]]) {
+            found = true;
+            break;
+        }
+        ++s;
+    }
+}
+
 bool
 Tokenizer::token(SBuf &returnedToken, const CharacterSet &whitespace)
 {
@@ -13,13 +41,7 @@ Tokenizer::token(SBuf &returnedToken, const CharacterSet &whitespace)
 bool
 Tokenizer::prefix(SBuf &returnedToken, const CharacterSet &tokenChars)
 {
-    SBuf::size_type prefixLen = 0;
-    const SBuf::size_type len=buf_.length();
-    while (prefixLen < len) {
-        if (!tokenChars[buf_[prefixLen]])
-            break;
-        ++prefixLen;
-    }
+    SBuf::size_type prefixLen = findPrefixLen(tokenChars);
     if (prefixLen == 0)
         return false;
     returnedToken = buf_.consume(prefixLen);
@@ -29,21 +51,30 @@ Tokenizer::prefix(SBuf &returnedToken, const CharacterSet &tokenChars)
 bool
 Tokenizer::skip(const CharacterSet &tokenChars)
 {
-    //TODO
-    return false;
+    SBuf::size_type prefixLen = findPrefixLen(tokenChars);
+    if (prefixLen == 0)
+        return false;
+    buf_.consume(prefixLen);
+    return true;
 }
 
 bool
 Tokenizer::skip(const SBuf &tokenToSkip)
 {
-    //TODO
+    if (buf_.startsWith(tokenToSkip)) {
+        buf_.consume(tokenToSkip.length());
+        return true;
+    }
     return false;
 }
 
 bool
 Tokenizer::skip(const char tokenChar)
 {
-    //TODO
+    if (buf_[0] == tokenChar) {
+        buf_.consume(1);
+        return true;
+    }
     return false;
 }
 } /* namespace Parser */

diff --git a/src/parser/Tokenizer.h b/src/parser/Tokenizer.h
index 3f01ed9a86483b101818077bd3ac02d9a978a3bd..66a78460b7efb564766883509a61fadb90eeb18b 100644 (file)
--- a/src/parser/Tokenizer.h
+++ b/src/parser/Tokenizer.h
@@ -38,6 +38,11 @@ public:
    /// Skips a given character (a token).
    bool skip(const char tokenChar);
 
+protected:
+    //obtain the length of the longest prefix in buf_ only made of chars in tokenChars
+    SBuf::size_type findPrefixLen(const CharacterSet& tokenChars);
+    SBuf::size_type findFirstOf(const CharacterSet& tokenChars);
+
 private:
    SBuf buf_; ///< yet unparsed input
 };

diff --git a/src/parser/testTokenizer.cc b/src/parser/testTokenizer.cc
index 2efd18311585c78d2964608d957df7bb18558643..afb6d754d1ac11bf0aabaf3fe16bc4b30463b462 100644 (file)
--- a/src/parser/testTokenizer.cc
+++ b/src/parser/testTokenizer.cc
@@ -53,6 +53,30 @@ testTokenizer::testTokenizerPrefix()
 void
 testTokenizer::testTokenizerSkip()
 {
+    Parser::Tokenizer t(text);
+    SBuf s;
+
+    // first scenario: patterns match
+    // prep for test
+    CPPUNIT_ASSERT(t.prefix(s,alpha));
+    CPPUNIT_ASSERT_EQUAL(SBuf("GET"),s);
+
+    // test skip testing character set
+    CPPUNIT_ASSERT(t.skip(whitespace));
+    // check that skip was right
+    CPPUNIT_ASSERT(t.prefix(s,alpha));
+    CPPUNIT_ASSERT_EQUAL(SBuf("http"),s);
+
+    //check skip prefix
+    CPPUNIT_ASSERT(t.skip(SBuf("://")));
+    // verify
+    CPPUNIT_ASSERT(t.prefix(s,alpha));
+    CPPUNIT_ASSERT_EQUAL(SBuf("resource"),s);
+
+    // no skip
+    CPPUNIT_ASSERT(!t.skip(alpha));
+    CPPUNIT_ASSERT(!t.skip(SBuf("://")));
+    CPPUNIT_ASSERT(!t.skip('a'));
 
 }