Minor simplification to the tokenizer. Slightly smaller and faster.

FossilOrigin-Name: 9570b6b43df3bc5ce314cded20bca8be9e968efe

diff --git a/manifest b/manifest
index 4022e2709ba55bd64cdd4552a70f69e3cc2fd951..86aa0b58ec548605cdd1e9ed771f2434d6717df2 100644 (file)
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Improve\sthe\sLemon\sparser\stemplate\s(lempar.c)\sso\sthat\sit\savoids\sunnecessary\nwork\swhen\sthe\sgrammer\sdefines\sYYNOERRORRECOVERY\s(as\sSQLite\sdoes).\s\sSlightly\nsmaller\sand\sfaster\scode\sresults.
-D 2016-02-16T01:01:43.571
+C Minor\ssimplification\sto\sthe\stokenizer.\s\sSlightly\ssmaller\sand\sfaster.
+D 2016-02-16T13:04:19.100
 F Makefile.in 4e90dc1521879022aa9479268a4cd141d1771142
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 30f075dc4f27a07abb76088946b2944178d85347
@@ -406,7 +406,7 @@ F src/test_windirent.c 8f5fada630348558d5745b334702f301da1ffc61
 F src/test_windirent.h b12055cab6227f7be10f5c19296f67c60cc5e2a5
 F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9
 F src/threads.c 4ae07fa022a3dc7c5beb373cf744a85d3c5c6c3c
-F src/tokenize.c 2a234093ff16791ab3ec5ab60dc7fcf6af575e1c
+F src/tokenize.c 32aeca12f0d57a5c1c9a88d63e46ed2ee795cdb4
 F src/treeview.c dc39ccf04e9331237388b9cb73289c9d87ea050b
 F src/trigger.c e14840ee0c3e549e758ec9bf3e4146e166002280
 F src/update.c a7eeeaffad59c6506f01303a071dac11de8269ca
@@ -1427,7 +1427,7 @@ F tool/vdbe_profile.tcl 246d0da094856d72d2c12efec03250d71639d19f
 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
 F tool/warnings.sh a98af506df552f3b3c0d904f94e4cdc4e1a6d598
 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P e52c90758c53cbb0f74e0d8582e4c3c1bee04150
-R 14b39a221176c006cbe52f2c99fdaa8f
+P 9235b0cf6a37712ae9e5deeb1e5ee064dd5511fa
+R a61b67e5f930748de130d8b41c0bc53a
 U drh
-Z ff87b99d3af559ad671417b818afef7b
+Z d1a1b630582c40b1f2edd9cd2e01402a

diff --git a/manifest.uuid b/manifest.uuid
index 361d87de8f5ffa2c1a2070182db9b934564346ea..ee978cebbb745533defd8815859e0a1bc7cb477a 100644 (file)
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-9235b0cf6a37712ae9e5deeb1e5ee064dd5511fa
\ No newline at end of file
+9570b6b43df3bc5ce314cded20bca8be9e968efe
\ No newline at end of file

diff --git a/src/tokenize.c b/src/tokenize.c
index c43a039dde088525a3f24362abe0a877cfe1271d..4a5fc0a04ab7e6efab6d35a311fff3f841e31524 100644 (file)
--- a/src/tokenize.c
+++ b/src/tokenize.c
@@ -523,18 +523,17 @@ int sqlite3RunParser(Parse *pParse, const char *zSql, char **pzErrMsg){
         break;
       }
     }else{
-      if( tokenType==TK_SEMI ) pParse->zTail = &zSql[i];
       sqlite3Parser(pEngine, tokenType, pParse->sLastToken, pParse);
       lastTokenParsed = tokenType;
       if( pParse->rc!=SQLITE_OK || db->mallocFailed ) break;
     }
   }
   assert( nErr==0 );
+  pParse->zTail = &zSql[i];
   if( pParse->rc==SQLITE_OK && db->mallocFailed==0 ){
     assert( zSql[i]==0 );
     if( lastTokenParsed!=TK_SEMI ){
       sqlite3Parser(pEngine, TK_SEMI, pParse->sLastToken, pParse);
-      pParse->zTail = &zSql[i];
     }
     if( pParse->rc==SQLITE_OK && db->mallocFailed==0 ){
       sqlite3Parser(pEngine, 0, pParse->sLastToken, pParse);