Add the "%token" control to the lemon parser. Not currently used by SQLite.

FossilOrigin-Name: a6e4c5ae8f29bc2e7f2088426341254e9281d19db9dc9a14abc376d56dad4c4b

diff --git a/manifest b/manifest
index 3d6434952dd714bb7047e2e3529110c3e933a46b..a6487d0ed2871165dc2b75bfd2dfbe3ac0977484 100644 (file)
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Faster\simplementation\sof\sresolveP2Values().
-D 2017-08-02T02:46:43.052
+C Add\sthe\s"%token"\scontrol\sto\sthe\slemon\sparser.\s\sNot\scurrently\sused\sby\sSQLite.
+D 2017-08-02T03:21:11.634
 F Makefile.in d9873c9925917cca9990ee24be17eb9613a668012c85a343aef7e5536ae266e8
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 02b469e9dcd5b7ee63fc1fb05babc174260ee4cfa4e0ef2e48c3c6801567a016
@@ -1564,7 +1564,7 @@ F tool/genfkey.README cf68fddd4643bbe3ff8e31b8b6d8b0a1b85e20f4
 F tool/genfkey.test 4196a8928b78f51d54ef58e99e99401ab2f0a7e5
 F tool/getlock.c f4c39b651370156cae979501a7b156bdba50e7ce
 F tool/kvtest-speed.sh 4761a9c4b3530907562314d7757995787f7aef8f
-F tool/lemon.c 5a04dff28578a67415cea5bf981b893c50cebfdd4388fb21254d1892525edfd8
+F tool/lemon.c e6056373044d55296d21f81467dba7632bbb81dc49af072b3f0e76338771497e
 F tool/lempar.c 10579a61dc2290182725e7abdefe311dd8b521a8f7f0aabbfc571e9012a09eaf
 F tool/libvers.c caafc3b689638a1d88d44bc5f526c2278760d9b9
 F tool/loadfts.c c3c64e4d5e90e8ba41159232c2189dba4be7b862
@@ -1640,7 +1640,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 436a89b91901851ce21bf0cb997291b48888c52788b904822083d8dfac32b84b
-R 8f124303feac23f1db170c2d495f87ca
+P 82e46fe0d497f871e652a579f25e77de7ef05d56484418961a1296f65f19415e
+R 0bde291cb23e7d07a8d60315e9f12ad7
 U drh
-Z b5679bcf5a11ae0ca338d3d7ebf72479
+Z d6f72abd78388d81bef7f991d66fd8f3

diff --git a/manifest.uuid b/manifest.uuid
index e1ab2986b10635116ced9ac5b8af6463912b3163..dffcb9b284bfe6b1a00ea3f7f31555036f018685 100644 (file)
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-82e46fe0d497f871e652a579f25e77de7ef05d56484418961a1296f65f19415e
\ No newline at end of file
+a6e4c5ae8f29bc2e7f2088426341254e9281d19db9dc9a14abc376d56dad4c4b
\ No newline at end of file

diff --git a/tool/lemon.c b/tool/lemon.c
index fe0fb4c6155a45fa58d848c482d2c6e70dc5411d..acc5450c993d9481a387416f24bd6d43e1126a8e 100644 (file)
--- a/tool/lemon.c
+++ b/tool/lemon.c
@@ -2155,7 +2155,8 @@ enum e_state {
   WAITING_FOR_FALLBACK_ID,
   WAITING_FOR_WILDCARD_ID,
   WAITING_FOR_CLASS_ID,
-  WAITING_FOR_CLASS_TOKEN
+  WAITING_FOR_CLASS_TOKEN,
+  WAITING_FOR_TOKEN_NAME
 };
 struct pstate {
   char *filename;       /* Name of the input file */
@@ -2470,6 +2471,8 @@ to follow the previous rule.");
         }else if( strcmp(x,"fallback")==0 ){
           psp->fallback = 0;
           psp->state = WAITING_FOR_FALLBACK_ID;
+        }else if( strcmp(x,"token")==0 ){
+          psp->state = WAITING_FOR_TOKEN_NAME;
         }else if( strcmp(x,"wildcard")==0 ){
           psp->state = WAITING_FOR_WILDCARD_ID;
         }else if( strcmp(x,"token_class")==0 ){
@@ -2624,6 +2627,26 @@ to follow the previous rule.");
         }
       }
       break;
+    case WAITING_FOR_TOKEN_NAME:
+      /* Tokens do not have to be declared before use.  But they can be
+      ** in order to control their assigned integer number.  The number for
+      ** each token is assigned when it is first seen.  So by including
+      **
+      **     %token ONE TWO THREE
+      **
+      ** early in the grammar file, that assigns small consecutive values
+      ** to each of the tokens ONE TWO and THREE.
+      */
+      if( x[0]=='.' ){
+        psp->state = WAITING_FOR_DECL_OR_RULE;
+      }else if( !ISUPPER(x[0]) ){
+        ErrorMsg(psp->filename, psp->tokenlineno,
+          "%%token argument \"%s\" should be a token", x);
+        psp->errorcnt++;
+      }else{
+        (void)Symbol_new(x);
+      }
+      break;
     case WAITING_FOR_WILDCARD_ID:
       if( x[0]=='.' ){
         psp->state = WAITING_FOR_DECL_OR_RULE;