Reduce the number of distinct token symbols in the parser so that the parser

can store tokens in a single byte and thus be smaller in size. (CVS 2776)

FossilOrigin-Name: 2dfc9863919c9eb1fd1064ab2817d752dd7f9293

diff --git a/Makefile.in b/Makefile.in
index db4714422514109370e95d02a0382a80415bb37a..e6dbc4e51e6ebc61ae60e9d88c403f8a6a8a702b 100644 (file)
--- a/Makefile.in
+++ b/Makefile.in
@@ -339,9 +339,11 @@ parse.lo:  parse.c $(HDR)
 
 parse.h:       parse.c
 
-parse.c:       $(TOP)/src/parse.y lemon$(BEXE)
+parse.c:       $(TOP)/src/parse.y lemon$(BEXE) $(TOP)/addopcodes.awk
        cp $(TOP)/src/parse.y .
        ./lemon $(OPTS) parse.y
+       mv parse.h parse.h.temp
+       awk -f $(TOP)/addopcodes.awk parse.h.temp >parse.h
 
 pragma.lo:     $(TOP)/src/pragma.c $(HDR)
        $(LTCOMPILE) -c $(TOP)/src/pragma.c

diff --git a/addopcodes.awk b/addopcodes.awk
new file mode 100644 (file)
index 0000000..b806b1d
--- /dev/null
+++ b/addopcodes.awk
@@ -0,0 +1,32 @@
+#!/usr/bin/awk
+#
+# This script appends additional token codes to the end of the
+# parse.h file that lemon generates.  These extra token codes are
+# not used by the parser.  But they are used by the tokenizer and/or
+# the code generator.
+#
+#
+BEGIN {
+  max = 0
+}
+/^#define TK_/ {
+  print $0
+  if( max<$3 ) max = $3
+}
+END {
+  printf "#define TK_%-29s %4d\n", "TO_TEXT",         max+1
+  printf "#define TK_%-29s %4d\n", "TO_BLOB",         max+2
+  printf "#define TK_%-29s %4d\n", "TO_NUMERIC",      max+3
+  printf "#define TK_%-29s %4d\n", "TO_INT",          max+4
+  printf "#define TK_%-29s %4d\n", "TO_REAL",         max+5
+  printf "#define TK_%-29s %4d\n", "END_OF_FILE",     max+6
+  printf "#define TK_%-29s %4d\n", "ILLEGAL",         max+7
+  printf "#define TK_%-29s %4d\n", "SPACE",           max+8
+  printf "#define TK_%-29s %4d\n", "UNCLOSED_STRING", max+9
+  printf "#define TK_%-29s %4d\n", "COMMENT",         max+10
+  printf "#define TK_%-29s %4d\n", "FUNCTION",        max+11
+  printf "#define TK_%-29s %4d\n", "COLUMN",          max+12
+  printf "#define TK_%-29s %4d\n", "AGG_FUNCTION",    max+13
+  printf "#define TK_%-29s %4d\n", "AGG_COLUMN",      max+14
+  printf "#define TK_%-29s %4d\n", "CONST_FUNC",      max+15
+}

diff --git a/main.mk b/main.mk
index d8247f6899279f7ab0d2392702d82fcdd0e7a05f..ba90583fe36254f0ddfab8091934166dc60bdd65 100644 (file)
--- a/main.mk
+++ b/main.mk
@@ -273,9 +273,11 @@ parse.o:   parse.c $(HDR)
 
 parse.h:       parse.c
 
-parse.c:       $(TOP)/src/parse.y lemon
+parse.c:       $(TOP)/src/parse.y lemon $(TOP)/addopcodes.awk
        cp $(TOP)/src/parse.y .
        ./lemon $(OPTS) parse.y
+       mv parse.h parse.h.temp
+       awk -f $(TOP)/addopcodes.awk parse.h.temp >parse.h
 
 pragma.o:      $(TOP)/src/pragma.c $(HDR)
        $(TCCX) $(TCL_FLAGS) -c $(TOP)/src/pragma.c

diff --git a/manifest b/manifest
index 3b9e2bbbe069069f5ba56fca4da8cf6a97e2f678..4ef45d21d7fd15e95aa77139574634ca4c250b97 100644 (file)
--- a/manifest
+++ b/manifest
@@ -1,10 +1,11 @@
-C Comment\schanges.\s\sNo\schanges\sto\scode.\s(CVS\s2775)
-D 2005-11-24T14:34:36
-F Makefile.in 12784cdce5ffc8dfb707300c34e4f1eb3b8a14f1
+C Reduce\sthe\snumber\sof\sdistinct\stoken\ssymbols\sin\sthe\sparser\sso\sthat\sthe\sparser\ncan\sstore\stokens\sin\sa\ssingle\sbyte\sand\sthus\sbe\ssmaller\sin\ssize.\s(CVS\s2776)
+D 2005-11-24T22:22:30
+F Makefile.in 38d98f08ec6d3643ec7b2afa1377f677da72c55e
 F Makefile.linux-gcc aee18d8a05546dcf1888bd4547e442008a49a092
 F README 9c4e2d6706bdcc3efdd773ce752a8cdab4f90028
 F VERSION b818cce180263e590a00ad4509a713892c3eecea
 F aclocal.m4 7daea4c35e88de30d5a3f6f7a2ab99720e803bbd
+F addopcodes.awk 701697fae48376375ec8532c3d04e910cfeef352
 F art/2005osaward.gif 0d1851b2a7c1c9d0ccce545f3e14bca42d7fd248
 F art/SQLite.eps 9b43cc99cfd2be687d386faea6862ea68d6a72b2
 F art/SQLite.gif 1bbb94484963f1382e27e1c5e86dd0c1061eba2b
@@ -18,10 +19,10 @@ F doc/lemon.html f0f682f50210928c07e562621c3b7e8ab912a538
 F doc/report1.txt a031aaf37b185e4fa540223cb516d3bccec7eeac
 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895
 F ltmain.sh f6b283068efa69f06eb8aa1fe4bddfdbdeb35826
-F main.mk 72176d7299c2fd6ca2850623195a6984f75380a6
+F main.mk c2c2f377be833dff5ec17840b9329d561ceec280
 F mkdll.sh 5ec23622515d5bf8969404e80cfb5e220ddf0512
 F mkopcodec.awk bd46ad001c98dfbab07b1713cb8e692fa0e5415d
-F mkopcodeh.awk c5ed62f101d408e64bcad0a8c036d85e99905480
+F mkopcodeh.awk 071dbba4eaf56c8d643baf4604a043af35683316
 F mkso.sh 125868def279650a07d3f0f5e8476fecf99613fd
 F publish.sh ed0aba4ffdfadb36597d03ce8efdae96efc038cb
 F spec.template b2f6c4e488cbc3b993a57deba22cbc36203c4da3
@@ -59,7 +60,7 @@ F src/os_win.c fbccc85e7011174068c27d54256746321a1f0059
 F src/os_win.h 41a946bea10f61c158ce8645e7646b29d44f122b
 F src/pager.c ca23cdff9e67a8e826e4796c60f06db6aab69b72
 F src/pager.h e7b41ce8e7b5f629d456708b7ad9a8c8ede37140
-F src/parse.y 9399852d7b406f832e3fee879296142d4e677090
+F src/parse.y e4d57c2fd5cc02f19822ec41f6dc2bfc9bc85609
 F src/pragma.c b40189967155a522433b8470f363192a927ba22c
 F src/prepare.c fc098db25d2a121affb08686cf04833fd50452d4
 F src/printf.c 3ea3a17d25d7ac498efc18007c70371a42c968f8
@@ -85,7 +86,7 @@ F src/vdbe.c f90a0fff5dac2ae11f880fb54e78132640199c44
 F src/vdbe.h 8729a4ee16ff9aeab2af9667df3cf300ff978e13
 F src/vdbeInt.h 7824d7be3b659ad177c8f151d9612b45b1805878
 F src/vdbeapi.c 85bbe1d0243a89655433d60711b4bd71979b59cd
-F src/vdbeaux.c eb1ce3a40d37a1a7e92749e0ef72e3224fa5e55f
+F src/vdbeaux.c eb5f7185a4a714b352c0b6dc804ad3180e03ec06
 F src/vdbefifo.c 9efb94c8c3f4c979ebd0028219483f88e57584f5
 F src/vdbemem.c cd9609c1e7f71ec76d9840c84c3a57ebfa6539cf
 F src/where.c 9c260db859047a44fe8219716ee5f0d2bd647420
@@ -320,7 +321,7 @@ F www/tclsqlite.tcl ddcf912ea48695603c8ed7efb29f0812ef8d1b49
 F www/vdbe.tcl 87a31ace769f20d3627a64fa1fade7fed47b90d0
 F www/version3.tcl a99cf5f6d8bd4d5537584a2b342f0fb9fa601d8b
 F www/whentouse.tcl 97e2b5cd296f7d8057e11f44427dea8a4c2db513
-P 0d8bd133a7530fb56ecb742472865f5a72a0ea5e
-R 06e316110dea962e3b3f65ea961ede69
+P 786e23b2959c6103bfc0b45535afc49f22bc8a67
+R 1f185dbb096824806e8e102eaef1c213
 U drh
-Z cd4222744b819f869ca54e34e06e5d1f
+Z 2cdab2f9b86e4120cd36a35ab0e49ed6

diff --git a/manifest.uuid b/manifest.uuid
index 30b14489153a37fcef439c913f880a13abca244a..6fa401b67288ea324e0f13f1a97835acb9790a02 100644 (file)
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-786e23b2959c6103bfc0b45535afc49f22bc8a67
\ No newline at end of file
+2dfc9863919c9eb1fd1064ab2817d752dd7f9293
\ No newline at end of file

diff --git a/mkopcodeh.awk b/mkopcodeh.awk
index 6984e25521da775cbc3fbf17074732d2ab8f2a55..24d359b0651b863fede40fbab9508b988ec7251d 100644 (file)
--- a/mkopcodeh.awk
+++ b/mkopcodeh.awk
@@ -23,6 +23,20 @@
 # code generator run (infinitesimally) faster and more importantly it makes
 # the total library smaller.
 #
+# This script also scans for lines of the form:
+#
+#       case OP_aaaa:       /* no-push */
+#
+# When the no-push comment is found on an opcode, it means that that
+# opcode does not leave a result on the stack.  But identifying which
+# opcodes leave results on the stack it is possible to determine a
+# much smaller upper bound on the size of the stack.  This allows
+# a smaller stack to be allocated, which is important to embedded
+# systems with limited memory space.  This script generates a series
+# of "NOPUSH_MASK" defines that contain bitmaps of opcodes that leave
+# results on the stack.  The NOPUSH_MASK defines are used in vdbeaux.c
+# to help determine the maximum stack size.
+#
 
 
 # Remember the TK_ values from the parse.h file

diff --git a/src/parse.y b/src/parse.y
index fc57ce1fe5df2243006a0b6c93b1f457e0ba3384..1fb8989668da7bddee2f35f452f5e3221510354c 100644 (file)
--- a/src/parse.y
+++ b/src/parse.y
@@ -14,7 +14,7 @@
 ** the parser.  Lemon will also generate a header file containing
 ** numeric codes for all of the tokens.
 **
-** @(#) $Id: parse.y,v 1.184 2005/11/14 22:29:05 drh Exp $
+** @(#) $Id: parse.y,v 1.185 2005/11/24 22:22:30 drh Exp $
 */
 
 // All token codes are small integers with #defines that begin with "TK_"
@@ -88,16 +88,6 @@ struct AttachKey { int type;  Token key; };
 
 } // end %include
 
-// These are extra tokens used by the lexer but never seen by the
-// parser.  We put them in a rule so that the parser generator will
-// add them to the parse.h output file.
-//
-%nonassoc END_OF_FILE ILLEGAL SPACE UNCLOSED_STRING COMMENT FUNCTION
-          COLUMN AGG_FUNCTION AGG_COLUMN CONST_FUNC.
-
-// Extra tokens used by the code generator by never seen by the parser.
-%nonassoc TO_TEXT TO_BLOB TO_NUMERIC TO_INT TO_REAL.
-
 // Input is a single SQL command
 input ::= cmdlist.
 cmdlist ::= cmdlist ecmd.

diff --git a/src/vdbeaux.c b/src/vdbeaux.c
index 26f829f07dd036e8d7de6612a4207246d9c4c7ac..9aa22731fc8faaad05f3de8afc175ef2fd09c1b1 100644 (file)
--- a/src/vdbeaux.c
+++ b/src/vdbeaux.c
@@ -202,6 +202,7 @@ static int opcodeNoPush(u8 op){
     NOPUSH_MASK_6 + (NOPUSH_MASK_7<<16),
     NOPUSH_MASK_8 + (NOPUSH_MASK_9<<16)
   };
+  assert( op>=0 && op<32*5 );
   return (masks[op>>5] & (1<<(op&0x1F)));
 }