From: drh Date: Thu, 24 Nov 2005 22:22:29 +0000 (+0000) Subject: Reduce the number of distinct token symbols in the parser so that the parser X-Git-Tag: version-3.6.10~3384 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=fa3b19ef3d871571de542186d8250d5eaaaa00f3;p=thirdparty%2Fsqlite.git Reduce the number of distinct token symbols in the parser so that the parser can store tokens in a single byte and thus be smaller in size. (CVS 2776) FossilOrigin-Name: 2dfc9863919c9eb1fd1064ab2817d752dd7f9293 --- diff --git a/Makefile.in b/Makefile.in index db47144225..e6dbc4e51e 100644 --- a/Makefile.in +++ b/Makefile.in @@ -339,9 +339,11 @@ parse.lo: parse.c $(HDR) parse.h: parse.c -parse.c: $(TOP)/src/parse.y lemon$(BEXE) +parse.c: $(TOP)/src/parse.y lemon$(BEXE) $(TOP)/addopcodes.awk cp $(TOP)/src/parse.y . ./lemon $(OPTS) parse.y + mv parse.h parse.h.temp + awk -f $(TOP)/addopcodes.awk parse.h.temp >parse.h pragma.lo: $(TOP)/src/pragma.c $(HDR) $(LTCOMPILE) -c $(TOP)/src/pragma.c diff --git a/addopcodes.awk b/addopcodes.awk new file mode 100644 index 0000000000..b806b1d96c --- /dev/null +++ b/addopcodes.awk @@ -0,0 +1,32 @@ +#!/usr/bin/awk +# +# This script appends additional token codes to the end of the +# parse.h file that lemon generates. These extra token codes are +# not used by the parser. But they are used by the tokenizer and/or +# the code generator. +# +# +BEGIN { + max = 0 +} +/^#define TK_/ { + print $0 + if( max<$3 ) max = $3 +} +END { + printf "#define TK_%-29s %4d\n", "TO_TEXT", max+1 + printf "#define TK_%-29s %4d\n", "TO_BLOB", max+2 + printf "#define TK_%-29s %4d\n", "TO_NUMERIC", max+3 + printf "#define TK_%-29s %4d\n", "TO_INT", max+4 + printf "#define TK_%-29s %4d\n", "TO_REAL", max+5 + printf "#define TK_%-29s %4d\n", "END_OF_FILE", max+6 + printf "#define TK_%-29s %4d\n", "ILLEGAL", max+7 + printf "#define TK_%-29s %4d\n", "SPACE", max+8 + printf "#define TK_%-29s %4d\n", "UNCLOSED_STRING", max+9 + printf "#define TK_%-29s %4d\n", "COMMENT", max+10 + printf "#define TK_%-29s %4d\n", "FUNCTION", max+11 + printf "#define TK_%-29s %4d\n", "COLUMN", max+12 + printf "#define TK_%-29s %4d\n", "AGG_FUNCTION", max+13 + printf "#define TK_%-29s %4d\n", "AGG_COLUMN", max+14 + printf "#define TK_%-29s %4d\n", "CONST_FUNC", max+15 +} diff --git a/main.mk b/main.mk index d8247f6899..ba90583fe3 100644 --- a/main.mk +++ b/main.mk @@ -273,9 +273,11 @@ parse.o: parse.c $(HDR) parse.h: parse.c -parse.c: $(TOP)/src/parse.y lemon +parse.c: $(TOP)/src/parse.y lemon $(TOP)/addopcodes.awk cp $(TOP)/src/parse.y . ./lemon $(OPTS) parse.y + mv parse.h parse.h.temp + awk -f $(TOP)/addopcodes.awk parse.h.temp >parse.h pragma.o: $(TOP)/src/pragma.c $(HDR) $(TCCX) $(TCL_FLAGS) -c $(TOP)/src/pragma.c diff --git a/manifest b/manifest index 3b9e2bbbe0..4ef45d21d7 100644 --- a/manifest +++ b/manifest @@ -1,10 +1,11 @@ -C Comment\schanges.\s\sNo\schanges\sto\scode.\s(CVS\s2775) -D 2005-11-24T14:34:36 -F Makefile.in 12784cdce5ffc8dfb707300c34e4f1eb3b8a14f1 +C Reduce\sthe\snumber\sof\sdistinct\stoken\ssymbols\sin\sthe\sparser\sso\sthat\sthe\sparser\ncan\sstore\stokens\sin\sa\ssingle\sbyte\sand\sthus\sbe\ssmaller\sin\ssize.\s(CVS\s2776) +D 2005-11-24T22:22:30 +F Makefile.in 38d98f08ec6d3643ec7b2afa1377f677da72c55e F Makefile.linux-gcc aee18d8a05546dcf1888bd4547e442008a49a092 F README 9c4e2d6706bdcc3efdd773ce752a8cdab4f90028 F VERSION b818cce180263e590a00ad4509a713892c3eecea F aclocal.m4 7daea4c35e88de30d5a3f6f7a2ab99720e803bbd +F addopcodes.awk 701697fae48376375ec8532c3d04e910cfeef352 F art/2005osaward.gif 0d1851b2a7c1c9d0ccce545f3e14bca42d7fd248 F art/SQLite.eps 9b43cc99cfd2be687d386faea6862ea68d6a72b2 F art/SQLite.gif 1bbb94484963f1382e27e1c5e86dd0c1061eba2b @@ -18,10 +19,10 @@ F doc/lemon.html f0f682f50210928c07e562621c3b7e8ab912a538 F doc/report1.txt a031aaf37b185e4fa540223cb516d3bccec7eeac F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 F ltmain.sh f6b283068efa69f06eb8aa1fe4bddfdbdeb35826 -F main.mk 72176d7299c2fd6ca2850623195a6984f75380a6 +F main.mk c2c2f377be833dff5ec17840b9329d561ceec280 F mkdll.sh 5ec23622515d5bf8969404e80cfb5e220ddf0512 F mkopcodec.awk bd46ad001c98dfbab07b1713cb8e692fa0e5415d -F mkopcodeh.awk c5ed62f101d408e64bcad0a8c036d85e99905480 +F mkopcodeh.awk 071dbba4eaf56c8d643baf4604a043af35683316 F mkso.sh 125868def279650a07d3f0f5e8476fecf99613fd F publish.sh ed0aba4ffdfadb36597d03ce8efdae96efc038cb F spec.template b2f6c4e488cbc3b993a57deba22cbc36203c4da3 @@ -59,7 +60,7 @@ F src/os_win.c fbccc85e7011174068c27d54256746321a1f0059 F src/os_win.h 41a946bea10f61c158ce8645e7646b29d44f122b F src/pager.c ca23cdff9e67a8e826e4796c60f06db6aab69b72 F src/pager.h e7b41ce8e7b5f629d456708b7ad9a8c8ede37140 -F src/parse.y 9399852d7b406f832e3fee879296142d4e677090 +F src/parse.y e4d57c2fd5cc02f19822ec41f6dc2bfc9bc85609 F src/pragma.c b40189967155a522433b8470f363192a927ba22c F src/prepare.c fc098db25d2a121affb08686cf04833fd50452d4 F src/printf.c 3ea3a17d25d7ac498efc18007c70371a42c968f8 @@ -85,7 +86,7 @@ F src/vdbe.c f90a0fff5dac2ae11f880fb54e78132640199c44 F src/vdbe.h 8729a4ee16ff9aeab2af9667df3cf300ff978e13 F src/vdbeInt.h 7824d7be3b659ad177c8f151d9612b45b1805878 F src/vdbeapi.c 85bbe1d0243a89655433d60711b4bd71979b59cd -F src/vdbeaux.c eb1ce3a40d37a1a7e92749e0ef72e3224fa5e55f +F src/vdbeaux.c eb5f7185a4a714b352c0b6dc804ad3180e03ec06 F src/vdbefifo.c 9efb94c8c3f4c979ebd0028219483f88e57584f5 F src/vdbemem.c cd9609c1e7f71ec76d9840c84c3a57ebfa6539cf F src/where.c 9c260db859047a44fe8219716ee5f0d2bd647420 @@ -320,7 +321,7 @@ F www/tclsqlite.tcl ddcf912ea48695603c8ed7efb29f0812ef8d1b49 F www/vdbe.tcl 87a31ace769f20d3627a64fa1fade7fed47b90d0 F www/version3.tcl a99cf5f6d8bd4d5537584a2b342f0fb9fa601d8b F www/whentouse.tcl 97e2b5cd296f7d8057e11f44427dea8a4c2db513 -P 0d8bd133a7530fb56ecb742472865f5a72a0ea5e -R 06e316110dea962e3b3f65ea961ede69 +P 786e23b2959c6103bfc0b45535afc49f22bc8a67 +R 1f185dbb096824806e8e102eaef1c213 U drh -Z cd4222744b819f869ca54e34e06e5d1f +Z 2cdab2f9b86e4120cd36a35ab0e49ed6 diff --git a/manifest.uuid b/manifest.uuid index 30b1448915..6fa401b672 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -786e23b2959c6103bfc0b45535afc49f22bc8a67 \ No newline at end of file +2dfc9863919c9eb1fd1064ab2817d752dd7f9293 \ No newline at end of file diff --git a/mkopcodeh.awk b/mkopcodeh.awk index 6984e25521..24d359b065 100644 --- a/mkopcodeh.awk +++ b/mkopcodeh.awk @@ -23,6 +23,20 @@ # code generator run (infinitesimally) faster and more importantly it makes # the total library smaller. # +# This script also scans for lines of the form: +# +# case OP_aaaa: /* no-push */ +# +# When the no-push comment is found on an opcode, it means that that +# opcode does not leave a result on the stack. But identifying which +# opcodes leave results on the stack it is possible to determine a +# much smaller upper bound on the size of the stack. This allows +# a smaller stack to be allocated, which is important to embedded +# systems with limited memory space. This script generates a series +# of "NOPUSH_MASK" defines that contain bitmaps of opcodes that leave +# results on the stack. The NOPUSH_MASK defines are used in vdbeaux.c +# to help determine the maximum stack size. +# # Remember the TK_ values from the parse.h file diff --git a/src/parse.y b/src/parse.y index fc57ce1fe5..1fb8989668 100644 --- a/src/parse.y +++ b/src/parse.y @@ -14,7 +14,7 @@ ** the parser. Lemon will also generate a header file containing ** numeric codes for all of the tokens. ** -** @(#) $Id: parse.y,v 1.184 2005/11/14 22:29:05 drh Exp $ +** @(#) $Id: parse.y,v 1.185 2005/11/24 22:22:30 drh Exp $ */ // All token codes are small integers with #defines that begin with "TK_" @@ -88,16 +88,6 @@ struct AttachKey { int type; Token key; }; } // end %include -// These are extra tokens used by the lexer but never seen by the -// parser. We put them in a rule so that the parser generator will -// add them to the parse.h output file. -// -%nonassoc END_OF_FILE ILLEGAL SPACE UNCLOSED_STRING COMMENT FUNCTION - COLUMN AGG_FUNCTION AGG_COLUMN CONST_FUNC. - -// Extra tokens used by the code generator by never seen by the parser. -%nonassoc TO_TEXT TO_BLOB TO_NUMERIC TO_INT TO_REAL. - // Input is a single SQL command input ::= cmdlist. cmdlist ::= cmdlist ecmd. diff --git a/src/vdbeaux.c b/src/vdbeaux.c index 26f829f07d..9aa22731fc 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -202,6 +202,7 @@ static int opcodeNoPush(u8 op){ NOPUSH_MASK_6 + (NOPUSH_MASK_7<<16), NOPUSH_MASK_8 + (NOPUSH_MASK_9<<16) }; + assert( op>=0 && op<32*5 ); return (masks[op>>5] & (1<<(op&0x1F))); }