From: drh Date: Mon, 7 Sep 2015 19:52:55 +0000 (+0000) Subject: Change the parser engine so that it (once again) waits for a lookahead token X-Git-Tag: version-3.9.0~141^2~1 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=a248a722cf721cf294bd591ccb2e7398aeb6ecdf;p=thirdparty%2Fsqlite.git Change the parser engine so that it (once again) waits for a lookahead token before reducing, even in a SHIFTREDUCE action. FossilOrigin-Name: 2c17a1358353a0845b039283be79353f033e2491 --- diff --git a/manifest b/manifest index a6e735d796..c4c835e219 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C For\sthe\sLemon-generated\sparser,\sadd\sa\snew\saction\stype\sSHIFTREDUCE\sand\suse\sit\nto\sfurther\scompress\sthe\sparser\stables\sand\simprove\sparser\sperformance. -D 2015-09-07T18:23:37.162 +C Change\sthe\sparser\sengine\sso\sthat\sit\s(once\sagain)\swaits\sfor\sa\slookahead\stoken\nbefore\sreducing,\seven\sin\sa\sSHIFTREDUCE\saction. +D 2015-09-07T19:52:55.484 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in f85066ce844a28b671aaeeff320921cd0ce36239 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -302,7 +302,7 @@ F src/hwtime.h d32741c8f4df852c7d959236615444e2b1063b08 F src/insert.c 076dc5876e261a9908603d54cfc5344cd680166c F src/journal.c b4124532212b6952f42eb2c12fa3c25701d8ba8d F src/legacy.c ba1863ea58c4c840335a84ec276fc2b25e22bc4e -F src/lempar.c 406f2e85d2552d88bd4b994cc9c567c0fbc9f483 +F src/lempar.c d98d6c2229d5ed5e628a63819fc53ed5b292d4bd F src/loadext.c dfcee8c7c032cd0fd55af3e0fc1fcfb01e426df2 F src/main.c e17fcffae4306a9b8334faf3bac80d7396850b54 F src/malloc.c 021012e28a81ffdabf4c30ec3df6ce1f6cc93f1d @@ -328,7 +328,7 @@ F src/os_win.c 40b3af7a47eb1107d0d69e592bec345a3b7b798a F src/os_win.h eb7a47aa17b26b77eb97e4823f20a00b8bda12ca F src/pager.c 4784012f80b2197c61ff6eaf4f5c7026d93253fd F src/pager.h 6d435f563b3f7fcae4b84433b76a6ac2730036e2 -F src/parse.y 9e1777c68d5bb483d7217327e524dcfe5263b01c +F src/parse.y f599aa5e871a493330d567ced93de696f61f48f7 F src/pcache.c 24be750c79272e0ca7b6e007bc94999700f3e5ef F src/pcache.h 9968603796240cdf83da7e7bef76edf90619cea9 F src/pcache1.c bf2afe64a3dedb8643c8dcbd94a145cc80ab2a67 @@ -395,7 +395,7 @@ F src/test_vfs.c 3b65d42e18b262805716bd96178c81da8f2d9283 F src/test_vfstrace.c bab9594adc976cbe696ff3970728830b4c5ed698 F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9 F src/threads.c 6bbcc9fe50c917864d48287b4792d46d6e873481 -F src/tokenize.c 9655e20ade774e5c8e580634f1359288eba3fada +F src/tokenize.c 83c6ed569423a3af83a83973b444cf7123be33a6 F src/treeview.c 154f0acc622fa3514de8777dcedf4c8a8802b4ce F src/trigger.c 322f23aad694e8f31d384dcfa386d52a48d3c52f F src/update.c 3c5bc9570df3bfafa0db36828406a8a14e4c426e @@ -866,7 +866,7 @@ F test/minmax.test 42fbad0e81afaa6e0de41c960329f2b2c3526efd F test/minmax2.test b44bae787fc7b227597b01b0ca5575c7cb54d3bc F test/minmax3.test cc1e8b010136db0d01a6f2a29ba5a9f321034354 F test/minmax4.test 936941484ebdceb8adec7c86b6cd9b6e5e897c1f -F test/misc1.test 0be38a2944dd3e63dfb2899f24446339487d17c7 +F test/misc1.test 3f1c479c5a093a6280f378c0fbff1c2701486660 F test/misc2.test 00d7de54eda90e237fc9a38b9e5ccc769ebf6d4d F test/misc3.test cf3dda47d5dda3e53fc5804a100d3c82be736c9d F test/misc4.test 0d8be3466adf123a7791a66ba2bc8e8d229e87f3 @@ -1339,7 +1339,7 @@ F tool/genfkey.README cf68fddd4643bbe3ff8e31b8b6d8b0a1b85e20f4 F tool/genfkey.test 4196a8928b78f51d54ef58e99e99401ab2f0a7e5 F tool/getlock.c f4c39b651370156cae979501a7b156bdba50e7ce F tool/lemon.c 0c455691cc1e59a8f782d51a83dd6bbd7c5c44e7 -F tool/lempar.c 1522366692ef87584d4eacdbb1e95bdd0bc7f3b6 +F tool/lempar.c 9bec5f85673746e5ed92084e79931afae443a6be F tool/loadfts.c c3c64e4d5e90e8ba41159232c2189dba4be7b862 F tool/logest.c eef612f8adf4d0993dafed0416064cf50d5d33c6 F tool/mkautoconfamal.sh d1a2da0e15b2ed33d60af35c7e9d483f13a8eb9f @@ -1383,7 +1383,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P b6ffb7e471e51ff69668154ad2c8790e466c9d37 -R 90541ca0731ebb2fb0e53dfdeffae5a4 +P 531c3974b3d586c1989cde905b2fb4681239a570 +R 9764b1afa291a8128703e02e85f419ca U drh -Z 08f2caec8f48c0982df46628b104ff5c +Z 7cd6903444e24935012e1ca631512fff diff --git a/manifest.uuid b/manifest.uuid index 1eb95138eb..5557c822f6 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -531c3974b3d586c1989cde905b2fb4681239a570 \ No newline at end of file +2c17a1358353a0845b039283be79353f033e2491 \ No newline at end of file diff --git a/src/lempar.c b/src/lempar.c index d014ebcc75..f983ff752f 100644 --- a/src/lempar.c +++ b/src/lempar.c @@ -167,9 +167,13 @@ static const YYCODETYPE yyFallback[] = { ** + The semantic value stored at this level of the stack. This is ** the information used by the action routines in the grammar. ** It is sometimes called the "minor" token. +** +** After the "shift" half of a SHIFTREDUCE action, the stateno field +** actually contains the reduce action for the second half of the +** SHIFTREDUCE. */ struct yyStackEntry { - YYACTIONTYPE stateno; /* The state-number */ + YYACTIONTYPE stateno; /* The state-number, or reduce action in SHIFTREDUCE */ YYCODETYPE major; /* The major token value. This is the code ** number for the token at this stack level */ YYMINORTYPE minor; /* The user-supplied minor token value. This @@ -403,6 +407,7 @@ static int yy_find_shift_action( int i; int stateno = pParser->yystack[pParser->yyidx].stateno; + if( stateno>=YY_MIN_REDUCE ) return stateno; if( stateno>YY_SHIFT_COUNT || (i = yy_shift_ofst[stateno])==YY_SHIFT_USE_DFLT ){ return yy_default[stateno]; @@ -506,10 +511,32 @@ static void yyStackOverflow(yyParser *yypParser, YYMINORTYPE *yypMinor){ ParseARG_STORE; /* Suppress warning about unused %extra_argument var */ } +/* +** Print tracing information for a SHIFT action +*/ +#ifndef NDEBUG +static void yyTraceShift(yyParser *yypParser, int yyNewState){ + if( yyTraceFILE ){ + int i; + if( yyNewStateyyidx; i++) + fprintf(yyTraceFILE," %s",yyTokenName[yypParser->yystack[i].major]); + fprintf(yyTraceFILE,"\n"); + }else{ + fprintf(yyTraceFILE,"%sShift *\n",yyTracePrompt); + } + } +} +#else +# define yyTraceShift(X,Y) +#endif + /* ** Perform a shift action. Return the number of errors. */ -static int yy_shift( +static void yy_shift( yyParser *yypParser, /* The parser to be shifted */ int yyNewState, /* The new state to shift in */ int yyMajor, /* The major token to shift in */ @@ -525,14 +552,14 @@ static int yy_shift( #if YYSTACKDEPTH>0 if( yypParser->yyidx>=YYSTACKDEPTH ){ yyStackOverflow(yypParser, yypMinor); - return 1; + return; } #else if( yypParser->yyidx>=yypParser->yystksz ){ yyGrowStack(yypParser); if( yypParser->yyidx>=yypParser->yystksz ){ yyStackOverflow(yypParser, yypMinor); - return 1; + return; } } #endif @@ -540,21 +567,7 @@ static int yy_shift( yytos->stateno = (YYACTIONTYPE)yyNewState; yytos->major = (YYCODETYPE)yyMajor; yytos->minor = *yypMinor; -#ifndef NDEBUG - if( yyTraceFILE && yypParser->yyidx>0 ){ - int i; - if( yyNewStateyyidx; i++) - fprintf(yyTraceFILE," %s",yyTokenName[yypParser->yystack[i].major]); - fprintf(yyTraceFILE,"\n"); - }else{ - fprintf(yyTraceFILE,"%sShift *\n",yyTracePrompt); - } - } -#endif - return 0; + yyTraceShift(yypParser, yyNewState); } /* The following table contains information about every rule that @@ -628,6 +641,7 @@ static void yy_reduce( yypParser->yyidx -= yysize; yyact = yy_find_reduce_action(yymsp[-yysize].stateno,(YYCODETYPE)yygoto); if( yyact <= YY_MAX_SHIFTREDUCE ){ + if( yyact>YY_MAX_SHIFT ) yyact += YY_MIN_REDUCE - YY_MIN_SHIFTREDUCE; /* If the reduce action popped at least ** one element off the stack, then we can push the new element back ** onto the stack here, and skip the stack overflow test in yy_shift(). @@ -638,16 +652,9 @@ static void yy_reduce( yymsp->stateno = (YYACTIONTYPE)yyact; yymsp->major = (YYCODETYPE)yygoto; yymsp->minor = yygotominor; -#ifndef NDEBUG - if( yyTraceFILE ){ - fprintf(yyTraceFILE,"%sShift %d\n",yyTracePrompt,yyact); - } -#endif + yyTraceShift(yypParser, yyact); }else{ - if( yy_shift(yypParser,yyact,yygoto,&yygotominor) ) yyact = 0; - } - if( yyact>=YY_MIN_SHIFTREDUCE ){ - yy_reduce(yypParser, yyact - YY_MIN_SHIFTREDUCE); + yy_shift(yypParser,yyact,yygoto,&yygotominor); } }else{ assert( yyact == YY_ACCEPT_ACTION ); @@ -775,13 +782,10 @@ void Parse( do{ yyact = yy_find_shift_action(yypParser,(YYCODETYPE)yymajor); if( yyact <= YY_MAX_SHIFTREDUCE ){ - if( yy_shift(yypParser,yyact,yymajor,&yyminorunion)==0 ){ - yypParser->yyerrcnt--; - yymajor = YYNOCODE; - if( yyact > YY_MAX_SHIFT ){ - yy_reduce(yypParser, yyact-YY_MIN_SHIFTREDUCE); - } - } + if( yyact > YY_MAX_SHIFT ) yyact += YY_MIN_REDUCE - YY_MIN_SHIFTREDUCE; + yy_shift(yypParser,yyact,yymajor,&yyminorunion); + yypParser->yyerrcnt--; + yymajor = YYNOCODE; }else if( yyact <= YY_MAX_REDUCE ){ yy_reduce(yypParser,yyact-YY_MIN_REDUCE); }else{ @@ -883,7 +887,7 @@ void Parse( #endif } }while( yymajor!=YYNOCODE && yypParser->yyidx>=0 ); -#ifdef SQLITE_DEBUG +#ifndef NDEBUG if( yyTraceFILE ){ fprintf(yyTraceFILE,"%sReturn\n",yyTracePrompt); } diff --git a/src/parse.y b/src/parse.y index cdfef7bd3b..e99feeefc1 100644 --- a/src/parse.y +++ b/src/parse.y @@ -587,7 +587,7 @@ from(A) ::= FROM seltablist(X). { // stl_prefix(A) ::= seltablist(X) joinop(Y). { A = X; - if( A && A->nSrc>0 ) A->a[A->nSrc-1].fg.jointype = (u8)Y; + if( ALWAYS(A && A->nSrc>0) ) A->a[A->nSrc-1].fg.jointype = (u8)Y; } stl_prefix(A) ::= . {A = 0;} seltablist(A) ::= stl_prefix(X) nm(Y) dbnm(D) as(Z) indexed_opt(I) diff --git a/src/tokenize.c b/src/tokenize.c index 30a8ad06d0..6b5ad27901 100644 --- a/src/tokenize.c +++ b/src/tokenize.c @@ -459,12 +459,12 @@ abort_parse: assert( zSql[i]==0 ); if( lastTokenParsed!=TK_SEMI ){ sqlite3Parser(pEngine, TK_SEMI, pParse->sLastToken, pParse); + pParse->zTail = &zSql[i]; } if( pParse->rc==SQLITE_OK && db->mallocFailed==0 ){ sqlite3Parser(pEngine, 0, pParse->sLastToken, pParse); } } - pParse->zTail = &zSql[i]; #ifdef YYTRACKMAXSTACKDEPTH sqlite3_mutex_enter(sqlite3MallocMutex()); sqlite3StatusSet(SQLITE_STATUS_PARSER_STACK, diff --git a/test/misc1.test b/test/misc1.test index be4ab8646d..25e9bd813e 100644 --- a/test/misc1.test +++ b/test/misc1.test @@ -646,7 +646,7 @@ do_catchsql_test misc1-21.1 { } {1 {near "#0": syntax error}} do_catchsql_test misc1-21.2 { VALUES(0,0x0MATCH#0; -} {1 {near "#0": syntax error}} +} {1 {near ";": syntax error}} # 2015-04-15 do_execsql_test misc1-22.1 { diff --git a/tool/lempar.c b/tool/lempar.c index ddace228f6..161632a6c7 100644 --- a/tool/lempar.c +++ b/tool/lempar.c @@ -161,9 +161,13 @@ static const YYCODETYPE yyFallback[] = { ** + The semantic value stored at this level of the stack. This is ** the information used by the action routines in the grammar. ** It is sometimes called the "minor" token. +** +** After the "shift" half of a SHIFTREDUCE action, the stateno field +** actually contains the reduce action for the second half of the +** SHIFTREDUCE. */ struct yyStackEntry { - YYACTIONTYPE stateno; /* The state-number */ + YYACTIONTYPE stateno; /* The state-number, or reduce action in SHIFTREDUCE */ YYCODETYPE major; /* The major token value. This is the code ** number for the token at this stack level */ YYMINORTYPE minor; /* The user-supplied minor token value. This @@ -392,7 +396,8 @@ static int yy_find_shift_action( ){ int i; int stateno = pParser->yystack[pParser->yyidx].stateno; - + + if( stateno>=YY_MIN_REDUCE ) return stateno; if( stateno>YY_SHIFT_COUNT || (i = yy_shift_ofst[stateno])==YY_SHIFT_USE_DFLT ){ return yy_default[stateno]; @@ -496,10 +501,32 @@ static void yyStackOverflow(yyParser *yypParser, YYMINORTYPE *yypMinor){ ParseARG_STORE; /* Suppress warning about unused %extra_argument var */ } +/* +** Print tracing information for a SHIFT action +*/ +#ifndef NDEBUG +static void yyTraceShift(yyParser *yypParser, int yyNewState){ + if( yyTraceFILE ){ + int i; + if( yyNewStateyyidx; i++) + fprintf(yyTraceFILE," %s",yyTokenName[yypParser->yystack[i].major]); + fprintf(yyTraceFILE,"\n"); + }else{ + fprintf(yyTraceFILE,"%sShift *\n",yyTracePrompt); + } + } +} +#else +# define yyTraceShift(X,Y) +#endif + /* ** Perform a shift action. Return the number of errors. */ -static int yy_shift( +static void yy_shift( yyParser *yypParser, /* The parser to be shifted */ int yyNewState, /* The new state to shift in */ int yyMajor, /* The major token to shift in */ @@ -515,14 +542,14 @@ static int yy_shift( #if YYSTACKDEPTH>0 if( yypParser->yyidx>=YYSTACKDEPTH ){ yyStackOverflow(yypParser, yypMinor); - return 1; + return; } #else if( yypParser->yyidx>=yypParser->yystksz ){ yyGrowStack(yypParser); if( yypParser->yyidx>=yypParser->yystksz ){ yyStackOverflow(yypParser, yypMinor); - return 1; + return; } } #endif @@ -530,21 +557,7 @@ static int yy_shift( yytos->stateno = (YYACTIONTYPE)yyNewState; yytos->major = (YYCODETYPE)yyMajor; yytos->minor = *yypMinor; -#ifndef NDEBUG - if( yyTraceFILE && yypParser->yyidx>0 ){ - int i; - if( yyNewStateyyidx; i++) - fprintf(yyTraceFILE," %s",yyTokenName[yypParser->yystack[i].major]); - fprintf(yyTraceFILE,"\n"); - }else{ - fprintf(yyTraceFILE,"%sShift *\n",yyTracePrompt); - } - } -#endif - return 0; + yyTraceShift(yypParser, yyNewState); } /* The following table contains information about every rule that @@ -616,7 +629,8 @@ static void yy_reduce( yysize = yyRuleInfo[yyruleno].nrhs; yypParser->yyidx -= yysize; yyact = yy_find_reduce_action(yymsp[-yysize].stateno,(YYCODETYPE)yygoto); - if( yyact < YY_MAX_SHIFTREDUCE ){ + if( yyact <= YY_MAX_SHIFTREDUCE ){ + if( yyact>YY_MAX_SHIFT ) yyact += YY_MIN_REDUCE - YY_MIN_SHIFTREDUCE; /* If the reduce action popped at least ** one element off the stack, then we can push the new element back ** onto the stack here, and skip the stack overflow test in yy_shift(). @@ -627,16 +641,9 @@ static void yy_reduce( yymsp->stateno = (YYACTIONTYPE)yyact; yymsp->major = (YYCODETYPE)yygoto; yymsp->minor = yygotominor; -#ifndef NDEBUG - if( yyTraceFILE ){ - fprintf(yyTraceFILE,"%sShift %d\n",yyTracePrompt,yyact); - } -#endif + yyTraceShift(yypParser, yyact); }else{ - if( yy_shift(yypParser,yyact,yygoto,&yygotominor) ) yyact = 0; - } - if( yyact>=YY_MIN_SHIFTREDUCE ){ - yy_reduce(yypParser, yyact - YY_MIN_SHIFTREDUCE); + yy_shift(yypParser,yyact,yygoto,&yygotominor); } }else{ assert( yyact == YY_ACCEPT_ACTION ); @@ -760,13 +767,10 @@ void Parse( do{ yyact = yy_find_shift_action(yypParser,(YYCODETYPE)yymajor); if( yyact <= YY_MAX_SHIFTREDUCE ){ - if( yy_shift(yypParser,yyact,yymajor,&yyminorunion)==0 ){ - yypParser->yyerrcnt--; - yymajor = YYNOCODE; - if( yyact > YY_MAX_SHIFT ){ - yy_reduce(yypParser, yyact-YY_MIN_SHIFTREDUCE); - } - } + if( yyact > YY_MAX_SHIFT ) yyact += YY_MIN_REDUCE - YY_MIN_SHIFTREDUCE; + yy_shift(yypParser,yyact,yymajor,&yyminorunion); + yypParser->yyerrcnt--; + yymajor = YYNOCODE; }else if( yyact <= YY_MAX_REDUCE ){ yy_reduce(yypParser,yyact-YY_MIN_REDUCE); }else{