Change the parser engine so that it (once again) waits for a lookahead token

before reducing, even in a SHIFTREDUCE action.

FossilOrigin-Name: 2c17a1358353a0845b039283be79353f033e2491

diff --git a/manifest b/manifest
index a6e735d796c609ac06cd69734d4cc571b3410a2d..c4c835e2199b8cb774ab43ba02d0001c71e1a073 100644 (file)
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C For\sthe\sLemon-generated\sparser,\sadd\sa\snew\saction\stype\sSHIFTREDUCE\sand\suse\sit\nto\sfurther\scompress\sthe\sparser\stables\sand\simprove\sparser\sperformance.
-D 2015-09-07T18:23:37.162
+C Change\sthe\sparser\sengine\sso\sthat\sit\s(once\sagain)\swaits\sfor\sa\slookahead\stoken\nbefore\sreducing,\seven\sin\sa\sSHIFTREDUCE\saction.
+D 2015-09-07T19:52:55.484
 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
 F Makefile.in f85066ce844a28b671aaeeff320921cd0ce36239
 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@@ -302,7 +302,7 @@ F src/hwtime.h d32741c8f4df852c7d959236615444e2b1063b08
 F src/insert.c 076dc5876e261a9908603d54cfc5344cd680166c
 F src/journal.c b4124532212b6952f42eb2c12fa3c25701d8ba8d
 F src/legacy.c ba1863ea58c4c840335a84ec276fc2b25e22bc4e
-F src/lempar.c 406f2e85d2552d88bd4b994cc9c567c0fbc9f483
+F src/lempar.c d98d6c2229d5ed5e628a63819fc53ed5b292d4bd
 F src/loadext.c dfcee8c7c032cd0fd55af3e0fc1fcfb01e426df2
 F src/main.c e17fcffae4306a9b8334faf3bac80d7396850b54
 F src/malloc.c 021012e28a81ffdabf4c30ec3df6ce1f6cc93f1d
@@ -328,7 +328,7 @@ F src/os_win.c 40b3af7a47eb1107d0d69e592bec345a3b7b798a
 F src/os_win.h eb7a47aa17b26b77eb97e4823f20a00b8bda12ca
 F src/pager.c 4784012f80b2197c61ff6eaf4f5c7026d93253fd
 F src/pager.h 6d435f563b3f7fcae4b84433b76a6ac2730036e2
-F src/parse.y 9e1777c68d5bb483d7217327e524dcfe5263b01c
+F src/parse.y f599aa5e871a493330d567ced93de696f61f48f7
 F src/pcache.c 24be750c79272e0ca7b6e007bc94999700f3e5ef
 F src/pcache.h 9968603796240cdf83da7e7bef76edf90619cea9
 F src/pcache1.c bf2afe64a3dedb8643c8dcbd94a145cc80ab2a67
@@ -395,7 +395,7 @@ F src/test_vfs.c 3b65d42e18b262805716bd96178c81da8f2d9283
 F src/test_vfstrace.c bab9594adc976cbe696ff3970728830b4c5ed698
 F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9
 F src/threads.c 6bbcc9fe50c917864d48287b4792d46d6e873481
-F src/tokenize.c 9655e20ade774e5c8e580634f1359288eba3fada
+F src/tokenize.c 83c6ed569423a3af83a83973b444cf7123be33a6
 F src/treeview.c 154f0acc622fa3514de8777dcedf4c8a8802b4ce
 F src/trigger.c 322f23aad694e8f31d384dcfa386d52a48d3c52f
 F src/update.c 3c5bc9570df3bfafa0db36828406a8a14e4c426e
@@ -866,7 +866,7 @@ F test/minmax.test 42fbad0e81afaa6e0de41c960329f2b2c3526efd
 F test/minmax2.test b44bae787fc7b227597b01b0ca5575c7cb54d3bc
 F test/minmax3.test cc1e8b010136db0d01a6f2a29ba5a9f321034354
 F test/minmax4.test 936941484ebdceb8adec7c86b6cd9b6e5e897c1f
-F test/misc1.test 0be38a2944dd3e63dfb2899f24446339487d17c7
+F test/misc1.test 3f1c479c5a093a6280f378c0fbff1c2701486660
 F test/misc2.test 00d7de54eda90e237fc9a38b9e5ccc769ebf6d4d
 F test/misc3.test cf3dda47d5dda3e53fc5804a100d3c82be736c9d
 F test/misc4.test 0d8be3466adf123a7791a66ba2bc8e8d229e87f3
@@ -1339,7 +1339,7 @@ F tool/genfkey.README cf68fddd4643bbe3ff8e31b8b6d8b0a1b85e20f4
 F tool/genfkey.test 4196a8928b78f51d54ef58e99e99401ab2f0a7e5
 F tool/getlock.c f4c39b651370156cae979501a7b156bdba50e7ce
 F tool/lemon.c 0c455691cc1e59a8f782d51a83dd6bbd7c5c44e7
-F tool/lempar.c 1522366692ef87584d4eacdbb1e95bdd0bc7f3b6
+F tool/lempar.c 9bec5f85673746e5ed92084e79931afae443a6be
 F tool/loadfts.c c3c64e4d5e90e8ba41159232c2189dba4be7b862
 F tool/logest.c eef612f8adf4d0993dafed0416064cf50d5d33c6
 F tool/mkautoconfamal.sh d1a2da0e15b2ed33d60af35c7e9d483f13a8eb9f
@@ -1383,7 +1383,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1
 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b
 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P b6ffb7e471e51ff69668154ad2c8790e466c9d37
-R 90541ca0731ebb2fb0e53dfdeffae5a4
+P 531c3974b3d586c1989cde905b2fb4681239a570
+R 9764b1afa291a8128703e02e85f419ca
 U drh
-Z 08f2caec8f48c0982df46628b104ff5c
+Z 7cd6903444e24935012e1ca631512fff

diff --git a/manifest.uuid b/manifest.uuid
index 1eb95138ebe5e44572f2055850ba367ed19e0d33..5557c822f609d8d7175a2d4d546b32ea1195ecd6 100644 (file)
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-531c3974b3d586c1989cde905b2fb4681239a570
\ No newline at end of file
+2c17a1358353a0845b039283be79353f033e2491
\ No newline at end of file

diff --git a/src/lempar.c b/src/lempar.c
index d014ebcc752df02be95f5ec852ca451a72b708ad..f983ff752fc71fd7d6714802373d77d0bd383cb7 100644 (file)
--- a/src/lempar.c
+++ b/src/lempar.c
@@ -167,9 +167,13 @@ static const YYCODETYPE yyFallback[] = {
 **   +  The semantic value stored at this level of the stack.  This is
 **      the information used by the action routines in the grammar.
 **      It is sometimes called the "minor" token.
+**
+** After the "shift" half of a SHIFTREDUCE action, the stateno field
+** actually contains the reduce action for the second half of the
+** SHIFTREDUCE.
 */
 struct yyStackEntry {
-  YYACTIONTYPE stateno;  /* The state-number */
+  YYACTIONTYPE stateno;  /* The state-number, or reduce action in SHIFTREDUCE */
   YYCODETYPE major;      /* The major token value.  This is the code
                          ** number for the token at this stack level */
   YYMINORTYPE minor;     /* The user-supplied minor token value.  This
@@ -403,6 +407,7 @@ static int yy_find_shift_action(
   int i;
   int stateno = pParser->yystack[pParser->yyidx].stateno;
  
+  if( stateno>=YY_MIN_REDUCE ) return stateno;
   if( stateno>YY_SHIFT_COUNT
    || (i = yy_shift_ofst[stateno])==YY_SHIFT_USE_DFLT ){
     return yy_default[stateno];
@@ -506,10 +511,32 @@ static void yyStackOverflow(yyParser *yypParser, YYMINORTYPE *yypMinor){
    ParseARG_STORE; /* Suppress warning about unused %extra_argument var */
 }
 
+/*
+** Print tracing information for a SHIFT action
+*/
+#ifndef NDEBUG
+static void yyTraceShift(yyParser *yypParser, int yyNewState){
+  if( yyTraceFILE ){
+    int i;
+    if( yyNewState<YYNSTATE ){
+      fprintf(yyTraceFILE,"%sShift %d\n",yyTracePrompt,yyNewState);
+      fprintf(yyTraceFILE,"%sStack:",yyTracePrompt);
+      for(i=1; i<=yypParser->yyidx; i++)
+        fprintf(yyTraceFILE," %s",yyTokenName[yypParser->yystack[i].major]);
+      fprintf(yyTraceFILE,"\n");
+    }else{
+      fprintf(yyTraceFILE,"%sShift *\n",yyTracePrompt);
+    }
+  }
+}
+#else
+# define yyTraceShift(X,Y)
+#endif
+
 /*
 ** Perform a shift action.  Return the number of errors.
 */
-static int yy_shift(
+static void yy_shift(
   yyParser *yypParser,          /* The parser to be shifted */
   int yyNewState,               /* The new state to shift in */
   int yyMajor,                  /* The major token to shift in */
@@ -525,14 +552,14 @@ static int yy_shift(
 #if YYSTACKDEPTH>0 
   if( yypParser->yyidx>=YYSTACKDEPTH ){
     yyStackOverflow(yypParser, yypMinor);
-    return 1;
+    return;
   }
 #else
   if( yypParser->yyidx>=yypParser->yystksz ){
     yyGrowStack(yypParser);
     if( yypParser->yyidx>=yypParser->yystksz ){
       yyStackOverflow(yypParser, yypMinor);
-      return 1;
+      return;
     }
   }
 #endif
@@ -540,21 +567,7 @@ static int yy_shift(
   yytos->stateno = (YYACTIONTYPE)yyNewState;
   yytos->major = (YYCODETYPE)yyMajor;
   yytos->minor = *yypMinor;
-#ifndef NDEBUG
-  if( yyTraceFILE && yypParser->yyidx>0 ){
-    int i;
-    if( yyNewState<YYNSTATE ){
-      fprintf(yyTraceFILE,"%sShift %d\n",yyTracePrompt,yyNewState);
-      fprintf(yyTraceFILE,"%sStack:",yyTracePrompt);
-      for(i=1; i<=yypParser->yyidx; i++)
-        fprintf(yyTraceFILE," %s",yyTokenName[yypParser->yystack[i].major]);
-      fprintf(yyTraceFILE,"\n");
-    }else{
-      fprintf(yyTraceFILE,"%sShift *\n",yyTracePrompt);
-    }
-  }
-#endif
-  return 0;
+  yyTraceShift(yypParser, yyNewState);
 }
 
 /* The following table contains information about every rule that
@@ -628,6 +641,7 @@ static void yy_reduce(
   yypParser->yyidx -= yysize;
   yyact = yy_find_reduce_action(yymsp[-yysize].stateno,(YYCODETYPE)yygoto);
   if( yyact <= YY_MAX_SHIFTREDUCE ){
+    if( yyact>YY_MAX_SHIFT ) yyact += YY_MIN_REDUCE - YY_MIN_SHIFTREDUCE;
     /* If the reduce action popped at least
     ** one element off the stack, then we can push the new element back
     ** onto the stack here, and skip the stack overflow test in yy_shift().
@@ -638,16 +652,9 @@ static void yy_reduce(
       yymsp->stateno = (YYACTIONTYPE)yyact;
       yymsp->major = (YYCODETYPE)yygoto;
       yymsp->minor = yygotominor;
-#ifndef NDEBUG
-      if( yyTraceFILE ){
-        fprintf(yyTraceFILE,"%sShift %d\n",yyTracePrompt,yyact);
-      }
-#endif
+      yyTraceShift(yypParser, yyact);
     }else{
-      if( yy_shift(yypParser,yyact,yygoto,&yygotominor) ) yyact = 0;
-    }
-    if( yyact>=YY_MIN_SHIFTREDUCE ){
-      yy_reduce(yypParser, yyact - YY_MIN_SHIFTREDUCE);
+      yy_shift(yypParser,yyact,yygoto,&yygotominor);
     }
   }else{
     assert( yyact == YY_ACCEPT_ACTION );
@@ -775,13 +782,10 @@ void Parse(
   do{
     yyact = yy_find_shift_action(yypParser,(YYCODETYPE)yymajor);
     if( yyact <= YY_MAX_SHIFTREDUCE ){
-      if( yy_shift(yypParser,yyact,yymajor,&yyminorunion)==0 ){
-        yypParser->yyerrcnt--;
-        yymajor = YYNOCODE;
-        if( yyact > YY_MAX_SHIFT ){
-          yy_reduce(yypParser, yyact-YY_MIN_SHIFTREDUCE);
-        }
-      }
+      if( yyact > YY_MAX_SHIFT ) yyact += YY_MIN_REDUCE - YY_MIN_SHIFTREDUCE;
+      yy_shift(yypParser,yyact,yymajor,&yyminorunion);
+      yypParser->yyerrcnt--;
+      yymajor = YYNOCODE;
     }else if( yyact <= YY_MAX_REDUCE ){
       yy_reduce(yypParser,yyact-YY_MIN_REDUCE);
     }else{
@@ -883,7 +887,7 @@ void Parse(
 #endif
     }
   }while( yymajor!=YYNOCODE && yypParser->yyidx>=0 );
-#ifdef SQLITE_DEBUG
+#ifndef NDEBUG
   if( yyTraceFILE ){
     fprintf(yyTraceFILE,"%sReturn\n",yyTracePrompt);
   }

diff --git a/src/parse.y b/src/parse.y
index cdfef7bd3b4d378d9feceade69fbe9104d65f371..e99feeefc148d4a0a4ae58f13dbcffc64508fc90 100644 (file)
--- a/src/parse.y
+++ b/src/parse.y
@@ -587,7 +587,7 @@ from(A) ::= FROM seltablist(X). {
 //
 stl_prefix(A) ::= seltablist(X) joinop(Y).    {
    A = X;
-   if( A && A->nSrc>0 ) A->a[A->nSrc-1].fg.jointype = (u8)Y;
+   if( ALWAYS(A && A->nSrc>0) ) A->a[A->nSrc-1].fg.jointype = (u8)Y;
 }
 stl_prefix(A) ::= .                           {A = 0;}
 seltablist(A) ::= stl_prefix(X) nm(Y) dbnm(D) as(Z) indexed_opt(I)

diff --git a/src/tokenize.c b/src/tokenize.c
index 30a8ad06d0129020f35a6bea9c621b83c48c0122..6b5ad279010c18f178c58d005cda54ff350966d3 100644 (file)
--- a/src/tokenize.c
+++ b/src/tokenize.c
@@ -459,12 +459,12 @@ abort_parse:
     assert( zSql[i]==0 );
     if( lastTokenParsed!=TK_SEMI ){
       sqlite3Parser(pEngine, TK_SEMI, pParse->sLastToken, pParse);
+      pParse->zTail = &zSql[i];
     }
     if( pParse->rc==SQLITE_OK && db->mallocFailed==0 ){
       sqlite3Parser(pEngine, 0, pParse->sLastToken, pParse);
     }
   }
-  pParse->zTail = &zSql[i];
 #ifdef YYTRACKMAXSTACKDEPTH
   sqlite3_mutex_enter(sqlite3MallocMutex());
   sqlite3StatusSet(SQLITE_STATUS_PARSER_STACK,

diff --git a/test/misc1.test b/test/misc1.test
index be4ab8646d36c27824c55a58a9486d4d7d6ce867..25e9bd813ec13271fc6e7c095a4ab9b27552174f 100644 (file)
--- a/test/misc1.test
+++ b/test/misc1.test
@@ -646,7 +646,7 @@ do_catchsql_test misc1-21.1 {
 } {1 {near "#0": syntax error}}
 do_catchsql_test misc1-21.2 {
   VALUES(0,0x0MATCH#0;
-} {1 {near "#0": syntax error}}
+} {1 {near ";": syntax error}}
 
 # 2015-04-15
 do_execsql_test misc1-22.1 {

diff --git a/tool/lempar.c b/tool/lempar.c
index ddace228f6b4afd85a9732574b45048b7cc24918..161632a6c7a414d901059f4a4792f53fbcc1376c 100644 (file)
--- a/tool/lempar.c
+++ b/tool/lempar.c
@@ -161,9 +161,13 @@ static const YYCODETYPE yyFallback[] = {
 **   +  The semantic value stored at this level of the stack.  This is
 **      the information used by the action routines in the grammar.
 **      It is sometimes called the "minor" token.
+**
+** After the "shift" half of a SHIFTREDUCE action, the stateno field
+** actually contains the reduce action for the second half of the
+** SHIFTREDUCE.
 */
 struct yyStackEntry {
-  YYACTIONTYPE stateno;  /* The state-number */
+  YYACTIONTYPE stateno;  /* The state-number, or reduce action in SHIFTREDUCE */
   YYCODETYPE major;      /* The major token value.  This is the code
                          ** number for the token at this stack level */
   YYMINORTYPE minor;     /* The user-supplied minor token value.  This
@@ -392,7 +396,8 @@ static int yy_find_shift_action(
 ){
   int i;
   int stateno = pParser->yystack[pParser->yyidx].stateno;
- 
+
+  if( stateno>=YY_MIN_REDUCE ) return stateno; 
   if( stateno>YY_SHIFT_COUNT
    || (i = yy_shift_ofst[stateno])==YY_SHIFT_USE_DFLT ){
     return yy_default[stateno];
@@ -496,10 +501,32 @@ static void yyStackOverflow(yyParser *yypParser, YYMINORTYPE *yypMinor){
    ParseARG_STORE; /* Suppress warning about unused %extra_argument var */
 }
 
+/*
+** Print tracing information for a SHIFT action
+*/
+#ifndef NDEBUG
+static void yyTraceShift(yyParser *yypParser, int yyNewState){
+  if( yyTraceFILE ){
+    int i;
+    if( yyNewState<YYNSTATE ){
+      fprintf(yyTraceFILE,"%sShift %d\n",yyTracePrompt,yyNewState);
+      fprintf(yyTraceFILE,"%sStack:",yyTracePrompt);
+      for(i=1; i<=yypParser->yyidx; i++)
+        fprintf(yyTraceFILE," %s",yyTokenName[yypParser->yystack[i].major]);
+      fprintf(yyTraceFILE,"\n");
+    }else{
+      fprintf(yyTraceFILE,"%sShift *\n",yyTracePrompt);
+    }
+  }
+}
+#else
+# define yyTraceShift(X,Y)
+#endif
+
 /*
 ** Perform a shift action.  Return the number of errors.
 */
-static int yy_shift(
+static void yy_shift(
   yyParser *yypParser,          /* The parser to be shifted */
   int yyNewState,               /* The new state to shift in */
   int yyMajor,                  /* The major token to shift in */
@@ -515,14 +542,14 @@ static int yy_shift(
 #if YYSTACKDEPTH>0 
   if( yypParser->yyidx>=YYSTACKDEPTH ){
     yyStackOverflow(yypParser, yypMinor);
-    return 1;
+    return;
   }
 #else
   if( yypParser->yyidx>=yypParser->yystksz ){
     yyGrowStack(yypParser);
     if( yypParser->yyidx>=yypParser->yystksz ){
       yyStackOverflow(yypParser, yypMinor);
-      return 1;
+      return;
     }
   }
 #endif
@@ -530,21 +557,7 @@ static int yy_shift(
   yytos->stateno = (YYACTIONTYPE)yyNewState;
   yytos->major = (YYCODETYPE)yyMajor;
   yytos->minor = *yypMinor;
-#ifndef NDEBUG
-  if( yyTraceFILE && yypParser->yyidx>0 ){
-    int i;
-    if( yyNewState<YYNSTATE ){
-      fprintf(yyTraceFILE,"%sShift %d\n",yyTracePrompt,yyNewState);
-      fprintf(yyTraceFILE,"%sStack:",yyTracePrompt);
-      for(i=1; i<=yypParser->yyidx; i++)
-        fprintf(yyTraceFILE," %s",yyTokenName[yypParser->yystack[i].major]);
-      fprintf(yyTraceFILE,"\n");
-    }else{
-      fprintf(yyTraceFILE,"%sShift *\n",yyTracePrompt);
-    }
-  }
-#endif
-  return 0;
+  yyTraceShift(yypParser, yyNewState);
 }
 
 /* The following table contains information about every rule that
@@ -616,7 +629,8 @@ static void yy_reduce(
   yysize = yyRuleInfo[yyruleno].nrhs;
   yypParser->yyidx -= yysize;
   yyact = yy_find_reduce_action(yymsp[-yysize].stateno,(YYCODETYPE)yygoto);
-  if( yyact < YY_MAX_SHIFTREDUCE ){
+  if( yyact <= YY_MAX_SHIFTREDUCE ){
+    if( yyact>YY_MAX_SHIFT ) yyact += YY_MIN_REDUCE - YY_MIN_SHIFTREDUCE;
     /* If the reduce action popped at least
     ** one element off the stack, then we can push the new element back
     ** onto the stack here, and skip the stack overflow test in yy_shift().
@@ -627,16 +641,9 @@ static void yy_reduce(
       yymsp->stateno = (YYACTIONTYPE)yyact;
       yymsp->major = (YYCODETYPE)yygoto;
       yymsp->minor = yygotominor;
-#ifndef NDEBUG
-      if( yyTraceFILE ){
-        fprintf(yyTraceFILE,"%sShift %d\n",yyTracePrompt,yyact);
-      }
-#endif
+      yyTraceShift(yypParser, yyact);
     }else{
-      if( yy_shift(yypParser,yyact,yygoto,&yygotominor) ) yyact = 0;
-    }
-    if( yyact>=YY_MIN_SHIFTREDUCE ){
-      yy_reduce(yypParser, yyact - YY_MIN_SHIFTREDUCE);
+      yy_shift(yypParser,yyact,yygoto,&yygotominor);
     }
   }else{
     assert( yyact == YY_ACCEPT_ACTION );
@@ -760,13 +767,10 @@ void Parse(
   do{
     yyact = yy_find_shift_action(yypParser,(YYCODETYPE)yymajor);
     if( yyact <= YY_MAX_SHIFTREDUCE ){
-      if( yy_shift(yypParser,yyact,yymajor,&yyminorunion)==0 ){
-        yypParser->yyerrcnt--;
-        yymajor = YYNOCODE;
-        if( yyact > YY_MAX_SHIFT ){
-          yy_reduce(yypParser, yyact-YY_MIN_SHIFTREDUCE);
-        }
-      }
+      if( yyact > YY_MAX_SHIFT ) yyact += YY_MIN_REDUCE - YY_MIN_SHIFTREDUCE;
+      yy_shift(yypParser,yyact,yymajor,&yyminorunion);
+      yypParser->yyerrcnt--;
+      yymajor = YYNOCODE;
     }else if( yyact <= YY_MAX_REDUCE ){
       yy_reduce(yypParser,yyact-YY_MIN_REDUCE);
     }else{