Yet another attempt at controlling the parser stack size.

FossilOrigin-Name: cb19986dc6bc483df21e082e54a14cb6d7540b1734259e6d326d676908ac0172

diff --git a/doc/lemon.html b/doc/lemon.html
index 965f305c04169d0209cdab229df162f6fc651c0b..63efa950d8d32a7977bdcbdaee6640b2871c677b 100644 (file)
--- a/doc/lemon.html
+++ b/doc/lemon.html
@@ -696,6 +696,7 @@ other than that, the order of directives in Lemon is arbitrary.</p>
 <li><tt><a href='#pright'>%right</a></tt>
 <li><tt><a href='#reallc'>%realloc</a></tt>
 <li><tt><a href='#stack_overflow'>%stack_overflow</a></tt>
+<li><tt><a href='#reallc'>%stack_resizer</a></tt>
 <li><tt><a href='#stack_size'>%stack_size</a></tt>
 <li><tt><a href='#start_symbol'>%start_symbol</a></tt>
 <li><tt><a href='#syntax_error'>%syntax_error</a></tt>
@@ -1203,7 +1204,8 @@ the wildcard token and some other token, the other token is always used.
 The wildcard token is only matched if there are no alternatives.</p>
 
 <a id='reallc'></a>
-<h4>4.4.26 The <tt>%realloc</tt> and <tt>%free</tt> directives</h4>
+<h4>4.4.26 The <tt>%realloc</tt>, <tt>%free</tt>, and 
+<tt>%stack_resizer</tt> directives</h4>
 
 <p>The <tt>%realloc</tt> and <tt>%free</tt> directives defines function
 that allocate and free heap memory.  The signatures of these functions
@@ -1217,6 +1219,22 @@ parse stack space is exceeded.  The initial parser stack size
 is specified by either <tt>%stack_size</tt> or the
 -DYYSTACKDEPTH compile-time flag.
 
+<p>The <tt>%stack_resizer</tt> directive defines a function that computes
+the new size of the stack whenever it needs resizing.  The function takes
+two arguments:  The old stack size, and the %extra_context value.  The
+function returns an integer which is the new stack size.  If %stack_resizer
+is not defined, then the stack size roughly doubles with each new allocation.
+The %stack_resizer function is entirely optional.  The parser works fine
+without it.  The %stack_resizer function merely gives the application more
+control over the stack size, and offers an opportunity to raise warnings or
+errors if the parser stack grows too large.
+
+<p>If the %stack_resizer function returns an integer that is less than or
+equal to its first parameter, then %stack_overflow is invoked and parsing
+stops.
+
+<p>The %stack_resizer function only works if %extra_context is also defined.
+
 <a id='errors'></a>
 <h2>5.0 Error Processing</h2>
 

diff --git a/manifest b/manifest
index 94b2a3a9ee797a4ef4c57ec6a6723381fe938948..26ff83bd7c6cd96394f0f4db52fe58d44ddea9f5 100644 (file)
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\sincorrect\s"#line"\sgeneration\sin\sLemon.
-D 2025-11-18T10:38:41.356
+C Yet\sanother\sattempt\sat\scontrolling\sthe\sparser\sstack\ssize.
+D 2025-11-18T13:03:08.837
 F .fossil-settings/binary-glob 61195414528fb3ea9693577e1980230d78a1f8b0a54c78cf1b9b24d0a409ed6a x
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
@@ -60,7 +60,7 @@ F doc/compile-for-unix.md c9dce1ddd4bf0d25efccc5c63eb047e78c01ce06a6ff29c73e0a8a
 F doc/compile-for-windows.md f9e74d74da88f384edd5809f825035e071608f00f7f39c0e448df7b3982f979c
 F doc/json-enhancements.md e356fc834781f1f1aa22ee300027a270b2c960122468499bf347bb123ce1ea4f
 F doc/jsonb.md acd77fc3a709f51242655ad7803510c886aa8304202fa9cf2abc5f5c4e9d7ae5
-F doc/lemon.html 89ea833a6f71773ab1a9063fbb7fb9b32147bc0b1057b53ecab94a3b30c0aef5
+F doc/lemon.html 60b3836301ac8555622523569fc5cd3bae5b7a8cd1abab545b6ec14800c90aec
 F doc/pager-invariants.txt 83aa3a4724b2d7970cc3f3461f0295c46d4fc19a835a5781cbb35cb52feb0577
 F doc/tcl-extension-testing.md b88861804fc1eaf83249f8e206334189b61e150c360e1b80d0dcf91af82354f5
 F doc/testrunner.md 5ee928637e03f136a25fef852c5ed975932e31927bd9b05a574424ae18c31019
@@ -720,7 +720,7 @@ F src/os_win.c a89b501fc195085c7d6c9eec7f5bd782625e94bb2a96b000f4d009703df1083f
 F src/os_win.h 4c247cdb6d407c75186c94a1e84d5a22cbae4adcec93fcae8d2bc1f956fd1f19
 F src/pager.c a81461de271ac4886ad75b7ca2cca8157a48635820c4646cd2714acdc2c17e5f
 F src/pager.h 6137149346e6c8a3ddc1eeb40aee46381e9bc8b0fcc6dda8a1efde993c2275b8
-F src/parse.y 619c3e92a54686c5e47923688c4b9bf7ec534a4690db5677acc28b299c403250
+F src/parse.y f54dff6d07f337f9cea8cb3e28144e47877fd2f211fc3a5c5e238e456d51cc8a
 F src/pcache.c 588cc3c5ccaaadde689ed35ce5c5c891a1f7b1f4d1f56f6cf0143b74d8ee6484
 F src/pcache.h 1497ce1b823cf00094bb0cf3bac37b345937e6f910890c626b16512316d3abf5
 F src/pcache1.c 131ca0daf4e66b4608d2945ae76d6ed90de3f60539afbd5ef9ec65667a5f2fcd
@@ -2101,8 +2101,8 @@ F tool/genfkey.README e550911fa984c8255ebed2ef97824125d83806eb5232582700de949edf
 F tool/genfkey.test b6afd7b825d797a1e1274f519ab5695373552ecad5cd373530c63533638a5a4f
 F tool/getlock.c f4c39b651370156cae979501a7b156bdba50e7ce
 F tool/index_usage.c f62a0c701b2c7ff2f3e21d206f093c123f222dbf07136a10ffd1ca15a5c706c5
-F tool/lemon.c 861099d4d328251c85a35cfe3da81e00e35a0708be5aab0e1c7feb157b9735e8
-F tool/lempar.c bdffd3b233a4e4e78056c9c01fadd2bb3fe902435abde3bce3d769fdf0d5cca2
+F tool/lemon.c acb9b4e4d6a9826d40163b1da71e049859c7230217c39a6a63dc81ed8d28c906
+F tool/lempar.c ff3c9b32cf36152616dd2bb78ea0fdd3ec6c0f2e76ec55f0d2b50dcb35122b46
 F tool/libvers.c caafc3b689638a1d88d44bc5f526c2278760d9b9
 F tool/loadfts.c 63412f9790e5e8538fbde0b4f6db154aaaf80f7a10a01e3c94d14b773a8dd5a6
 F tool/logest.c c34e5944318415de513d29a6098df247a9618c96d83c38d4abd88641fe46e669
@@ -2166,8 +2166,11 @@ F tool/version-info.c 33d0390ef484b3b1cb685d59362be891ea162123cea181cb8e6d2cf6dd
 F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee87c1b31a7
 F tool/warnings.sh d924598cf2f55a4ecbc2aeb055c10bd5f48114793e7ba25f9585435da29e7e98
 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P ea48567ac54e4949a8b68977a58a5de7946e074ae8737133071d02f40ac97f34
-R 956e0d8d894da87712e963f1d74f3842
+P 5c0214df2c0a7470ac2edca0c483a3edd3c39ef0739688ab9a06e23882200360
+R 77442b17c4e08c97334e79ed87f9944e
+T *branch * parser-stack-size
+T *sym-parser-stack-size *
+T -sym-trunk *
 U drh
-Z 160fdfae770c78987ceedbc701266298
+Z df6b29f678f7d3bd6a12aec2b921e4cb
 # Remove this line to create a well-formed Fossil manifest.

diff --git a/manifest.tags b/manifest.tags
index bec971799ff1b8ee641c166c7aeb22d12c785393..e66df07e70151fbb082067b4ab2c4b52259768ac 100644 (file)
--- a/manifest.tags
+++ b/manifest.tags
@@ -1,2 +1,2 @@
-branch trunk
-tag trunk
+branch parser-stack-size
+tag parser-stack-size

diff --git a/manifest.uuid b/manifest.uuid
index 2a8ebfafeb37b385a2745b17c3eb7e08bb44d562..6e103a40ec6713fb36522856bc962a03df89981d 100644 (file)
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-5c0214df2c0a7470ac2edca0c483a3edd3c39ef0739688ab9a06e23882200360
+cb19986dc6bc483df21e082e54a14cb6d7540b1734259e6d326d676908ac0172

diff --git a/src/parse.y b/src/parse.y
index 617eb7303b6ee2f2293544a5cace5e4d95f908be..1dd5d700be3b898bedf8015a583035bd4fbb5a84 100644 (file)
--- a/src/parse.y
+++ b/src/parse.y
@@ -22,8 +22,9 @@
 }
 
 // Function used to enlarge the parser stack, if needed
-%realloc parserStackRealloc
-%free    sqlite3_free
+%stack_resizer parserStackResize
+%realloc       parserStackRealloc
+%free          sqlite3_free
 
 // All token codes are small integers with #defines that begin with "TK_"
 %token_prefix TK_
@@ -49,7 +50,7 @@
   }
 }
 %stack_overflow {
-  sqlite3OomFault(pParse->db);
+  if( pParse->nErr==0 ) sqlite3OomFault(pParse->db);
 }
 
 // The name of the generated procedure that implements the parser
@@ -586,6 +587,18 @@ cmd ::= select(X).  {
   static void *parserStackRealloc(void *pOld, sqlite3_uint64 newSize){
     return sqlite3FaultSim(700) ? 0 : sqlite3_realloc(pOld, newSize);
   }
+
+  /* Determine how big to make the stack */
+  static int parserStackResize(int oldSize, Parse *pParse){
+    int newSize;
+    int limit;
+    limit = pParse->db->aLimit[SQLITE_LIMIT_EXPR_DEPTH];
+    if( limit ) limit = limit*4 +100;
+    newSize = oldSize*2 + 100;
+    if( newSize>limit ) newSize = limit;
+    if( newSize<=oldSize ) sqlite3ErrorMsg(pParse, "Recursion limit");
+    return newSize;
+  }
 }
 
 %ifndef SQLITE_OMIT_CTE

diff --git a/tool/lemon.c b/tool/lemon.c
index 7dc705267310302192923b800542b488125d8914..b88da9d28766bc6e0d4cbf8ddbe795d047f2106a 100644 (file)
--- a/tool/lemon.c
+++ b/tool/lemon.c
@@ -494,6 +494,7 @@ struct lemon {
   char *filename;          /* Name of the input file */
   char *outname;           /* Name of the current output file */
   char *tokenprefix;       /* A prefix added to token names in the .h file */
+  char *resizeFunc;        /* Function to compute new stack size */
   char *reallocFunc;       /* Function to use to allocate stack space */
   char *freeFunc;          /* Function to use to free stack space */
   int nconflict;           /* Number of parsing conflicts */
@@ -2638,6 +2639,9 @@ static void parseonetoken(struct pstate *psp)
         }else if( strcmp(x,"default_type")==0 ){
           psp->declargslot = &(psp->gp->vartype);
           psp->insertLineMacro = 0;
+        }else if( strcmp(x,"stack_resizer")==0 ){
+          psp->declargslot = &(psp->gp->resizeFunc);
+          psp->insertLineMacro = 0;
         }else if( strcmp(x,"realloc")==0 ){
           psp->declargslot = &(psp->gp->reallocFunc);
           psp->insertLineMacro = 0;
@@ -4407,6 +4411,13 @@ static void writeRuleText(FILE *out, struct rule *rp){
   }
 }
 
+/*
+** Return true if the string is not NULL and not empty.
+*/
+static int notnull(const char *z){
+  return z && z[0];
+}
+
 
 /* Generate C source code for the parser */
 void ReportTable(
@@ -4611,25 +4622,33 @@ void ReportTable(
     fprintf(out,"#define %sARG_FETCH\n",name); lineno++;
     fprintf(out,"#define %sARG_STORE\n",name); lineno++;
   }
+  fprintf(out, "#undef YYREALLOC\n"); lineno++;
   if( lemp->reallocFunc ){
     fprintf(out,"#define YYREALLOC %s\n", lemp->reallocFunc); lineno++;
   }else{
     fprintf(out,"#define YYREALLOC realloc\n"); lineno++;
   }
+  fprintf(out, "#undef YYFREE\n"); lineno++;
   if( lemp->freeFunc ){
     fprintf(out,"#define YYFREE %s\n", lemp->freeFunc); lineno++;
   }else{
     fprintf(out,"#define YYFREE free\n"); lineno++;
   }
+  fprintf(out, "#undef YYDYNSTACK\n"); lineno++;
   if( lemp->reallocFunc && lemp->freeFunc ){
     fprintf(out,"#define YYDYNSTACK 1\n"); lineno++;
   }else{
     fprintf(out,"#define YYDYNSTACK 0\n"); lineno++;
   }
-  if( lemp->ctx && lemp->ctx[0] ){
+  fprintf(out, "#undef YYRESIZE\n"); lineno++;
+  if( notnull(lemp->ctx) ){
     i = lemonStrlen(lemp->ctx);
     while( i>=1 && ISSPACE(lemp->ctx[i-1]) ) i--;
     while( i>=1 && (ISALNUM(lemp->ctx[i-1]) || lemp->ctx[i-1]=='_') ) i--;
+    if( notnull(lemp->resizeFunc) ){
+      fprintf(out,"#define YYRESIZE %s\n", lemp->resizeFunc); lineno++;
+    }
+    fprintf(out,"#define %sCTX(P) ((P)->%s)\n",name,&lemp->ctx[i]); lineno++;
     fprintf(out,"#define %sCTX_SDECL %s;\n",name,lemp->ctx);  lineno++;
     fprintf(out,"#define %sCTX_PDECL ,%s\n",name,lemp->ctx);  lineno++;
     fprintf(out,"#define %sCTX_PARAM ,%s\n",name,&lemp->ctx[i]);  lineno++;
@@ -4647,10 +4666,13 @@ void ReportTable(
   if( mhflag ){
     fprintf(out,"#endif\n"); lineno++;
   }
+  fprintf(out, "#undef YYERRORSYMBOL\n"); lineno++;
+  fprintf(out, "#undef YYERRSYMDT\n"); lineno++;
   if( lemp->errsym && lemp->errsym->useCnt ){
     fprintf(out,"#define YYERRORSYMBOL %d\n",lemp->errsym->index); lineno++;
     fprintf(out,"#define YYERRSYMDT yy%d\n",lemp->errsym->dtnum); lineno++;
   }
+  fprintf(out,"#undef YYFALLBACK\n"); lineno++;
   if( lemp->has_fallback ){
     fprintf(out,"#define YYFALLBACK 1\n");  lineno++;
   }

diff --git a/tool/lempar.c b/tool/lempar.c
index 74314efeaa4c05c5dce82a21746265110f9e4ed0..1bffbf6067e949ea11b41bbf40b2103ddd89e42a 100644 (file)
--- a/tool/lempar.c
+++ b/tool/lempar.c
@@ -300,7 +300,12 @@ static int yyGrowStack(yyParser *p){
   int idx;
   yyStackEntry *pNew;
 
+#ifdef YYRESIZE
+  newSize = YYRESIZE(oldSize, ParseCTX(p));
+  if( newSize<=oldSize ) return 1;
+#else
   newSize = oldSize*2 + 100;
+#endif
   idx = (int)(p->yytos - p->yystack);
   if( p->yystack==p->yystk0 ){
     pNew = YYREALLOC(0, newSize*sizeof(pNew[0]));