%syntax_error
@@ -1203,7 +1204,8 @@ the wildcard token and some other token, the other token is always used.
The wildcard token is only matched if there are no alternatives.
-4.4.26 The %realloc and %free directives
+4.4.26 The %realloc, %free, and
+%stack_resizer directives
The %realloc and %free directives defines function
that allocate and free heap memory. The signatures of these functions
@@ -1217,6 +1219,22 @@ parse stack space is exceeded. The initial parser stack size
is specified by either %stack_size or the
-DYYSTACKDEPTH compile-time flag.
+
The %stack_resizer directive defines a function that computes
+the new size of the stack whenever it needs resizing. The function takes
+two arguments: The old stack size, and the %extra_context value. The
+function returns an integer which is the new stack size. If %stack_resizer
+is not defined, then the stack size roughly doubles with each new allocation.
+The %stack_resizer function is entirely optional. The parser works fine
+without it. The %stack_resizer function merely gives the application more
+control over the stack size, and offers an opportunity to raise warnings or
+errors if the parser stack grows too large.
+
+
If the %stack_resizer function returns an integer that is less than or
+equal to its first parameter, then %stack_overflow is invoked and parsing
+stops.
+
+
The %stack_resizer function only works if %extra_context is also defined.
+
5.0 Error Processing
diff --git a/manifest b/manifest
index 94b2a3a9ee..26ff83bd7c 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\sincorrect\s"#line"\sgeneration\sin\sLemon.
-D 2025-11-18T10:38:41.356
+C Yet\sanother\sattempt\sat\scontrolling\sthe\sparser\sstack\ssize.
+D 2025-11-18T13:03:08.837
F .fossil-settings/binary-glob 61195414528fb3ea9693577e1980230d78a1f8b0a54c78cf1b9b24d0a409ed6a x
F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
@@ -60,7 +60,7 @@ F doc/compile-for-unix.md c9dce1ddd4bf0d25efccc5c63eb047e78c01ce06a6ff29c73e0a8a
F doc/compile-for-windows.md f9e74d74da88f384edd5809f825035e071608f00f7f39c0e448df7b3982f979c
F doc/json-enhancements.md e356fc834781f1f1aa22ee300027a270b2c960122468499bf347bb123ce1ea4f
F doc/jsonb.md acd77fc3a709f51242655ad7803510c886aa8304202fa9cf2abc5f5c4e9d7ae5
-F doc/lemon.html 89ea833a6f71773ab1a9063fbb7fb9b32147bc0b1057b53ecab94a3b30c0aef5
+F doc/lemon.html 60b3836301ac8555622523569fc5cd3bae5b7a8cd1abab545b6ec14800c90aec
F doc/pager-invariants.txt 83aa3a4724b2d7970cc3f3461f0295c46d4fc19a835a5781cbb35cb52feb0577
F doc/tcl-extension-testing.md b88861804fc1eaf83249f8e206334189b61e150c360e1b80d0dcf91af82354f5
F doc/testrunner.md 5ee928637e03f136a25fef852c5ed975932e31927bd9b05a574424ae18c31019
@@ -720,7 +720,7 @@ F src/os_win.c a89b501fc195085c7d6c9eec7f5bd782625e94bb2a96b000f4d009703df1083f
F src/os_win.h 4c247cdb6d407c75186c94a1e84d5a22cbae4adcec93fcae8d2bc1f956fd1f19
F src/pager.c a81461de271ac4886ad75b7ca2cca8157a48635820c4646cd2714acdc2c17e5f
F src/pager.h 6137149346e6c8a3ddc1eeb40aee46381e9bc8b0fcc6dda8a1efde993c2275b8
-F src/parse.y 619c3e92a54686c5e47923688c4b9bf7ec534a4690db5677acc28b299c403250
+F src/parse.y f54dff6d07f337f9cea8cb3e28144e47877fd2f211fc3a5c5e238e456d51cc8a
F src/pcache.c 588cc3c5ccaaadde689ed35ce5c5c891a1f7b1f4d1f56f6cf0143b74d8ee6484
F src/pcache.h 1497ce1b823cf00094bb0cf3bac37b345937e6f910890c626b16512316d3abf5
F src/pcache1.c 131ca0daf4e66b4608d2945ae76d6ed90de3f60539afbd5ef9ec65667a5f2fcd
@@ -2101,8 +2101,8 @@ F tool/genfkey.README e550911fa984c8255ebed2ef97824125d83806eb5232582700de949edf
F tool/genfkey.test b6afd7b825d797a1e1274f519ab5695373552ecad5cd373530c63533638a5a4f
F tool/getlock.c f4c39b651370156cae979501a7b156bdba50e7ce
F tool/index_usage.c f62a0c701b2c7ff2f3e21d206f093c123f222dbf07136a10ffd1ca15a5c706c5
-F tool/lemon.c 861099d4d328251c85a35cfe3da81e00e35a0708be5aab0e1c7feb157b9735e8
-F tool/lempar.c bdffd3b233a4e4e78056c9c01fadd2bb3fe902435abde3bce3d769fdf0d5cca2
+F tool/lemon.c acb9b4e4d6a9826d40163b1da71e049859c7230217c39a6a63dc81ed8d28c906
+F tool/lempar.c ff3c9b32cf36152616dd2bb78ea0fdd3ec6c0f2e76ec55f0d2b50dcb35122b46
F tool/libvers.c caafc3b689638a1d88d44bc5f526c2278760d9b9
F tool/loadfts.c 63412f9790e5e8538fbde0b4f6db154aaaf80f7a10a01e3c94d14b773a8dd5a6
F tool/logest.c c34e5944318415de513d29a6098df247a9618c96d83c38d4abd88641fe46e669
@@ -2166,8 +2166,11 @@ F tool/version-info.c 33d0390ef484b3b1cb685d59362be891ea162123cea181cb8e6d2cf6dd
F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee87c1b31a7
F tool/warnings.sh d924598cf2f55a4ecbc2aeb055c10bd5f48114793e7ba25f9585435da29e7e98
F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P ea48567ac54e4949a8b68977a58a5de7946e074ae8737133071d02f40ac97f34
-R 956e0d8d894da87712e963f1d74f3842
+P 5c0214df2c0a7470ac2edca0c483a3edd3c39ef0739688ab9a06e23882200360
+R 77442b17c4e08c97334e79ed87f9944e
+T *branch * parser-stack-size
+T *sym-parser-stack-size *
+T -sym-trunk *
U drh
-Z 160fdfae770c78987ceedbc701266298
+Z df6b29f678f7d3bd6a12aec2b921e4cb
# Remove this line to create a well-formed Fossil manifest.
diff --git a/manifest.tags b/manifest.tags
index bec971799f..e66df07e70 100644
--- a/manifest.tags
+++ b/manifest.tags
@@ -1,2 +1,2 @@
-branch trunk
-tag trunk
+branch parser-stack-size
+tag parser-stack-size
diff --git a/manifest.uuid b/manifest.uuid
index 2a8ebfafeb..6e103a40ec 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-5c0214df2c0a7470ac2edca0c483a3edd3c39ef0739688ab9a06e23882200360
+cb19986dc6bc483df21e082e54a14cb6d7540b1734259e6d326d676908ac0172
diff --git a/src/parse.y b/src/parse.y
index 617eb7303b..1dd5d700be 100644
--- a/src/parse.y
+++ b/src/parse.y
@@ -22,8 +22,9 @@
}
// Function used to enlarge the parser stack, if needed
-%realloc parserStackRealloc
-%free sqlite3_free
+%stack_resizer parserStackResize
+%realloc parserStackRealloc
+%free sqlite3_free
// All token codes are small integers with #defines that begin with "TK_"
%token_prefix TK_
@@ -49,7 +50,7 @@
}
}
%stack_overflow {
- sqlite3OomFault(pParse->db);
+ if( pParse->nErr==0 ) sqlite3OomFault(pParse->db);
}
// The name of the generated procedure that implements the parser
@@ -586,6 +587,18 @@ cmd ::= select(X). {
static void *parserStackRealloc(void *pOld, sqlite3_uint64 newSize){
return sqlite3FaultSim(700) ? 0 : sqlite3_realloc(pOld, newSize);
}
+
+ /* Determine how big to make the stack */
+ static int parserStackResize(int oldSize, Parse *pParse){
+ int newSize;
+ int limit;
+ limit = pParse->db->aLimit[SQLITE_LIMIT_EXPR_DEPTH];
+ if( limit ) limit = limit*4 +100;
+ newSize = oldSize*2 + 100;
+ if( newSize>limit ) newSize = limit;
+ if( newSize<=oldSize ) sqlite3ErrorMsg(pParse, "Recursion limit");
+ return newSize;
+ }
}
%ifndef SQLITE_OMIT_CTE
diff --git a/tool/lemon.c b/tool/lemon.c
index 7dc7052673..b88da9d287 100644
--- a/tool/lemon.c
+++ b/tool/lemon.c
@@ -494,6 +494,7 @@ struct lemon {
char *filename; /* Name of the input file */
char *outname; /* Name of the current output file */
char *tokenprefix; /* A prefix added to token names in the .h file */
+ char *resizeFunc; /* Function to compute new stack size */
char *reallocFunc; /* Function to use to allocate stack space */
char *freeFunc; /* Function to use to free stack space */
int nconflict; /* Number of parsing conflicts */
@@ -2638,6 +2639,9 @@ static void parseonetoken(struct pstate *psp)
}else if( strcmp(x,"default_type")==0 ){
psp->declargslot = &(psp->gp->vartype);
psp->insertLineMacro = 0;
+ }else if( strcmp(x,"stack_resizer")==0 ){
+ psp->declargslot = &(psp->gp->resizeFunc);
+ psp->insertLineMacro = 0;
}else if( strcmp(x,"realloc")==0 ){
psp->declargslot = &(psp->gp->reallocFunc);
psp->insertLineMacro = 0;
@@ -4407,6 +4411,13 @@ static void writeRuleText(FILE *out, struct rule *rp){
}
}
+/*
+** Return true if the string is not NULL and not empty.
+*/
+static int notnull(const char *z){
+ return z && z[0];
+}
+
/* Generate C source code for the parser */
void ReportTable(
@@ -4611,25 +4622,33 @@ void ReportTable(
fprintf(out,"#define %sARG_FETCH\n",name); lineno++;
fprintf(out,"#define %sARG_STORE\n",name); lineno++;
}
+ fprintf(out, "#undef YYREALLOC\n"); lineno++;
if( lemp->reallocFunc ){
fprintf(out,"#define YYREALLOC %s\n", lemp->reallocFunc); lineno++;
}else{
fprintf(out,"#define YYREALLOC realloc\n"); lineno++;
}
+ fprintf(out, "#undef YYFREE\n"); lineno++;
if( lemp->freeFunc ){
fprintf(out,"#define YYFREE %s\n", lemp->freeFunc); lineno++;
}else{
fprintf(out,"#define YYFREE free\n"); lineno++;
}
+ fprintf(out, "#undef YYDYNSTACK\n"); lineno++;
if( lemp->reallocFunc && lemp->freeFunc ){
fprintf(out,"#define YYDYNSTACK 1\n"); lineno++;
}else{
fprintf(out,"#define YYDYNSTACK 0\n"); lineno++;
}
- if( lemp->ctx && lemp->ctx[0] ){
+ fprintf(out, "#undef YYRESIZE\n"); lineno++;
+ if( notnull(lemp->ctx) ){
i = lemonStrlen(lemp->ctx);
while( i>=1 && ISSPACE(lemp->ctx[i-1]) ) i--;
while( i>=1 && (ISALNUM(lemp->ctx[i-1]) || lemp->ctx[i-1]=='_') ) i--;
+ if( notnull(lemp->resizeFunc) ){
+ fprintf(out,"#define YYRESIZE %s\n", lemp->resizeFunc); lineno++;
+ }
+ fprintf(out,"#define %sCTX(P) ((P)->%s)\n",name,&lemp->ctx[i]); lineno++;
fprintf(out,"#define %sCTX_SDECL %s;\n",name,lemp->ctx); lineno++;
fprintf(out,"#define %sCTX_PDECL ,%s\n",name,lemp->ctx); lineno++;
fprintf(out,"#define %sCTX_PARAM ,%s\n",name,&lemp->ctx[i]); lineno++;
@@ -4647,10 +4666,13 @@ void ReportTable(
if( mhflag ){
fprintf(out,"#endif\n"); lineno++;
}
+ fprintf(out, "#undef YYERRORSYMBOL\n"); lineno++;
+ fprintf(out, "#undef YYERRSYMDT\n"); lineno++;
if( lemp->errsym && lemp->errsym->useCnt ){
fprintf(out,"#define YYERRORSYMBOL %d\n",lemp->errsym->index); lineno++;
fprintf(out,"#define YYERRSYMDT yy%d\n",lemp->errsym->dtnum); lineno++;
}
+ fprintf(out,"#undef YYFALLBACK\n"); lineno++;
if( lemp->has_fallback ){
fprintf(out,"#define YYFALLBACK 1\n"); lineno++;
}
diff --git a/tool/lempar.c b/tool/lempar.c
index 74314efeaa..1bffbf6067 100644
--- a/tool/lempar.c
+++ b/tool/lempar.c
@@ -300,7 +300,12 @@ static int yyGrowStack(yyParser *p){
int idx;
yyStackEntry *pNew;
+#ifdef YYRESIZE
+ newSize = YYRESIZE(oldSize, ParseCTX(p));
+ if( newSize<=oldSize ) return 1;
+#else
newSize = oldSize*2 + 100;
+#endif
idx = (int)(p->yytos - p->yystack);
if( p->yystack==p->yystk0 ){
pNew = YYREALLOC(0, newSize*sizeof(pNew[0]));