From 8bee11a41ef9a90f8c9046db725dcf962700127c Mon Sep 17 00:00:00 2001
From: mistachkin <mistachkin@noemail.net>
Date: Mon, 29 Oct 2018 17:53:23 +0000
Subject: [PATCH] Add the sqlite3_normalized_sql() API.

FossilOrigin-Name: 592b66e8058dd03a056a036e2606247c9efdb06d15eebe9bcc455f7f55e30ae6
---
 Makefile.in         |  40 +++---
 Makefile.msc        |   4 +
 main.mk             |  34 +++---
 manifest            |  55 +++++----
 manifest.uuid       |   2 +-
 src/build.c         |   6 +
 src/callback.c      |  19 ++-
 src/ctime.c         |   3 +
 src/expr.c          |   8 ++
 src/hash.c          |  56 +++++++++
 src/loadext.c       |   8 +-
 src/prepare.c       | 288 +++++++++++++++++++++++++++++++++++++++++++
 src/sqlite.h.in     |  21 +++-
 src/sqlite3ext.h    |   5 +
 src/sqliteInt.h     |  23 ++++
 src/test1.c         | 100 ++++++++++++++-
 src/test_config.c   |   6 +
 src/tokenize.c      |  67 ++++++++++
 src/vdbeInt.h       |   3 +
 src/vdbeapi.c       |  10 ++
 src/vdbeaux.c       |  15 +++
 test/normalize.test | 289 ++++++++++++++++++++++++++++++++++++++++++++
 22 files changed, 996 insertions(+), 66 deletions(-)

diff --git a/Makefile.in b/Makefile.in
index 3d44a96b31..2cd8471bb5 100644
--- a/Makefile.in
+++ b/Makefile.in
@@ -22,7 +22,7 @@ TOP = @abs_srcdir@
 #
 BCC = @BUILD_CC@ @BUILD_CFLAGS@
 
-# TCC is the C Compile and options for use in building executables that 
+# TCC is the C Compile and options for use in building executables that
 # will run on the target platform.  (BCC and TCC are usually the
 # same unless your are cross-compiling.)  Separate CC and CFLAGS macros
 # are provide so that these aspects of the build process can be changed
@@ -35,7 +35,7 @@ TCC += -I${TOP}/ext/fts3 -I${TOP}/ext/async -I${TOP}/ext/session
 
 # Define this for the autoconf-based build, so that the code knows it can
 # include the generated config.h
-# 
+#
 TCC += -D_HAVE_SQLITE_CONFIG_H -DBUILD_sqlite
 
 # Define -DNDEBUG to compile without debugging (i.e., for production usage)
@@ -66,7 +66,7 @@ LIBREADLINE = @TARGET_READLINE_LIBS@
 TCC += -DSQLITE_THREADSAFE=@SQLITE_THREADSAFE@
 
 # Any target libraries which libsqlite must be linked against
-# 
+#
 TLIBS = @LIBS@ $(LIBS)
 
 # Flags controlling use of the in memory btree implementation
@@ -78,8 +78,8 @@ TLIBS = @LIBS@ $(LIBS)
 TEMP_STORE = -DSQLITE_TEMP_STORE=@TEMP_STORE@
 
 # Enable/disable loadable extensions, and other optional features
-# based on configuration. (-DSQLITE_OMIT*, -DSQLITE_ENABLE*).  
-# The same set of OMIT and ENABLE flags should be passed to the 
+# based on configuration. (-DSQLITE_OMIT*, -DSQLITE_ENABLE*).
+# The same set of OMIT and ENABLE flags should be passed to the
 # LEMON parser generator and the mkkeywordhash tool as well.
 OPT_FEATURE_FLAGS = @OPT_FEATURE_FLAGS@
 
@@ -126,8 +126,8 @@ SHLIB_SUFFIX = @TCL_SHLIB_SUFFIX@
 # If gcov support was enabled by the configure script, add the appropriate
 # flags here.  It's not always as easy as just having the user add the right
 # CFLAGS / LDFLAGS, because libtool wants to use CFLAGS when linking, which
-# causes build errors with -fprofile-arcs -ftest-coverage with some GCCs.  
-# Supposedly GCC does the right thing if you use --coverage, but in 
+# causes build errors with -fprofile-arcs -ftest-coverage with some GCCs.
+# Supposedly GCC does the right thing if you use --coverage, but in
 # practice it still fails.  See:
 #
 # http://www.mail-archive.com/debian-gcc@lists.debian.org/msg26197.html
@@ -425,7 +425,7 @@ TESTSRC = \
   $(TOP)/ext/fts3/fts3_term.c \
   $(TOP)/ext/fts3/fts3_test.c  \
   $(TOP)/ext/session/test_session.c \
-  $(TOP)/ext/rbu/test_rbu.c 
+  $(TOP)/ext/rbu/test_rbu.c
 
 # Statically linked extensions
 #
@@ -507,7 +507,7 @@ TESTSRC2 = \
   $(TOP)/ext/fts3/fts3_write.c \
   $(TOP)/ext/async/sqlite3async.c \
   $(TOP)/ext/session/sqlite3session.c \
-  $(TOP)/ext/misc/stmt.c 
+  $(TOP)/ext/misc/stmt.c
 
 # Header files used by all library source files.
 #
@@ -602,7 +602,7 @@ FUZZCHECK_OPT = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_MEMSYS5 -DSQLITE_OSS_FUZZ
 FUZZCHECK_OPT += -DSQLITE_MAX_MEMORY=50000000
 FUZZCHECK_OPT += -DSQLITE_PRINTF_PRECISION_LIMIT=1000
 FUZZCHECK_SRC = $(TOP)/test/fuzzcheck.c $(TOP)/test/ossfuzz.c
-DBFUZZ_OPT = 
+DBFUZZ_OPT =
 
 # This is the default Makefile target.  The objects listed here
 # are what get build when you type just "make" with no arguments.
@@ -1143,7 +1143,7 @@ FTS5_SRC = \
    $(TOP)/ext/fts5/fts5_varint.c \
    $(TOP)/ext/fts5/fts5_vocab.c  \
 
-fts5parse.c:	$(TOP)/ext/fts5/fts5parse.y lemon 
+fts5parse.c:	$(TOP)/ext/fts5/fts5parse.y lemon
 	cp $(TOP)/ext/fts5/fts5parse.y .
 	rm -f fts5parse.h
 	./lemon$(BEXE) $(OPTS) fts5parse.y
@@ -1170,7 +1170,7 @@ sqlite3rbu.lo:	$(TOP)/ext/rbu/sqlite3rbu.c $(HDR) $(EXTHDR)
 #
 TESTFIXTURE_FLAGS  = -DSQLITE_TEST=1 -DSQLITE_CRASH_TEST=1
 TESTFIXTURE_FLAGS += -DTCLSH_INIT_PROC=sqlite3TestInit
-TESTFIXTURE_FLAGS += -DSQLITE_SERVER=1 -DSQLITE_PRIVATE="" -DSQLITE_CORE 
+TESTFIXTURE_FLAGS += -DSQLITE_SERVER=1 -DSQLITE_PRIVATE="" -DSQLITE_CORE
 TESTFIXTURE_FLAGS += -DBUILD_sqlite
 TESTFIXTURE_FLAGS += -DSQLITE_SERIES_CONSTRAINT_VERIFY=1
 TESTFIXTURE_FLAGS += -DSQLITE_DEFAULT_PAGE_SIZE=1024
@@ -1186,6 +1186,10 @@ testfixture$(TEXE):	$(TESTFIXTURE_SRC)
 	$(LTLINK) -DSQLITE_NO_SYNC=1 $(TEMP_STORE) $(TESTFIXTURE_FLAGS) \
 		-o $@ $(TESTFIXTURE_SRC) $(LIBTCL) $(TLIBS)
 
+coretestprogs:	$(TESTPROGS)
+
+testprogs:	coretestprogs srcck1$(BEXE) fuzzcheck$(TEXE) sessionfuzz$(TEXE)
+
 # A very detailed test running most or all test cases
 fulltest:	$(TESTPROGS) fuzztest
 	./testfixture$(TEXE) $(TOP)/test/all.test $(TESTOPTS)
@@ -1245,7 +1249,7 @@ sqlite3_analyzer.c: sqlite3.c $(TOP)/src/tclsqlite.c $(TOP)/tool/spaceanal.tcl $
 sqlite3_analyzer$(TEXE): sqlite3_analyzer.c
 	$(LTLINK) sqlite3_analyzer.c -o $@ $(LIBTCL) $(TLIBS)
 
-sqltclsh.c: sqlite3.c $(TOP)/src/tclsqlite.c $(TOP)/tool/sqltclsh.tcl $(TOP)/ext/misc/appendvfs.c $(TOP)/tool/mkccode.tcl $(TOP)/tool/sqltclsh.c.in	
+sqltclsh.c: sqlite3.c $(TOP)/src/tclsqlite.c $(TOP)/tool/sqltclsh.tcl $(TOP)/ext/misc/appendvfs.c $(TOP)/tool/mkccode.tcl $(TOP)/tool/sqltclsh.c.in
 	$(TCLSH_CMD) $(TOP)/tool/mkccode.tcl $(TOP)/tool/sqltclsh.c.in >sqltclsh.c
 
 sqltclsh$(TEXE): sqltclsh.c
@@ -1312,7 +1316,7 @@ KV_OPT += -DSQLITE_DIRECT_OVERFLOW_READ
 kvtest$(TEXE):	$(TOP)/test/kvtest.c sqlite3.c
 	$(LTLINK) $(KV_OPT) -o $@ $(TOP)/test/kvtest.c sqlite3.c $(TLIBS)
 
-rbu$(EXE): $(TOP)/ext/rbu/rbu.c $(TOP)/ext/rbu/sqlite3rbu.c sqlite3.lo 
+rbu$(EXE): $(TOP)/ext/rbu/rbu.c $(TOP)/ext/rbu/sqlite3rbu.c sqlite3.lo
 	$(LTLINK) -I. -o $@ $(TOP)/ext/rbu/rbu.c sqlite3.lo $(TLIBS)
 
 loadfts$(EXE): $(TOP)/tool/loadfts.c libsqlite3.la
@@ -1340,7 +1344,7 @@ snapshot-tarball: sqlite3.c
 # The next two rules are used to support the "threadtest" target. Building
 # threadtest runs a few thread-safety tests that are implemented in C. This
 # target is invoked by the releasetest.tcl script.
-# 
+#
 THREADTEST3_SRC = $(TOP)/test/threadtest3.c    \
                   $(TOP)/test/tt3_checkpoint.c \
                   $(TOP)/test/tt3_index.c      \
@@ -1354,7 +1358,7 @@ threadtest3$(TEXE): sqlite3.lo $(THREADTEST3_SRC)
 threadtest: threadtest3$(TEXE)
 	./threadtest3$(TEXE)
 
-releasetest:	
+releasetest:
 	$(TCLSH_CMD) $(TOP)/test/releasetest.tcl
 
 # Standard install and cleanup targets
@@ -1362,7 +1366,7 @@ releasetest:
 lib_install:	libsqlite3.la
 	$(INSTALL) -d $(DESTDIR)$(libdir)
 	$(LTINSTALL) libsqlite3.la $(DESTDIR)$(libdir)
-	
+
 install:	sqlite3$(TEXE) lib_install sqlite3.h sqlite3.pc ${HAVE_TCL:1=tcl_install}
 	$(INSTALL) -d $(DESTDIR)$(bindir)
 	$(LTINSTALL) sqlite3$(TEXE) $(DESTDIR)$(bindir)
@@ -1380,7 +1384,7 @@ tcl_install:	lib_install libtclsqlite3.la pkgIndex.tcl
 	rm -f $(DESTDIR)$(TCLLIBDIR)/libtclsqlite3.la $(DESTDIR)$(TCLLIBDIR)/libtclsqlite3.a
 	$(INSTALL) -m 0644 pkgIndex.tcl $(DESTDIR)$(TCLLIBDIR)
 
-clean:	
+clean:
 	rm -f *.lo *.la *.o sqlite3$(TEXE) libsqlite3.la
 	rm -f sqlite3.h opcodes.*
 	rm -rf .libs .deps
diff --git a/Makefile.msc b/Makefile.msc
index b9fab7c408..2f09f0008d 100644
--- a/Makefile.msc
+++ b/Makefile.msc
@@ -2335,6 +2335,10 @@ extensiontest:	testfixture.exe testloadext.dll
 	@set PATH=$(LIBTCLPATH);$(PATH)
 	.\testfixture.exe $(TOP)\test\loadext.test $(TESTOPTS)
 
+coretestprogs:	$(TESTPROGS)
+
+testprogs:	coretestprogs srcck1.exe fuzzcheck.exe sessionfuzz.exe
+
 fulltest:	$(TESTPROGS) fuzztest
 	@set PATH=$(LIBTCLPATH);$(PATH)
 	.\testfixture.exe $(TOP)\test\all.test $(TESTOPTS)
diff --git a/main.mk b/main.mk
index 1df68076c1..e799896ebc 100644
--- a/main.mk
+++ b/main.mk
@@ -19,7 +19,7 @@
 # EXE              The suffix to add to executable files.  ".exe" for windows
 #                  and "" for Unix.
 #
-# TCC              C Compiler and options for use in building executables that 
+# TCC              C Compiler and options for use in building executables that
 #                  will run on the target platform.  This is usually the same
 #                  as BCC, unless you are cross-compiling.
 #
@@ -43,7 +43,7 @@
 
 # This is how we compile
 #
-TCCX =  $(TCC) $(OPTS) -I. -I$(TOP)/src -I$(TOP) 
+TCCX =  $(TCC) $(OPTS) -I. -I$(TOP)/src -I$(TOP)
 TCCX += -I$(TOP)/ext/rtree -I$(TOP)/ext/icu -I$(TOP)/ext/fts3
 TCCX += -I$(TOP)/ext/async -I$(TOP)/ext/userauth
 TCCX += -I$(TOP)/ext/session
@@ -236,7 +236,7 @@ SRC += \
   $(TOP)/ext/session/sqlite3session.h
 SRC += \
   $(TOP)/ext/userauth/userauth.c \
-  $(TOP)/ext/userauth/sqlite3userauth.h 
+  $(TOP)/ext/userauth/sqlite3userauth.h
 SRC += \
   $(TOP)/ext/rbu/sqlite3rbu.c \
   $(TOP)/ext/rbu/sqlite3rbu.h
@@ -251,7 +251,7 @@ FTS5_HDR = \
    $(TOP)/ext/fts5/fts5.h \
    $(TOP)/ext/fts5/fts5Int.h \
    fts5parse.h
-	   
+
 FTS5_SRC = \
    $(TOP)/ext/fts5/fts5_aux.c \
    $(TOP)/ext/fts5/fts5_buffer.c \
@@ -431,7 +431,7 @@ TESTSRC2 = \
   $(TOP)/ext/async/sqlite3async.c \
   $(TOP)/ext/misc/stmt.c \
   $(TOP)/ext/session/sqlite3session.c \
-  $(TOP)/ext/session/test_session.c 
+  $(TOP)/ext/session/test_session.c
 
 # Header files used by all library source files.
 #
@@ -484,7 +484,7 @@ EXTHDR += \
 EXTHDR += \
   $(TOP)/ext/fts5/fts5Int.h  \
   fts5parse.h                \
-  $(TOP)/ext/fts5/fts5.h 
+  $(TOP)/ext/fts5/fts5.h
 EXTHDR += \
   $(TOP)/ext/userauth/sqlite3userauth.h
 
@@ -800,7 +800,7 @@ rtree.o:	$(TOP)/ext/rtree/rtree.c $(HDR) $(EXTHDR)
 
 
 
-fts5parse.c:	$(TOP)/ext/fts5/fts5parse.y lemon 
+fts5parse.c:	$(TOP)/ext/fts5/fts5parse.y lemon
 	cp $(TOP)/ext/fts5/fts5parse.y .
 	rm -f fts5parse.h
 	./lemon $(OPTS) fts5parse.y
@@ -834,13 +834,13 @@ sqlite3_analyzer.c: sqlite3.c $(TOP)/src/tclsqlite.c $(TOP)/tool/spaceanal.tcl $
 	tclsh $(TOP)/tool/mkccode.tcl $(TOP)/tool/sqlite3_analyzer.c.in >sqlite3_analyzer.c
 
 sqlite3_analyzer$(EXE): sqlite3_analyzer.c
-	$(TCCX) $(TCL_FLAGS) sqlite3_analyzer.c -o $@ $(LIBTCL) $(THREADLIB) 
+	$(TCCX) $(TCL_FLAGS) sqlite3_analyzer.c -o $@ $(LIBTCL) $(THREADLIB)
 
 sqltclsh.c: sqlite3.c $(TOP)/src/tclsqlite.c $(TOP)/tool/sqltclsh.tcl $(TOP)/ext/misc/appendvfs.c $(TOP)/tool/mkccode.tcl
 	tclsh $(TOP)/tool/mkccode.tcl $(TOP)/tool/sqltclsh.c.in >sqltclsh.c
 
 sqltclsh$(EXE): sqltclsh.c
-	$(TCCX) $(TCL_FLAGS) sqltclsh.c -o $@ $(LIBTCL) $(THREADLIB) 
+	$(TCCX) $(TCL_FLAGS) sqltclsh.c -o $@ $(LIBTCL) $(THREADLIB)
 
 sqlite3_expert$(EXE): $(TOP)/ext/expert/sqlite3expert.h $(TOP)/ext/expert/sqlite3expert.c $(TOP)/ext/expert/expert.c sqlite3.c
 	$(TCCX) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION $(TOP)/ext/expert/sqlite3expert.c $(TOP)/ext/expert/expert.c sqlite3.c -o sqlite3_expert$(EXE) $(THREADLIB)
@@ -893,6 +893,10 @@ fts3-testfixture$(EXE): sqlite3.c fts3amal.c $(TESTSRC) $(TOP)/src/tclsqlite.c
 		$(TESTSRC) $(TOP)/src/tclsqlite.c sqlite3.c fts3amal.c       \
 		-o testfixture$(EXE) $(LIBTCL) $(THREADLIB)
 
+coretestprogs:	$(TESTPROGS)
+
+testprogs:	coretestprogs srcck1$(EXE) fuzzcheck$(EXE) sessionfuzz$(EXE)
+
 fulltest:	$(TESTPROGS) fuzztest
 	./testfixture$(EXE) $(TOP)/test/all.test $(TESTOPTS)
 
@@ -949,7 +953,7 @@ smoketest:	$(TESTPROGS) fuzzcheck$(EXE)
 # The next two rules are used to support the "threadtest" target. Building
 # threadtest runs a few thread-safety tests that are implemented in C. This
 # target is invoked by the releasetest.tcl script.
-# 
+#
 THREADTEST3_SRC = $(TOP)/test/threadtest3.c    \
                   $(TOP)/test/tt3_checkpoint.c \
                   $(TOP)/test/tt3_index.c      \
@@ -1013,12 +1017,12 @@ wordcount$(EXE):	$(TOP)/test/wordcount.c sqlite3.c
 		$(TOP)/test/wordcount.c sqlite3.c
 
 speedtest1$(EXE):	$(TOP)/test/speedtest1.c sqlite3.c
-	$(TCCX) -I. $(ST_OPT) -o speedtest1$(EXE) $(TOP)/test/speedtest1.c sqlite3.c $(THREADLIB) 
+	$(TCCX) -I. $(ST_OPT) -o speedtest1$(EXE) $(TOP)/test/speedtest1.c sqlite3.c $(THREADLIB)
 
 kvtest$(EXE):	$(TOP)/test/kvtest.c sqlite3.c
-	$(TCCX) -I. $(KV_OPT) -o kvtest$(EXE) $(TOP)/test/kvtest.c sqlite3.c $(THREADLIB) 
+	$(TCCX) -I. $(KV_OPT) -o kvtest$(EXE) $(TOP)/test/kvtest.c sqlite3.c $(THREADLIB)
 
-rbu$(EXE): $(TOP)/ext/rbu/rbu.c $(TOP)/ext/rbu/sqlite3rbu.c sqlite3.o 
+rbu$(EXE): $(TOP)/ext/rbu/rbu.c $(TOP)/ext/rbu/sqlite3rbu.c sqlite3.o
 	$(TCC) -I. -o rbu$(EXE) $(TOP)/ext/rbu/rbu.c sqlite3.o \
 	  $(THREADLIB)
 
@@ -1050,7 +1054,7 @@ install:	sqlite3 libsqlite3.a sqlite3.h
 	mv libsqlite3.a /usr/lib
 	mv sqlite3.h /usr/include
 
-clean:	
+clean:
 	rm -f *.o sqlite3 sqlite3.exe libsqlite3.a sqlite3.h opcodes.*
 	rm -f lemon lemon.exe lempar.c parse.* sqlite*.tar.gz
 	rm -f mkkeywordhash mkkeywordhash.exe keywordhash.h
@@ -1078,7 +1082,7 @@ clean:
 	rm -f sqlite3rc.h
 	rm -f shell.c sqlite3ext.h
 	rm -f sqlite3_analyzer sqlite3_analyzer.exe sqlite3_analyzer.c
-	rm -f sqlite3_expert sqlite3_expert.exe 
+	rm -f sqlite3_expert sqlite3_expert.exe
 	rm -f sqlite-*-output.vsix
 	rm -f mptester mptester.exe
 	rm -f fuzzershell fuzzershell.exe
diff --git a/manifest b/manifest
index e51359b125..4cd0556450 100644
--- a/manifest
+++ b/manifest
@@ -1,10 +1,10 @@
-C In\sthe\ssessions\smodule,\savoid\scollecting\srebase\sdata\sif\sthe\suser\shas\snot\nrequested\sit.
-D 2018-10-29T17:08:27.831
+C Add\sthe\ssqlite3_normalized_sql()\sAPI.
+D 2018-10-29T17:53:23.938
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
-F Makefile.in 15344f4e44dfd9ffb04e9867bdd352a8a5a86211b8919a6ca724e7063694320b
+F Makefile.in 783093f97550d2fd61618ca76ada6fbbfc6ec26c242a66e8c25a4d9d54cde899
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
-F Makefile.msc b946f8806a5d401a299453f61de80dfd1a9df14fa4902b299e6465e3c3134872
+F Makefile.msc fd51f9eba2cc0da0c26344b7f08addc16a2094640bb60e481dcd6408b901a293
 F README.md 377233394b905d3b2e2b33741289e093bc93f2e7adbe00923b2c5958c9a9edee
 F VERSION 654da1d4053fb09ffc33a3910e6d427182a7dcdc67e934fa83de2849ac83fccb
 F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50
@@ -426,7 +426,7 @@ F ext/userauth/userauth.c f81aa5a3ecacf406f170c62a144405858f6f6de51dbdc0920134e6
 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x
 F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8
 F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60
-F main.mk ff82d38126f8f0668b7990e0f1f3dcd74fa2d477c19b2e3feaaba586051e9b48
+F main.mk ab2257d67e9db1fa9ef6159fadc32ef59ab24b9734cd567622d795392c3b2d83
 F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83
 F mptest/config01.test 3c6adcbc50b991866855f1977ff172eb6d901271
 F mptest/config02.test 4415dfe36c48785f751e16e32c20b077c28ae504
@@ -448,26 +448,26 @@ F src/btmutex.c 8acc2f464ee76324bf13310df5692a262b801808984c1b79defb2503bbafadb6
 F src/btree.c 3f5e1a03db871e627bf5da21092bf7434ecfc5c5980bbd7d45eba13341340173
 F src/btree.h febb2e817be499570b7a2e32a9bbb4b607a9234f6b84bb9ae84916d4806e96f2
 F src/btreeInt.h 620ab4c7235f43572cf3ac2ac8723cbdf68073be4d29da24897c7b77dda5fd96
-F src/build.c 0b3d422770877d74ee6d54f4c122d82c48f7d04ee3bfb91702e402de7f5c45ac
-F src/callback.c 36caff1e7eb7deb58572d59c41cee8f064a11d00297616995c5050ea0cfc1288
+F src/build.c 675799caa8bdd73bea8f8c268a735cb208133ce875bf5c4cbcf7280bebfb2227
+F src/callback.c 789bd33d188146f66c0dd8306472a72d1c05f71924b24a91caf6bd45cf9aba73
 F src/complete.c a3634ab1e687055cd002e11b8f43eb75c17da23e
-F src/ctime.c 56e2f32d2e5491c152352bd53cffc9979ee1e1b70df39ec97a90920ae420a950
+F src/ctime.c 109e58d00f62e8e71ee1eb5944ac18b90171c928ab2e082e058056e1137cc20b
 F src/date.c ebe1dc7c8a347117bb02570f1a931c62dd78f4a2b1b516f4837d45b7d6426957
 F src/dbpage.c 4aa7f26198934dbd002e69418220eae3dbc71b010bbac32bd78faf86b52ce6c3
 F src/dbstat.c 5f96184b8a751b7c92b959b55679f56e409c3b3bbe98eaf5e43189c16d4df082
 F src/delete.c 107e28d3ef8bd72fd11953374ca9107cd74e8b09c3ded076a6048742d26ce7d2
-F src/expr.c 5cee8fb79b1952689af80ed71ed16ad295f29d85de30c7592993b05cf1ec1e06
+F src/expr.c 16dee9504d0c6a09de8aa188fb0989ec3fd48b9704abd4fc80539065f2b52adc
 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007
 F src/fkey.c 972a4ba14296bef2303a0abbad1e3d82bc3c61f9e6ce4e8e9528bdee68748812
 F src/func.c 7c288b4ce309b5a8b8473514b88e1f8e69a80134509a8c0db8e39c858e367e7f
 F src/global.c 9bf034fd560bdd514715170ed8460bb7f823cec113f0569ef3f18a20c7ccd128
-F src/hash.c a12580e143f10301ed5166ea4964ae2853d3905a511d4e0c44497245c7ce1f7a
+F src/hash.c 931ec82d7e070654a8facb42549bbb3a25720171d73ba94c3d3160580d01ef1f
 F src/hash.h ab34c5c54a9e9de2e790b24349ba5aab3dbb4fd4
 F src/hwtime.h 747c1bbe9df21a92e9c50f3bbec1de841dc5e5da
 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71
 F src/insert.c 0a214201afec77880a31a59c33d86b473a160fc5cc31981eab2041ae03d8bf2f
 F src/legacy.c 134ab3e3fae00a0f67a5187981d6935b24b337bcf0f4b3e5c9fa5763da95bf4e
-F src/loadext.c 30b140d0e5031924c56f802760506c0a235ced0dff9f3d95119aa86df12856e2
+F src/loadext.c 448eab53ecdb566a1259ee2d45ebff9c0bc4a2cf393774488775c33e4fbe89bf
 F src/main.c 6275ece0699a957c4709a7ebe29476f132adbe459d18a6b497e234e4669abf91
 F src/malloc.c 07295435093ce354c6d9063ac05a2eeae28bd251d2e63c48b3d67c12c76f7e18
 F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645
@@ -499,22 +499,22 @@ F src/pcache.h 072f94d29281cffd99e46c1539849f248c4b56ae7684c1f36626797fee375170
 F src/pcache1.c 716975564c15eb6679e97f734cec1bfd6c16ac3d4010f05f1f8e509fc7d19880
 F src/pragma.c 35efa85894f1ae533c03c64591dfc82f916ad78250591082bbae72a2811bceab
 F src/pragma.h e50df79399da8c2efc6bd4b7034e242d0dc6ab2016564f08e94103367098b1e4
-F src/prepare.c f8e260d940a0e08494c0f30744521b2f832d7263eca9d02b050cea0ba144b097
+F src/prepare.c bf148a889ed92589dd2e6b99b54107e8c7668ad2f69a41bb55b4bbc701fc6474
 F src/printf.c 0f1177cf1dd4d7827bf64d840768514ec76409abecaca9e8b577dbd065150381
 F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
 F src/resolve.c bc8c79e56439b111e7d9415e44940951f7087e9466c3a9d664558ef0faf31073
 F src/rowset.c d977b011993aaea002cab3e0bb2ce50cf346000dff94e944d547b989f4b1fe93
 F src/select.c 61e867a906f140b73baf4ce7a201ad6dcba30820969f5618ee40e9a0d32c6f5f
 F src/shell.c.in 248af8c0d785c98268353e58c96715313f3b091bf220d35366affc9a9c0d4e1d
-F src/sqlite.h.in 4b4c2f2daeeed4412ba9d81bc78092c69831fe6eda4f0ae5bf951da51a8dccec
+F src/sqlite.h.in 4f95d6f484ce247fa7cbb7382641d40919cfe9c3bf8091bc462638c7bac4efea
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
-F src/sqlite3ext.h 305adca1b5da4a33ce2db5bd236935768e951d5651bfe5560ed55cfcdbce6a63
-F src/sqliteInt.h 75d8266b27c287aeada717a541cf7b7543383fccdb1d7d45a5620f666e864c55
+F src/sqlite3ext.h 960f1b86c3610fa23cb6a267572a97dcf286e77aa0dd3b9b23292ffaa1ea8683
+F src/sqliteInt.h be74ca8df8831848718a9ddcd71af265807ca77ef25f1d3416a7b4378c4372fb
 F src/sqliteLimit.h 1513bfb7b20378aa0041e7022d04acb73525de35b80b252f1b83fedb4de6a76b
 F src/status.c 46e7aec11f79dad50965a5ca5fa9de009f7d6bde08be2156f1538a0a296d4d0e
 F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34
 F src/tclsqlite.c e72862a271348d779672b45a730c33fd0c535e630ff927e8ce4a0c908d1d28c6
-F src/test1.c 9bb042e4afedc570f78638993fc9cc1760d897d3b27dd72c20618044b2a8fa5e
+F src/test1.c e89148fdb0b3aa92af609669078c219d7b26bf442418b547f0db79d82a7d82f9
 F src/test2.c 3efb99ab7f1fc8d154933e02ae1378bac9637da5
 F src/test3.c 61798bb0d38b915067a8c8e03f5a534b431181f802659a6616f9b4ff7d872644
 F src/test4.c 18ec393bb4d0ad1de729f0b94da7267270f3d8e6
@@ -529,7 +529,7 @@ F src/test_backup.c bf5da90c9926df0a4b941f2d92825a01bbe090a0
 F src/test_bestindex.c 78809f11026f18a93fcfd798d9479cba37e1201c830260bf1edc674b2fa9b857
 F src/test_blob.c ae4a0620b478548afb67963095a7417cd06a4ec0a56adb453542203bfdcb31ce
 F src/test_btree.c 8b2dc8b8848cf3a4db93f11578f075e82252a274
-F src/test_config.c 3bbc5e593f308cbff426bb88c9dbf75deab551e5ddcece1251b8d9a40e55aef5
+F src/test_config.c 5ebafbcd5c75ac1c16bb0c8fe926dc325cc03e780943a88ca50e0d9a4fc4d2f5
 F src/test_delete.c e2fe07646dff6300b48d49b2fee2fe192ed389e834dd635e3b3bac0ce0bf9f8f
 F src/test_demovfs.c a0c3bdd45ed044115c2c9f7779e56eafff18741e
 F src/test_devsym.c 1960abbb234b97e9b920f07e99503fc04b443f62bbc3c6ff2c2cea2133e3b8a2
@@ -567,7 +567,7 @@ F src/test_windirent.h 90dfbe95442c9762357fe128dc7ae3dc199d006de93eb33ba3972e0a9
 F src/test_window.c cdae419fdcea5bad6dcd9368c685abdad6deb59e9fc8b84b153de513d394ba3f
 F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9
 F src/threads.c 4ae07fa022a3dc7c5beb373cf744a85d3c5c6c3c
-F src/tokenize.c 9f55961518f77793edd56eee860ecf035d4370ebbb0726ad2f6cada6637fd16b
+F src/tokenize.c 9e781e1ca80eefe7b5d6a9e2cd5c678c847da55fd6f093781fad7950934d4c83
 F src/treeview.c 0ef7dc77d6fe03172ba65dddfd3b3c557b7b7e217ca1963b7665beb266a0e2c0
 F src/trigger.c d3d78568f37fb2e6cdcc2d1e7b60156f15b0b600adec55b83c5d42f6cad250bd
 F src/update.c 1816d56c1bca1ba4e0ef98cac2f49be62858e9df1dc08844c7067eb41cc44274
@@ -577,9 +577,9 @@ F src/util.c d9eb0a6c4aae1b00a7369eadd7ca0bbe946cb4c953b6751aa20d357c2f482157
 F src/vacuum.c 36e7d21a20c0bf6ef4ef7c399d192b5239410b7c4d3c1070fba4e30810d0b855
 F src/vdbe.c 005e691ea4c7d51e6c1a69d9389aeb34700884c85f51681817ddea3fdc2fc39b
 F src/vdbe.h 5081dcc497777efe5e9ebe7330d283a044a005e4bdda2e2e984f03bf89a0d907
-F src/vdbeInt.h f1f35f70460698d8f5a2bdef1001114babf318e2983a067804e2ae077d8e9827
-F src/vdbeapi.c 2ba821c5929a2769e4b217dd85843479c718b8989d414723ec8af0616a83d611
-F src/vdbeaux.c 9fe7760a6b9739f21f3e19ad5364330b0f681998fc52c32358243b0060423474
+F src/vdbeInt.h 8a52b8db3d20f9755a965d864b8653052c7ef1ccadceb2e057047cd421f6336e
+F src/vdbeapi.c ecccfce6f614c33a95952efeec969d163e8349eac314ee2b7b163eda921b5eb0
+F src/vdbeaux.c f547901b1aa9e2d81c63f06893f633648e434180666a827aacb547d7d6c8a601
 F src/vdbeblob.c f5c70f973ea3a9e915d1693278a5f890dc78594300cf4d54e64f2b0917c94191
 F src/vdbemem.c 81329ab760e4ec0162119d9cd10193e0303c45c5935bb20c7ae9139d44dd6641
 F src/vdbesort.c 90aad5a92608f2dd771c96749beabdb562c9d881131a860a7a5bccf66dc3be7f
@@ -1135,7 +1135,7 @@ F test/mutex2.test bfeaeac2e73095b2ac32285d2756e3a65e681660
 F test/nan.test 437d40e6d0778b050d7750726c0cbd2c9936b81962926e8f8c48ca698f00f4d1
 F test/nockpt.test 8c43b25af63b0bd620cf1b003529e37b6f1dc53bd22690e96a1bd73f78dde53a
 F test/nolock.test f196cf8b8fbea4e2ca345140a2b3f3b0da45c76e
-F test/normalize.test 1dedf653ca33b0b55fd0c7967d2861a51f1801a7aa899a02d4c0d7adfcd5acdc
+F test/normalize.test 6a80564d2000702b5919ed2c1069fef0f95762142bc96a71b4c124a845165713
 F test/notify1.test 669b2b743618efdc18ca4b02f45423d5d2304abf
 F test/notify2.test 2ecabaa1305083856b7c39cf32816b612740c161
 F test/notify3.test 10ff25cde502e72a92053a2f215d64bece4ef934
@@ -1774,7 +1774,10 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P a0d47f25ae7bdf98f5b853f23776b3bf86bea7c0dda386664c1e3b1c363c518f
-R e9d59384eb4e31957429edb55e9fdfcd
-U dan
-Z e1a47567ef9afc1d6422d6b424bf5b3a
+P de72a773dd3ad58a7f2233e1fc06bf60deb8892a2719ea8e9b42e7d592c1279f
+R 3a0379f1d958d505b0867531ec02f91e
+T *branch * normalized_sql
+T *sym-normalized_sql *
+T -sym-trunk *
+U mistachkin
+Z aed093f5ed17f4e7a3a3d3f732fe7b71
diff --git a/manifest.uuid b/manifest.uuid
index 171c1f3dc2..df07ff9960 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-de72a773dd3ad58a7f2233e1fc06bf60deb8892a2719ea8e9b42e7d592c1279f
\ No newline at end of file
+592b66e8058dd03a056a036e2606247c9efdb06d15eebe9bcc455f7f55e30ae6
\ No newline at end of file
diff --git a/src/build.c b/src/build.c
index b4041389ba..5a5bea50fc 100644
--- a/src/build.c
+++ b/src/build.c
@@ -633,6 +633,12 @@ static void SQLITE_NOINLINE deleteTable(sqlite3 *db, Table *pTable){
 
   /* Delete the Table structure itself.
   */
+#ifdef SQLITE_ENABLE_NORMALIZE
+  if( pTable->pColHash ){
+    sqlite3HashClear(pTable->pColHash);
+    sqlite3_free(pTable->pColHash);
+  }
+#endif
   sqlite3DeleteColumnNames(db, pTable);
   sqlite3DbFree(db, pTable->zName);
   sqlite3DbFree(db, pTable->zColAff);
diff --git a/src/callback.c b/src/callback.c
index a629b6825e..faf6d520c4 100644
--- a/src/callback.c
+++ b/src/callback.c
@@ -295,6 +295,21 @@ static FuncDef *functionSearch(
   }
   return 0;
 }
+#ifdef SQLITE_ENABLE_NORMALIZE
+FuncDef *sqlite3FunctionSearchN(
+  int h,               /* Hash of the name */
+  const char *zFunc,   /* Name of function */
+  int nFunc            /* Length of the name */
+){
+  FuncDef *p;
+  for(p=sqlite3BuiltinFunctions.a[h]; p; p=p->u.pHash){
+    if( sqlite3StrNICmp(p->zName, zFunc, nFunc)==0 ){
+      return p;
+    }
+  }
+  return 0;
+}
+#endif /* SQLITE_ENABLE_NORMALIZE */
 
 /*
 ** Insert a new FuncDef into a FuncDefHash hash table.
@@ -308,7 +323,7 @@ void sqlite3InsertBuiltinFuncs(
     FuncDef *pOther;
     const char *zName = aDef[i].zName;
     int nName = sqlite3Strlen30(zName);
-    int h = (zName[0] + nName) % SQLITE_FUNC_HASH_SZ;
+    int h = SQLITE_FUNC_HASH(zName[0], nName);
     assert( zName[0]>='a' && zName[0]<='z' );
     pOther = functionSearch(h, zName);
     if( pOther ){
@@ -387,7 +402,7 @@ FuncDef *sqlite3FindFunction(
   */ 
   if( !createFlag && (pBest==0 || (db->mDbFlags & DBFLAG_PreferBuiltin)!=0) ){
     bestScore = 0;
-    h = (sqlite3UpperToLower[(u8)zName[0]] + nName) % SQLITE_FUNC_HASH_SZ;
+    h = SQLITE_FUNC_HASH(sqlite3UpperToLower[(u8)zName[0]], nName);
     p = functionSearch(h, zName);
     while( p ){
       int score = matchQuality(p, nArg, enc);
diff --git a/src/ctime.c b/src/ctime.c
index 5bc1e5c74f..27fc4fe6c2 100644
--- a/src/ctime.c
+++ b/src/ctime.c
@@ -268,6 +268,9 @@ static const char * const sqlite3azCompileOpt[] = {
 #if SQLITE_ENABLE_MULTIPLEX
   "ENABLE_MULTIPLEX",
 #endif
+#if SQLITE_ENABLE_NORMALIZE
+  "ENABLE_NORMALIZE",
+#endif
 #if SQLITE_ENABLE_NULL_TRIM
   "ENABLE_NULL_TRIM",
 #endif
diff --git a/src/expr.c b/src/expr.c
index b2854f9a5d..9c0ce1f87d 100644
--- a/src/expr.c
+++ b/src/expr.c
@@ -2149,6 +2149,14 @@ int sqlite3IsRowid(const char *z){
   if( sqlite3StrICmp(z, "OID")==0 ) return 1;
   return 0;
 }
+#ifdef SQLITE_ENABLE_NORMALIZE
+int sqlite3IsRowidN(const char *z, int n){
+  if( sqlite3StrNICmp(z, "_ROWID_", n)==0 ) return 1;
+  if( sqlite3StrNICmp(z, "ROWID", n)==0 ) return 1;
+  if( sqlite3StrNICmp(z, "OID", n)==0 ) return 1;
+  return 0;
+}
+#endif
 
 /*
 ** pX is the RHS of an IN operator.  If pX is a SELECT statement 
diff --git a/src/hash.c b/src/hash.c
index 79bb85ac01..fba9dc9f80 100644
--- a/src/hash.c
+++ b/src/hash.c
@@ -64,6 +64,20 @@ static unsigned int strHash(const char *z){
   }
   return h;
 }
+#ifdef SQLITE_ENABLE_NORMALIZE
+static unsigned int strHashN(const char *z, int n){
+  unsigned int h = 0;
+  int i;
+  for(i=0; i<n; i++){
+    /* Knuth multiplicative hashing.  (Sorting & Searching, p. 510).
+    ** 0x9e3779b1 is 2654435761 which is the closest prime number to
+    ** (2**32)*golden_ratio, where golden_ratio = (sqrt(5) - 1)/2. */
+    h += sqlite3UpperToLower[z[i]];
+    h *= 0x9e3779b1;
+  }
+  return h;
+}
+#endif /* SQLITE_ENABLE_NORMALIZE */
 
 
 /* Link pNew element into the hash table pH.  If pEntry!=0 then also
@@ -175,6 +189,40 @@ static HashElem *findElementWithHash(
   }
   return &nullElement;
 }
+#ifdef SQLITE_ENABLE_NORMALIZE
+static HashElem *findElementWithHashN(
+  const Hash *pH,     /* The pH to be searched */
+  const char *pKey,   /* The key we are searching for */
+  int nKey,           /* Number of key bytes to use */
+  unsigned int *pHash /* Write the hash value here */
+){
+  HashElem *elem;                /* Used to loop thru the element list */
+  int count;                     /* Number of elements left to test */
+  unsigned int h;                /* The computed hash */
+  static HashElem nullElement = { 0, 0, 0, 0 };
+
+  if( pH->ht ){   /*OPTIMIZATION-IF-TRUE*/
+    struct _ht *pEntry;
+    h = strHashN(pKey, nKey) % pH->htsize;
+    pEntry = &pH->ht[h];
+    elem = pEntry->chain;
+    count = pEntry->count;
+  }else{
+    h = 0;
+    elem = pH->first;
+    count = pH->count;
+  }
+  if( pHash ) *pHash = h;
+  while( count-- ){
+    assert( elem!=0 );
+    if( sqlite3StrNICmp(elem->pKey,pKey,nKey)==0 ){ 
+      return elem;
+    }
+    elem = elem->next;
+  }
+  return &nullElement;
+}
+#endif /* SQLITE_ENABLE_NORMALIZE */
 
 /* Remove a single entry from the hash table given a pointer to that
 ** element and a hash on the element's key.
@@ -219,6 +267,14 @@ void *sqlite3HashFind(const Hash *pH, const char *pKey){
   assert( pKey!=0 );
   return findElementWithHash(pH, pKey, 0)->data;
 }
+#ifdef SQLITE_ENABLE_NORMALIZE
+void *sqlite3HashFindN(const Hash *pH, const char *pKey, int nKey){
+  assert( pH!=0 );
+  assert( pKey!=0 );
+  assert( nKey>=0 );
+  return findElementWithHashN(pH, pKey, nKey, 0)->data;
+}
+#endif /* SQLITE_ENABLE_NORMALIZE */
 
 /* Insert an element into the hash table pH.  The key is pKey
 ** and the data is "data".
diff --git a/src/loadext.c b/src/loadext.c
index 72bfd5c51e..710227d7c4 100644
--- a/src/loadext.c
+++ b/src/loadext.c
@@ -451,7 +451,13 @@ static const sqlite3_api_routines sqlite3Apis = {
   sqlite3_str_length,
   sqlite3_str_value,
   /* Version 3.25.0 and later */
-  sqlite3_create_window_function
+  sqlite3_create_window_function,
+  /* Version 3.26.0 and later */
+#ifdef SQLITE_ENABLE_NORMALIZE
+  sqlite3_normalized_sql
+#else
+  0
+#endif
 };
 
 /*
diff --git a/src/prepare.c b/src/prepare.c
index 602e4dc49d..6d2b804136 100644
--- a/src/prepare.c
+++ b/src/prepare.c
@@ -709,6 +709,294 @@ static int sqlite3LockAndPrepare(
   return rc;
 }
 
+#ifdef SQLITE_ENABLE_NORMALIZE
+/*
+** Checks if the specified token is a table, column, or function name,
+** based on the databases associated with the statement being prepared.
+** If the function fails, zero is returned and pRc is filled with the
+** error code.
+*/
+static int shouldTreatAsIdentifier(
+  sqlite3 *db,        /* Database handle. */
+  const char *zToken, /* Pointer to start of token to be checked */
+  int nToken,         /* Length of token to be checked */
+  int *pRc            /* Pointer to error code upon failure */
+){
+  int bFound = 0;     /* Non-zero if token is an identifier name. */
+  int i, j;           /* Database and column loop indexes. */
+  Schema *pSchema;    /* Schema for current database. */
+  Hash *pHash;        /* Hash table of tables for current database. */
+  HashElem *e;        /* Hash element for hash table iteration. */
+  Table *pTab;        /* Database table for columns being checked. */
+
+  if( sqlite3IsRowidN(zToken, nToken) ){
+    return 1;
+  }
+  if( nToken>0 ){
+    int hash = SQLITE_FUNC_HASH(sqlite3UpperToLower[(u8)zToken[0]], nToken);
+    if( sqlite3FunctionSearchN(hash, zToken, nToken) ) return 1;
+  }
+  assert( db!=0 );
+  sqlite3_mutex_enter(db->mutex);
+  sqlite3BtreeEnterAll(db);
+  for(i=0; i<db->nDb; i++){
+    pHash = &db->aFunc;
+    if( sqlite3HashFindN(pHash, zToken, nToken) ){
+      bFound = 1;
+      break;
+    }
+    pSchema = db->aDb[i].pSchema;
+    if( pSchema==0 ) continue;
+    pHash = &pSchema->tblHash;
+    if( sqlite3HashFindN(pHash, zToken, nToken) ){
+      bFound = 1;
+      break;
+    }
+    for(e=sqliteHashFirst(pHash); e; e=sqliteHashNext(e)){
+      pTab = sqliteHashData(e);
+      if( pTab==0 ) continue;
+      pHash = pTab->pColHash;
+      if( pHash==0 ){
+        pTab->pColHash = pHash = sqlite3_malloc(sizeof(Hash));
+        if( pHash ){
+          sqlite3HashInit(pHash);
+          for(j=0; j<pTab->nCol; j++){
+            Column *pCol = &pTab->aCol[j];
+            sqlite3HashInsert(pHash, pCol->zName, pCol);
+          }
+        }else{
+          *pRc = SQLITE_NOMEM_BKPT;
+          bFound = 0;
+          goto done;
+        }
+      }
+      if( pHash && sqlite3HashFindN(pHash, zToken, nToken) ){
+        bFound = 1;
+        goto done;
+      }
+    }
+  }
+done:
+  sqlite3BtreeLeaveAll(db);
+  sqlite3_mutex_leave(db->mutex);
+  return bFound;
+}
+
+/*
+** Attempt to estimate the final output buffer size needed for the fully
+** normalized version of the specified SQL string.  This should take into
+** account any potential expansion that could occur (e.g. via IN clauses
+** being expanded, etc).  This size returned is the total number of bytes
+** including the NUL terminator.
+*/
+static int estimateNormalizedSize(
+  const char *zSql, /* The original SQL string */
+  int nSql,         /* Length of original SQL string */
+  u8 prepFlags      /* The flags passed to sqlite3_prepare_v3() */
+){
+  int nOut = nSql + 4;
+  const char *z = zSql;
+  while( nOut<nSql*5 ){
+    while( z[0]!=0 && z[0]!='I' && z[0]!='i' ){ z++; }
+    if( z[0]==0 ) break;
+    z++;
+    if( z[0]!='N' && z[0]!='n' ) break;
+    z++;
+    while( sqlite3Isspace(z[0]) ){ z++; }
+    if( z[0]!='(' ) break;
+    z++;
+    nOut += 5; /* ?,?,? */
+  }
+  return nOut;
+}
+
+/*
+** Copy the current token into the output buffer while dealing with quoted
+** identifiers.  By default, all letters will be converted into lowercase.
+** If the bUpper flag is set, uppercase will be used.  The piOut argument
+** will be used to update the target index into the output string.
+*/
+static void copyNormalizedToken(
+  const char *zSql, /* The original SQL string */
+  int iIn,          /* Current index into the original SQL string */
+  int nToken,       /* Number of bytes in the current token */
+  int tokenFlags,   /* Flags returned by the tokenizer */
+  char *zOut,       /* The output string */
+  int *piOut        /* Pointer to target index into the output string */
+){
+  int bQuoted = tokenFlags & SQLITE_TOKEN_QUOTED;
+  int bKeyword = tokenFlags & SQLITE_TOKEN_KEYWORD;
+  int j = *piOut, k = 0;
+  for(; k<nToken; k++){
+    if( bQuoted ){
+      if( k==0 && iIn>0 ){
+        zOut[j++] = '"';
+        continue;
+      }else if( k==nToken-1 ){
+        zOut[j++] = '"';
+        continue;
+      }
+    }
+    if( bKeyword ){
+      zOut[j++] = sqlite3Toupper(zSql[iIn+k]);
+    }else{
+      zOut[j++] = sqlite3Tolower(zSql[iIn+k]);
+    }
+  }
+  *piOut = j;
+}
+
+/*
+** Perform normalization of the SQL contained in the prepared statement and
+** store the result in the zNormSql field.  The schema for the associated
+** databases are consulted while performing the normalization in order to
+** determine if a token appears to be an identifier.  All identifiers are
+** left intact in the normalized SQL and all literals are replaced with a
+** single '?'.
+*/
+void sqlite3Normalize(
+  Vdbe *pVdbe,      /* VM being reprepared */
+  const char *zSql, /* The original SQL string */
+  int nSql,         /* Size of the input string in bytes */
+  u8 prepFlags      /* The flags passed to sqlite3_prepare_v3() */
+){
+  sqlite3 *db;           /* Database handle. */
+  char *z;               /* The output string */
+  int nZ;                /* Size of the output string in bytes */
+  int i;                 /* Next character to read from zSql[] */
+  int j;                 /* Next character to fill in on z[] */
+  int tokenType = 0;     /* Type of the next token */
+  int prevTokenType = 0; /* Type of the previous token, except spaces */
+  int n;                 /* Size of the next token */
+  int nParen = 0;        /* Nesting level of parenthesis */
+  Hash inHash;           /* Table of parenthesis levels to output index. */
+
+  db = sqlite3VdbeDb(pVdbe);
+  assert( db!=0 );
+  assert( pVdbe->zNormSql==0 );
+  if( zSql==0 ) return;
+  nZ = estimateNormalizedSize(zSql, nSql, prepFlags);
+  z = sqlite3DbMallocRawNN(db, nZ);
+  if( z==0 ) return;
+  sqlite3HashInit(&inHash);
+  for(i=j=0; i<nSql && zSql[i]; i+=n){
+    int flags = 0;
+    if( tokenType!=TK_SPACE ) prevTokenType = tokenType;
+    n = sqlite3GetTokenNormalized((unsigned char*)zSql+i, &tokenType, &flags);
+    switch( tokenType ){
+      case TK_SPACE: {
+        break;
+      }
+      case TK_ILLEGAL: {
+        sqlite3DbFree(db, z);
+        sqlite3HashClear(&inHash);
+        return;
+      }
+      case TK_STRING:
+      case TK_INTEGER:
+      case TK_FLOAT:
+      case TK_VARIABLE:
+      case TK_BLOB: {
+        z[j++] = '?';
+        break;
+      }
+      case TK_LP:
+      case TK_RP: {
+        if( tokenType==TK_LP ){
+          nParen++;
+          if( prevTokenType==TK_IN ){
+            assert( nParen<nSql );
+            sqlite3HashInsert(&inHash, zSql+nParen, SQLITE_INT_TO_PTR(j));
+          }
+        }else{
+          int jj;
+          assert( nParen<nSql );
+          jj = SQLITE_PTR_TO_INT(sqlite3HashFind(&inHash, zSql+nParen));
+          if( jj>0 ){
+            sqlite3HashInsert(&inHash, zSql+nParen, 0);
+            assert( jj+6<nZ );
+            memcpy(z+jj+1, "?,?,?", 5);
+            j = jj+6;
+            assert( nZ-1-j>=0 );
+            assert( nZ-1-j<nZ );
+            memset(z+j, 0, nZ-1-j);
+          }
+          nParen--;
+        }
+        assert( nParen>=0 );
+        /* Fall through */
+      }
+      case TK_MINUS:
+      case TK_SEMI:
+      case TK_PLUS:
+      case TK_STAR:
+      case TK_SLASH:
+      case TK_REM:
+      case TK_EQ:
+      case TK_LE:
+      case TK_NE:
+      case TK_LSHIFT:
+      case TK_LT:
+      case TK_RSHIFT:
+      case TK_GT:
+      case TK_GE:
+      case TK_BITOR:
+      case TK_CONCAT:
+      case TK_COMMA:
+      case TK_BITAND:
+      case TK_BITNOT:
+      case TK_DOT:
+      case TK_IN:
+      case TK_IS:
+      case TK_NOT:
+      case TK_NULL:
+      case TK_ID: {
+        if( tokenType==TK_NULL ){
+          if( prevTokenType==TK_IS || prevTokenType==TK_NOT ){
+            /* NULL is a keyword in this case, not a literal value */
+          }else{
+            /* Here the NULL is a literal value */
+            z[j++] = '?';
+            break;
+          }
+        }
+        if( j>0 && sqlite3IsIdChar(z[j-1]) && sqlite3IsIdChar(zSql[i]) ){
+          z[j++] = ' ';
+        }
+        if( tokenType==TK_ID ){
+          int i2 = i, n2 = n, rc = SQLITE_OK;
+          if( nParen>0 ){
+            assert( nParen<nSql );
+            sqlite3HashInsert(&inHash, zSql+nParen, 0);
+          }
+          if( flags&SQLITE_TOKEN_QUOTED ){ i2++; n2-=2; }
+          if( shouldTreatAsIdentifier(db, zSql+i2, n2, &rc)==0 ){
+            if( rc!=SQLITE_OK ){
+              sqlite3DbFree(db, z);
+              sqlite3HashClear(&inHash);
+              return;
+            }
+            if( sqlite3_keyword_check(zSql+i2, n2)==0 ){
+              z[j++] = '?';
+              break;
+            }
+          }
+        }
+        copyNormalizedToken(zSql, i, n, flags, z, &j);
+        break;
+      }
+    }
+  }
+  assert( j<nZ && "one" );
+  while( j>0 && z[j-1]==' ' ){ j--; }
+  if( j>0 && z[j-1]!=';' ){ z[j++] = ';'; }
+  z[j] = 0;
+  assert( j<nZ && "two" );
+  pVdbe->zNormSql = z;
+  sqlite3HashClear(&inHash);
+}
+#endif /* SQLITE_ENABLE_NORMALIZE */
+
 /*
 ** Rerun the compilation of a statement after a schema change.
 **
diff --git a/src/sqlite.h.in b/src/sqlite.h.in
index 2d090e3ec7..9ead2b7aa6 100644
--- a/src/sqlite.h.in
+++ b/src/sqlite.h.in
@@ -3609,9 +3609,19 @@ int sqlite3_limit(sqlite3*, int id, int newVal);
 ** on this hint by avoiding the use of [lookaside memory] so as not to
 ** deplete the limited store of lookaside memory. Future versions of
 ** SQLite may act on this hint differently.
+**
+** [[SQLITE_PREPARE_NORMALIZE]] ^(<dt>SQLITE_PREPARE_NORMALIZE</dt>
+** <dd>The SQLITE_PREPARE_NORMALIZE flag indicates that a normalized
+** representation of the SQL statement should be calculated and then
+** associated with the prepared statement, which can be obtained via
+** the [sqlite3_normalized_sql()] interface.  The semantics used to
+** normalize a SQL statement are unspecified and subject to change.
+** At a minimum, literal values will be replaced with suitable
+** placeholders.
 ** </dl>
 */
 #define SQLITE_PREPARE_PERSISTENT              0x01
+#define SQLITE_PREPARE_NORMALIZE               0x02
 
 /*
 ** CAPI3REF: Compiling An SQL Statement
@@ -3769,6 +3779,11 @@ int sqlite3_prepare16_v3(
 ** ^The sqlite3_expanded_sql(P) interface returns a pointer to a UTF-8
 ** string containing the SQL text of prepared statement P with
 ** [bound parameters] expanded.
+** ^The sqlite3_normalized_sql(P) interface returns a pointer to a UTF-8
+** string containing the normalized SQL text of prepared statement P.  The
+** semantics used to normalize a SQL statement are unspecified and subject
+** to change.  At a minimum, literal values will be replaced with suitable
+** placeholders.
 **
 ** ^(For example, if a prepared statement is created using the SQL
 ** text "SELECT $abc,:xyz" and if parameter $abc is bound to integer 2345
@@ -3784,14 +3799,16 @@ int sqlite3_prepare16_v3(
 ** bound parameter expansions.  ^The [SQLITE_OMIT_TRACE] compile-time
 ** option causes sqlite3_expanded_sql() to always return NULL.
 **
-** ^The string returned by sqlite3_sql(P) is managed by SQLite and is
-** automatically freed when the prepared statement is finalized.
+** ^The strings returned by sqlite3_sql(P) and sqlite3_normalized_sql(P)
+** are managed by SQLite and are automatically freed when the prepared
+** statement is finalized.
 ** ^The string returned by sqlite3_expanded_sql(P), on the other hand,
 ** is obtained from [sqlite3_malloc()] and must be free by the application
 ** by passing it to [sqlite3_free()].
 */
 const char *sqlite3_sql(sqlite3_stmt *pStmt);
 char *sqlite3_expanded_sql(sqlite3_stmt *pStmt);
+const char *sqlite3_normalized_sql(sqlite3_stmt *pStmt);
 
 /*
 ** CAPI3REF: Determine If An SQL Statement Writes The Database
diff --git a/src/sqlite3ext.h b/src/sqlite3ext.h
index 35d9950cf6..34c41fd5a9 100644
--- a/src/sqlite3ext.h
+++ b/src/sqlite3ext.h
@@ -310,12 +310,15 @@ struct sqlite3_api_routines {
   int (*str_errcode)(sqlite3_str*);
   int (*str_length)(sqlite3_str*);
   char *(*str_value)(sqlite3_str*);
+  /* Version 3.25.0 and later */
   int (*create_window_function)(sqlite3*,const char*,int,int,void*,
                             void (*xStep)(sqlite3_context*,int,sqlite3_value**),
                             void (*xFinal)(sqlite3_context*),
                             void (*xValue)(sqlite3_context*),
                             void (*xInv)(sqlite3_context*,int,sqlite3_value**),
                             void(*xDestroy)(void*));
+  /* Version 3.26.0 and later */
+  const char *(*normalized_sql)(sqlite3_stmt*);
 };
 
 /*
@@ -603,6 +606,8 @@ typedef int (*sqlite3_loadext_entry)(
 #define sqlite3_str_value              sqlite3_api->str_value
 /* Version 3.25.0 and later */
 #define sqlite3_create_window_function sqlite3_api->create_window_function
+/* Version 3.26.0 and later */
+#define sqlite3_normalized_sql         sqlite3_api->normalized_sql
 #endif /* !defined(SQLITE_CORE) && !defined(SQLITE_OMIT_LOAD_EXTENSION) */
 
 #if !defined(SQLITE_CORE) && !defined(SQLITE_OMIT_LOAD_EXTENSION)
diff --git a/src/sqliteInt.h b/src/sqliteInt.h
index f0ed023c6d..5a6042b2b5 100644
--- a/src/sqliteInt.h
+++ b/src/sqliteInt.h
@@ -1312,6 +1312,8 @@ struct FuncDefHash {
   FuncDef *a[SQLITE_FUNC_HASH_SZ];       /* Hash table for functions */
 };
 
+#define SQLITE_FUNC_HASH(C,L) (((C)+(L))%SQLITE_FUNC_HASH_SZ)
+
 #ifdef SQLITE_USER_AUTHENTICATION
 /*
 ** Information held in the "sqlite3" database connection object and used
@@ -1943,6 +1945,9 @@ struct VTable {
 struct Table {
   char *zName;         /* Name of the table or view */
   Column *aCol;        /* Information about each column */
+#ifdef SQLITE_ENABLE_NORMALIZE
+  Hash *pColHash;      /* All columns indexed by name */
+#endif
   Index *pIndex;       /* List of SQL indexes on this table. */
   Select *pSelect;     /* NULL for tables.  Points to definition if a view. */
   FKey *pFKey;         /* Linked list of all foreign keys in this table */
@@ -2279,6 +2284,12 @@ struct IndexSample {
   tRowcnt *anDLt;   /* Est. number of distinct keys less than this sample */
 };
 
+/*
+** Possible values to use within the flags argument to sqlite3GetToken().
+*/
+#define SQLITE_TOKEN_QUOTED    0x1 /* Token is a quoted identifier. */
+#define SQLITE_TOKEN_KEYWORD   0x2 /* Token is a keyword. */
+
 /*
 ** Each token coming out of the lexer is an instance of
 ** this structure.  Tokens are also used as part of an expression.
@@ -4000,6 +4011,9 @@ int sqlite3ExprIsInteger(Expr*, int*);
 int sqlite3ExprCanBeNull(const Expr*);
 int sqlite3ExprNeedsNoAffinityChange(const Expr*, char);
 int sqlite3IsRowid(const char*);
+#ifdef SQLITE_ENABLE_NORMALIZE
+int sqlite3IsRowidN(const char*, int);
+#endif
 void sqlite3GenerateRowDelete(
     Parse*,Table*,Trigger*,int,int,int,i16,u8,u8,u8,int);
 void sqlite3GenerateRowIndexDelete(Parse*, Table*, int, int, int*, int);
@@ -4026,6 +4040,9 @@ ExprList *sqlite3ExprListDup(sqlite3*,ExprList*,int);
 SrcList *sqlite3SrcListDup(sqlite3*,SrcList*,int);
 IdList *sqlite3IdListDup(sqlite3*,IdList*);
 Select *sqlite3SelectDup(sqlite3*,Select*,int);
+#ifdef SQLITE_ENABLE_NORMALIZE
+FuncDef *sqlite3FunctionSearchN(int,const char*,int);
+#endif
 void sqlite3InsertBuiltinFuncs(FuncDef*,int);
 FuncDef *sqlite3FindFunction(sqlite3*,const char*,int,u8,u8);
 void sqlite3RegisterBuiltinFunctions(void);
@@ -4229,6 +4246,9 @@ void sqlite3AlterFunctions(void);
 void sqlite3AlterRenameTable(Parse*, SrcList*, Token*);
 void sqlite3AlterRenameColumn(Parse*, SrcList*, Token*, Token*);
 int sqlite3GetToken(const unsigned char *, int *);
+#ifdef SQLITE_ENABLE_NORMALIZE
+int sqlite3GetTokenNormalized(const unsigned char *, int *, int *);
+#endif
 void sqlite3NestedParse(Parse*, const char*, ...);
 void sqlite3ExpirePreparedStatements(sqlite3*, int);
 int sqlite3CodeSubselect(Parse*, Expr *, int, int);
@@ -4386,6 +4406,9 @@ sqlite3_int64 sqlite3StmtCurrentTime(sqlite3_context*);
 int sqlite3VdbeParameterIndex(Vdbe*, const char*, int);
 int sqlite3TransferBindings(sqlite3_stmt *, sqlite3_stmt *);
 void sqlite3ParserReset(Parse*);
+#ifdef SQLITE_ENABLE_NORMALIZE
+void sqlite3Normalize(Vdbe*, const char*, int, u8);
+#endif
 int sqlite3Reprepare(Vdbe*);
 void sqlite3ExprListCheckLength(Parse*, ExprList*, const char*);
 CollSeq *sqlite3BinaryCompareCollSeq(Parse *, Expr *, Expr *);
diff --git a/src/test1.c b/src/test1.c
index d2c997bdfe..7fee969243 100644
--- a/src/test1.c
+++ b/src/test1.c
@@ -4218,6 +4218,7 @@ static int SQLITE_TCLAPI test_prepare_v2(
   char *zCopy = 0;                /* malloc() copy of zSql */
   int bytes;
   const char *zTail = 0;
+  const char **pzTail;
   sqlite3_stmt *pStmt = 0;
   char zBuf[50];
   int rc;
@@ -4242,7 +4243,8 @@ static int SQLITE_TCLAPI test_prepare_v2(
     zCopy = malloc(n);
     memcpy(zCopy, zSql, n);
   }
-  rc = sqlite3_prepare_v2(db, zCopy, bytes, &pStmt, objc>=5 ? &zTail : 0);
+  pzTail = objc>=5 ? &zTail : 0;
+  rc = sqlite3_prepare_v2(db, zCopy, bytes, &pStmt, pzTail);
   free(zCopy);
   zTail = &zSql[(zTail - zCopy)];
 
@@ -4269,6 +4271,79 @@ static int SQLITE_TCLAPI test_prepare_v2(
   return TCL_OK;
 }
 
+/*
+** Usage: sqlite3_prepare_v3 DB sql bytes flags ?tailvar?
+**
+** Compile up to <bytes> bytes of the supplied SQL string <sql> using
+** database handle <DB> and flags <flags>. The parameter <tailval> is
+** the name of a global variable that is set to the unused portion of
+** <sql> (if any). A STMT handle is returned.
+*/
+static int SQLITE_TCLAPI test_prepare_v3(
+  void * clientData,
+  Tcl_Interp *interp,
+  int objc,
+  Tcl_Obj *CONST objv[]
+){
+  sqlite3 *db;
+  const char *zSql;
+  char *zCopy = 0;                /* malloc() copy of zSql */
+  int bytes, flags;
+  const char *zTail = 0;
+  const char **pzTail;
+  sqlite3_stmt *pStmt = 0;
+  char zBuf[50];
+  int rc;
+
+  if( objc!=6 && objc!=5 ){
+    Tcl_AppendResult(interp, "wrong # args: should be \"", 
+       Tcl_GetString(objv[0]), " DB sql bytes flags tailvar", 0);
+    return TCL_ERROR;
+  }
+  if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ) return TCL_ERROR;
+  zSql = Tcl_GetString(objv[2]);
+  if( Tcl_GetIntFromObj(interp, objv[3], &bytes) ) return TCL_ERROR;
+  if( Tcl_GetIntFromObj(interp, objv[4], &flags) ) return TCL_ERROR;
+
+  /* Instead of using zSql directly, make a copy into a buffer obtained
+  ** directly from malloc(). The idea is to make it easier for valgrind
+  ** to spot buffer overreads.  */
+  if( bytes>=0 ){
+    zCopy = malloc(bytes);
+    memcpy(zCopy, zSql, bytes);
+  }else{
+    int n = (int)strlen(zSql) + 1;
+    zCopy = malloc(n);
+    memcpy(zCopy, zSql, n);
+  }
+  pzTail = objc>=6 ? &zTail : 0;
+  rc = sqlite3_prepare_v3(db, zCopy, bytes, (unsigned int)flags,&pStmt,pzTail);
+  free(zCopy);
+  zTail = &zSql[(zTail - zCopy)];
+
+  assert(rc==SQLITE_OK || pStmt==0);
+  Tcl_ResetResult(interp);
+  if( sqlite3TestErrCode(interp, db, rc) ) return TCL_ERROR;
+  if( rc==SQLITE_OK && zTail && objc>=6 ){
+    if( bytes>=0 ){
+      bytes = bytes - (int)(zTail-zSql);
+    }
+    Tcl_ObjSetVar2(interp, objv[5], 0, Tcl_NewStringObj(zTail, bytes), 0);
+  }
+  if( rc!=SQLITE_OK ){
+    assert( pStmt==0 );
+    sqlite3_snprintf(sizeof(zBuf), zBuf, "(%d) ", rc);
+    Tcl_AppendResult(interp, zBuf, sqlite3_errmsg(db), 0);
+    return TCL_ERROR;
+  }
+
+  if( pStmt ){
+    if( sqlite3TestMakePointerStr(interp, zBuf, pStmt) ) return TCL_ERROR;
+    Tcl_AppendResult(interp, zBuf, 0);
+  }
+  return TCL_OK;
+}
+
 /*
 ** Usage: sqlite3_prepare_tkt3134 DB
 **
@@ -4676,6 +4751,25 @@ static int SQLITE_TCLAPI test_ex_sql(
   sqlite3_free(z);
   return TCL_OK;
 }
+#ifdef SQLITE_ENABLE_NORMALIZE
+static int SQLITE_TCLAPI test_norm_sql(
+  void * clientData,
+  Tcl_Interp *interp,
+  int objc,
+  Tcl_Obj *CONST objv[]
+){
+  sqlite3_stmt *pStmt;
+
+  if( objc!=2 ){
+    Tcl_WrongNumArgs(interp, 1, objv, "STMT");
+    return TCL_ERROR;
+  }
+
+  if( getStmtPointer(interp, Tcl_GetString(objv[1]), &pStmt) ) return TCL_ERROR;
+  Tcl_SetResult(interp, (char *)sqlite3_normalized_sql(pStmt), TCL_VOLATILE);
+  return TCL_OK;
+}
+#endif /* SQLITE_ENABLE_NORMALIZE */
 
 /*
 ** Usage: sqlite3_column_count STMT 
@@ -7646,6 +7740,7 @@ int Sqlitetest1_Init(Tcl_Interp *interp){
      { "sqlite3_prepare",               test_prepare       ,0 },
      { "sqlite3_prepare16",             test_prepare16     ,0 },
      { "sqlite3_prepare_v2",            test_prepare_v2    ,0 },
+     { "sqlite3_prepare_v3",            test_prepare_v3    ,0 },
      { "sqlite3_prepare_tkt3134",       test_prepare_tkt3134, 0},
      { "sqlite3_prepare16_v2",          test_prepare16_v2  ,0 },
      { "sqlite3_finalize",              test_finalize      ,0 },
@@ -7657,6 +7752,9 @@ int Sqlitetest1_Init(Tcl_Interp *interp){
      { "sqlite3_step",                  test_step          ,0 },
      { "sqlite3_sql",                   test_sql           ,0 },
      { "sqlite3_expanded_sql",          test_ex_sql        ,0 },
+#ifdef SQLITE_ENABLE_NORMALIZE
+     { "sqlite3_normalized_sql",        test_norm_sql      ,0 },
+#endif
      { "sqlite3_next_stmt",             test_next_stmt     ,0 },
      { "sqlite3_stmt_readonly",         test_stmt_readonly ,0 },
      { "sqlite3_stmt_busy",             test_stmt_busy     ,0 },
diff --git a/src/test_config.c b/src/test_config.c
index f017abc307..05002349bd 100644
--- a/src/test_config.c
+++ b/src/test_config.c
@@ -762,6 +762,12 @@ Tcl_SetVar2(interp, "sqlite_options", "mergesort", "1", TCL_GLOBAL_ONLY);
   Tcl_SetVar2(interp, "sqlite_options", "uri_00_error", "0", TCL_GLOBAL_ONLY);
 #endif
 
+#if defined(SQLITE_ENABLE_NORMALIZE)
+  Tcl_SetVar2(interp, "sqlite_options", "normalize", "1", TCL_GLOBAL_ONLY);
+#else
+  Tcl_SetVar2(interp, "sqlite_options", "normalize", "0", TCL_GLOBAL_ONLY);
+#endif
+
 #ifdef SQLITE_OMIT_WINDOWFUNC
   Tcl_SetVar2(interp, "sqlite_options", "windowfunc", "0", TCL_GLOBAL_ONLY);
 #else
diff --git a/src/tokenize.c b/src/tokenize.c
index 262144ff7d..05ca86e74f 100644
--- a/src/tokenize.c
+++ b/src/tokenize.c
@@ -545,6 +545,73 @@ int sqlite3GetToken(const unsigned char *z, int *tokenType){
   return i;
 }
 
+#ifdef SQLITE_ENABLE_NORMALIZE
+/*
+** Return the length (in bytes) of the token that begins at z[0].
+** Store the token type in *tokenType before returning.  If flags has
+** SQLITE_TOKEN_NORMALIZE flag enabled, use the identifier token type
+** for keywords.  Add SQLITE_TOKEN_QUOTED to flags if the token was
+** actually a quoted identifier.  Add SQLITE_TOKEN_KEYWORD to flags
+** if the token was recognized as a keyword; this is useful when the
+** SQLITE_TOKEN_NORMALIZE flag is used, because it enables the caller
+** to differentiate between a keyword being treated as an identifier
+** (for normalization purposes) and an actual identifier.
+*/
+int sqlite3GetTokenNormalized(
+  const unsigned char *z,
+  int *tokenType,
+  int *flags
+){
+  int n;
+  unsigned char iClass = aiClass[*z];
+  if( iClass==CC_KYWD ){
+    int i;
+    for(i=1; aiClass[z[i]]<=CC_KYWD; i++){}
+    if( IdChar(z[i]) ){
+      /* This token started out using characters that can appear in keywords,
+      ** but z[i] is a character not allowed within keywords, so this must
+      ** be an identifier instead */
+      i++;
+      while( IdChar(z[i]) ){ i++; }
+      *tokenType = TK_ID;
+      return i;
+    }
+    *tokenType = TK_ID;
+    n = keywordCode((char*)z, i, tokenType);
+    /* If the token is no longer considered to be an identifier, then it is a
+    ** keyword of some kind.  Make the token back into an identifier and then
+    ** set the SQLITE_TOKEN_KEYWORD flag.  Several non-identifier tokens are
+    ** used verbatim, including IN, IS, NOT, and NULL. */
+    switch( *tokenType ){
+      case TK_ID: {
+        /* do nothing, handled by caller */
+        break;
+      }
+      case TK_IN:
+      case TK_IS:
+      case TK_NOT:
+      case TK_NULL: {
+        *flags |= SQLITE_TOKEN_KEYWORD;
+        break;
+      }
+      default: {
+        *tokenType = TK_ID;
+        *flags |= SQLITE_TOKEN_KEYWORD;
+        break;
+      }
+    }
+  }else{
+    n = sqlite3GetToken(z, tokenType);
+    /* If the token is considered to be an identifier and the character class
+    ** of the first character is a quote, set the SQLITE_TOKEN_QUOTED flag. */
+    if( *tokenType==TK_ID && (iClass==CC_QUOTE || iClass==CC_QUOTE2) ){
+      *flags |= SQLITE_TOKEN_QUOTED;
+    }
+  }
+  return n;
+}
+#endif /* SQLITE_ENABLE_NORMALIZE */
+
 /*
 ** Run the parser on the given SQL string.  The parser structure is
 ** passed in.  An SQLITE_ status code is returned.  If an error occurs
diff --git a/src/vdbeInt.h b/src/vdbeInt.h
index 107e5cab44..1cb7219611 100644
--- a/src/vdbeInt.h
+++ b/src/vdbeInt.h
@@ -406,6 +406,9 @@ struct Vdbe {
   yDbMask lockMask;       /* Subset of btreeMask that requires a lock */
   u32 aCounter[7];        /* Counters used by sqlite3_stmt_status() */
   char *zSql;             /* Text of the SQL statement that generated this */
+#ifdef SQLITE_ENABLE_NORMALIZE
+  char *zNormSql;         /* Normalization of the associated SQL statement */
+#endif
   void *pFree;            /* Free this when deleting the vdbe */
   VdbeFrame *pFrame;      /* Parent frame */
   VdbeFrame *pDelFrame;   /* List of frame objects to free on VM reset */
diff --git a/src/vdbeapi.c b/src/vdbeapi.c
index b21f70e7e0..59327bed38 100644
--- a/src/vdbeapi.c
+++ b/src/vdbeapi.c
@@ -1702,6 +1702,16 @@ char *sqlite3_expanded_sql(sqlite3_stmt *pStmt){
 #endif
 }
 
+#ifdef SQLITE_ENABLE_NORMALIZE
+/*
+** Return the normalized SQL associated with a prepared statement.
+*/
+const char *sqlite3_normalized_sql(sqlite3_stmt *pStmt){
+  Vdbe *p = (Vdbe *)pStmt;
+  return p ? p->zNormSql : 0;
+}
+#endif /* SQLITE_ENABLE_NORMALIZE */
+
 #ifdef SQLITE_ENABLE_PREUPDATE_HOOK
 /*
 ** Allocate and populate an UnpackedRecord structure based on the serialized
diff --git a/src/vdbeaux.c b/src/vdbeaux.c
index 99df435966..f1496a3abf 100644
--- a/src/vdbeaux.c
+++ b/src/vdbeaux.c
@@ -64,6 +64,13 @@ void sqlite3VdbeSetSql(Vdbe *p, const char *z, int n, u8 prepFlags){
   }
   assert( p->zSql==0 );
   p->zSql = sqlite3DbStrNDup(p->db, z, n);
+#ifdef SQLITE_ENABLE_NORMALIZE
+  assert( p->zNormSql==0 );
+  if( p->zSql && (prepFlags & SQLITE_PREPARE_NORMALIZE)!=0 ){
+    sqlite3Normalize(p, p->zSql, n, prepFlags);
+    assert( p->zNormSql!=0 || p->db->mallocFailed );
+  }
+#endif
 }
 
 /*
@@ -85,6 +92,11 @@ void sqlite3VdbeSwap(Vdbe *pA, Vdbe *pB){
   zTmp = pA->zSql;
   pA->zSql = pB->zSql;
   pB->zSql = zTmp;
+#ifdef SQLITE_ENABLE_NORMALIZE
+  zTmp = pA->zNormSql;
+  pA->zNormSql = pB->zNormSql;
+  pB->zNormSql = zTmp;
+#endif
   pB->expmask = pA->expmask;
   pB->prepFlags = pA->prepFlags;
   memcpy(pB->aCounter, pA->aCounter, sizeof(pB->aCounter));
@@ -3156,6 +3168,9 @@ void sqlite3VdbeClearObject(sqlite3 *db, Vdbe *p){
   vdbeFreeOpArray(db, p->aOp, p->nOp);
   sqlite3DbFree(db, p->aColName);
   sqlite3DbFree(db, p->zSql);
+#ifdef SQLITE_ENABLE_NORMALIZE
+  sqlite3DbFree(db, p->zNormSql);
+#endif
 #ifdef SQLITE_ENABLE_STMT_SCANSTATUS
   {
     int i;
diff --git a/test/normalize.test b/test/normalize.test
index 9e3ec9c967..e85bd00332 100644
--- a/test/normalize.test
+++ b/test/normalize.test
@@ -72,4 +72,293 @@ foreach {tnum sql norm} {
   do_test $tnum [list sqlite3_normalize $sql] $norm
 }
 
+ifcapable normalize {
+do_test 200 {
+  execsql {
+    CREATE TABLE t1(a,b);
+  }
+} {}
+do_test 201 {
+  set STMT [sqlite3_prepare_v3 $DB \
+      "SELECT a, b FROM t1 WHERE b = ? ORDER BY a;" -1 0 TAIL]
+
+  sqlite3_bind_null $STMT 1
+} {}
+do_test 202 {
+  sqlite3_normalized_sql $STMT
+} {}
+do_test 203 {
+  sqlite3_finalize $STMT
+} {SQLITE_OK}
+
+do_test 210 {
+  set STMT [sqlite3_prepare_v3 $DB \
+      "SELECT a, b FROM t1 WHERE b = ? ORDER BY a;" -1 2 TAIL]
+
+  sqlite3_bind_null $STMT 1
+} {}
+do_test 211 {
+  sqlite3_normalized_sql $STMT
+} {SELECT a,b FROM t1 WHERE b=?ORDER BY a;}
+do_test 212 {
+  sqlite3_finalize $STMT
+} {SQLITE_OK}
+
+do_test 220 {
+  set STMT [sqlite3_prepare_v3 $DB \
+      "SELECT a, b FROM t1 WHERE b = 'a' ORDER BY a;" -1 2 TAIL]
+} {/^[0-9A-Fa-f]+$/}
+do_test 221 {
+  sqlite3_normalized_sql $STMT
+} {SELECT a,b FROM t1 WHERE b=?ORDER BY a;}
+do_test 222 {
+  sqlite3_finalize $STMT
+} {SQLITE_OK}
+
+do_test 297 {
+  execsql {
+    DROP TABLE t1;
+  }
+} {}
+do_test 298 {
+  execsql {
+    CREATE TABLE t1(a,b,c,d,e,"col f",w,x,y,z);
+    CREATE TABLE t2(x,"col y");
+  }
+} {}
+do_test 299 {
+  sqlite3_create_function db
+} {SQLITE_OK}
+
+foreach {tnum sql flags norm} {
+  300
+  {SELECT * FROM t1 WHERE a IN (1) AND b=51.42}
+  0x2
+  {0 {SELECT*FROM t1 WHERE a IN(?,?,?)AND b=?;}}
+
+  310
+  {SELECT a, b+15, c FROM t1 WHERE d NOT IN (SELECT x FROM t2);}
+  0x2
+  {0 {SELECT a,b+?,c FROM t1 WHERE d NOT IN(SELECT x FROM t2);}}
+
+  320
+  { SELECT NULL, b FROM t1 -- comment text
+     WHERE d IN (WITH t(a) AS (VALUES(5)) /* CTE */
+                 SELECT a FROM t)
+        OR e='hello';
+  }
+  0x2
+  {0 {SELECT?,b FROM t1 WHERE d IN(WITH t(a)AS(VALUES(?))SELECT a FROM t)OR e=?;}}
+
+  321
+  {/*Initial comment*/
+   -- another comment line
+   SELECT NULL  /* comment */ , b FROM t1 -- comment text
+     WHERE d IN (WITH t(a) AS (VALUES(5)) /* CTE */
+                 SELECT a FROM t)
+        OR e='hello';
+  }
+  0x2
+  {0 {SELECT?,b FROM t1 WHERE d IN(WITH t(a)AS(VALUES(?))SELECT a FROM t)OR e=?;}}
+
+  330
+  {/* Query containing parameters */
+   SELECT x,$::abc(15),y,@abc,z,?99,w FROM t1 /* Trailing comment */}
+  0x2
+  {0 {SELECT x,?,y,?,z,?,w FROM t1;}}
+
+  340
+  {/* Long list on the RHS of IN */
+   SELECT 15 IN (1,2,3,(SELECT * FROM t1),'xyz',x'abcd',22*(x+5),null);}
+  0x2
+  {1 {(1) no such column: x}}
+
+  350
+  {SELECT x'abc'; -- illegal token}
+  0x2
+  {1 {(1) unrecognized token: "x'abc'"}}
+
+  360
+  {SELECT a,NULL,b FROM t1 WHERE c IS NOT NULL or D is null or e=5}
+  0x2
+  {0 {SELECT a,?,b FROM t1 WHERE c IS NOT NULL OR d IS NULL OR e=?;}}
+
+  370
+  {/* IN list exactly 5 bytes long */
+   SELECT * FROM t1 WHERE x IN (1,2,3);}
+  0x2
+  {0 {SELECT*FROM t1 WHERE x IN(?,?,?);}}
+
+  400
+  {SELECT a FROM t1 WHERE x IN (1,2,3) AND sqlite_version();}
+  0x2
+  {0 {SELECT a FROM t1 WHERE x IN(?,?,?)AND sqlite_version();}}
+
+  410
+  {SELECT a FROM t1 WHERE x IN (1,2,3) AND hex8();}
+  0x2
+  {1 {(1) wrong number of arguments to function hex8()}}
+
+  420
+  {SELECT a FROM t1 WHERE x IN (1,2,3) AND hex8('abc');}
+  0x2
+  {0 {SELECT a FROM t1 WHERE x IN(?,?,?)AND hex8(?);}}
+
+  430
+  {SELECT "a" FROM t1 WHERE "x" IN ("1","2",'3');}
+  0x2
+  {0 {SELECT"a"FROM t1 WHERE"x"IN(?,?,?);}}
+
+  440
+  {SELECT 'a' FROM t1 WHERE 'x';}
+  0x2
+  {0 {SELECT?FROM t1 WHERE?;}}
+
+  450
+  {SELECT [a] FROM t1 WHERE [x];}
+  0x2
+  {0 {SELECT"a"FROM t1 WHERE"x";}}
+
+  460
+  {SELECT * FROM t1 WHERE x IN (x);}
+  0x2
+  {0 {SELECT*FROM t1 WHERE x IN(x);}}
+
+  470
+  {SELECT * FROM t1 WHERE x IN (x,a);}
+  0x2
+  {0 {SELECT*FROM t1 WHERE x IN(x,a);}}
+
+  480
+  {SELECT * FROM t1 WHERE x IN ([x],"a");}
+  0x2
+  {0 {SELECT*FROM t1 WHERE x IN("x","a");}}
+
+  500
+  {SELECT * FROM t1 WHERE x IN ([x],"a",'b',sqlite_version());}
+  0x2
+  {0 {SELECT*FROM t1 WHERE x IN("x","a",?,sqlite_version());}}
+
+  520
+  {SELECT * FROM t1 WHERE x IN (SELECT x FROM t1);}
+  0x2
+  {0 {SELECT*FROM t1 WHERE x IN(SELECT x FROM t1);}}
+
+  540
+  {SELECT * FROM t1 WHERE x IN ((SELECT x FROM t1));}
+  0x2
+  {0 {SELECT*FROM t1 WHERE x IN(?,?,?);}}
+
+  550
+  {SELECT a, a+1, a||'b', a+"b" FROM t1;}
+  0x2
+  {0 {SELECT a,a+?,a||?,a+"b"FROM t1;}}
+
+  570
+  {SELECT * FROM t1 WHERE x IN (1);}
+  0x2
+  {0 {SELECT*FROM t1 WHERE x IN(?,?,?);}}
+
+  580
+  {SELECT * FROM t1 WHERE x IN (1,2);}
+  0x2
+  {0 {SELECT*FROM t1 WHERE x IN(?,?,?);}}
+
+  590
+  {SELECT * FROM t1 WHERE x IN (1,2,3);}
+  0x2
+  {0 {SELECT*FROM t1 WHERE x IN(?,?,?);}}
+
+  600
+  {SELECT * FROM t1 WHERE x IN (1,2,3,4);}
+  0x2
+  {0 {SELECT*FROM t1 WHERE x IN(?,?,?);}}
+
+  610
+  {SELECT * FROM t1 WHERE x IN (SELECT x FROM t1);}
+  0x2
+  {0 {SELECT*FROM t1 WHERE x IN(SELECT x FROM t1);}}
+
+  620
+  {SELECT * FROM t1 WHERE x IN (SELECT x FROM t1 WHERE x IN (1,2,3));}
+  0x2
+  {0 {SELECT*FROM t1 WHERE x IN(SELECT x FROM t1 WHERE x IN(?,?,?));}}
+
+  630
+  {SELECT * FROM t1 WHERE x IN (SELECT x FROM t1 WHERE x IN (x));}
+  0x2
+  {0 {SELECT*FROM t1 WHERE x IN(SELECT x FROM t1 WHERE x IN(x));}}
+
+  640
+  {SELECT x FROM t1 WHERE x IN (SELECT x FROM t1 WHERE x IN (
+   SELECT x FROM t1 WHERE x IN (SELECT x FROM t1 WHERE x IN (
+   SELECT x FROM t1 WHERE x IN (x)))));}
+  0x2
+  {0 {SELECT x FROM t1 WHERE x IN(SELECT x FROM t1 WHERE x IN(SELECT x FROM t1 WHERE x IN(SELECT x FROM t1 WHERE x IN(SELECT x FROM t1 WHERE x IN(x)))));}}
+
+  650
+  {SELECT x FROM t1 WHERE x IN (SELECT x FROM t1 WHERE x IN (
+   SELECT x FROM t1 WHERE x IN (SELECT x FROM t1 WHERE x IN (
+   SELECT x FROM t1 WHERE x IN (1)))));}
+  0x2
+  {0 {SELECT x FROM t1 WHERE x IN(SELECT x FROM t1 WHERE x IN(SELECT x FROM t1 WHERE x IN(SELECT x FROM t1 WHERE x IN(SELECT x FROM t1 WHERE x IN(?,?,?)))));}}
+
+  660
+  {SELECT x FROM t1 WHERE x IN (1) UNION ALL SELECT x FROM t1 WHERE x IN (1);}
+  0x2
+  {0 {SELECT x FROM t1 WHERE x IN(?,?,?)UNION ALL SELECT x FROM t1 WHERE x IN(?,?,?);}}
+
+  670
+  {SELECT "col f", [col f] FROM t1;}
+  0x2
+  {0 {SELECT"col f","col f"FROM t1;}}
+
+  680
+  {SELECT a, "col f" FROM t1 LEFT OUTER JOIN t2 ON [t1].[col f] == [t2].[col y];}
+  0x2
+  {0 {SELECT a,"col f"FROM t1 LEFT OUTER JOIN t2 ON"t1"."col f"=="t2"."col y";}}
+
+  690
+  {SELECT * FROM ( WITH x AS ( SELECT * FROM t1 WHERE x IN ( 1)) SELECT 10);}
+  0x2
+  {0 {SELECT*FROM(WITH x AS(SELECT*FROM t1 WHERE x IN(?,?,?))SELECT?);}}
+
+  700
+  {SELECT rowid, oid, _rowid_ FROM t1;}
+  0x2
+  {0 {SELECT rowid,oid,_rowid_ FROM t1;}}
+
+  710
+  {SELECT x FROM t1 WHERE x IS NULL;}
+  0x2
+  {0 {SELECT x FROM t1 WHERE x IS NULL;}}
+
+  740
+  {SELECT x FROM t1 WHERE x IS NOT NULL;}
+  0x2
+  {0 {SELECT x FROM t1 WHERE x IS NOT NULL;}}
+
+  750
+  {SELECT x FROM t1 WHERE x = NULL;}
+  0x2
+  {0 {SELECT x FROM t1 WHERE x=?;}}
+
+  760
+  {SELECT x FROM t1 WHERE x IN ([x] IS NOT NULL, NULL, 1, 'a', "b", x'00');}
+  0x2
+  {0 {SELECT x FROM t1 WHERE x IN("x"IS NOT NULL,?,?,?,"b",?);}}
+} {
+  do_test $tnum {
+    set code [catch {
+      set STMT [sqlite3_prepare_v3 $DB $sql -1 $flags TAIL]
+      sqlite3_normalized_sql $STMT
+    } res]
+    if {[info exists STMT]} {
+      sqlite3_finalize $STMT; unset STMT
+    }
+    list $code $res
+  } $norm
+}
+}
+
 finish_test
-- 
2.39.5