Merge the tokenizer buffer overrun bug-fix from trunk.

FossilOrigin-Name: 82e7e31b09097b24876f69df56cdd8f4c1aad1f1

diff --cc manifest
index d63e11ed1d5ee56b4ba585500dd31f1efd182395,6a10514189b048c27794bf20590c5700546e1265..d511052fea36434c13133bcefa6543ccd1f4eb39
--- 1/manifest
--- 2/manifest
+++ b/manifest
@@@ -1,6 -1,6 +1,6 @@@
- C Merge\srecent\sperformance\senhancements\sand\sthe\senhanced\sAPI_ARMOR\sfrom\strunk.
- D 2016-02-18T01:36:47.227
 -C Avoid\sa\spotential\sbuffer\soverrun\sif\san\sSQL\sstatement\sbeing\sparsed\sends\nwith\san\sillegal\s"!"\stoken.\s\s(This\sproblem\swas\sdetected\sby\sfuzzcheck\nrunning\sunder\svalgrind.\sThe\sproblem\swas\sintroduced\sby\scheck-in\s[9570b6b43df3].)
 -D 2016-02-18T14:49:28.741
 -F Makefile.in 4e90dc1521879022aa9479268a4cd141d1771142
++C Merge\sthe\stokenizer\sbuffer\soverrun\sbug-fix\sfrom\strunk.
++D 2016-02-18T14:53:52.680
 +F Makefile.in 6bc82e8fca16b46e6158b5614f16b68cc42cbe5d
  F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
  F Makefile.msc 30f075dc4f27a07abb76088946b2944178d85347
  F README.md 8ecc12493ff9f820cdea6520a9016001cb2e59b7
@@@ -855,13 -851,12 +855,13 @@@ F test/lock.test b984ab9034e7389be0d863
  F test/lock2.test 5242d8ac4e2d59c403aebff606af449b455aceff
  F test/lock3.test f271375930711ae044080f4fe6d6eda930870d00
  F test/lock4.test e175ae13865bc87680607563bafba21f31a26f12
 -F test/lock5.test c6c5e0ebcb21c61a572870cc86c0cb9f14cede38
 -F test/lock6.test ad5b387a3a8096afd3c68a55b9535056431b0cf5
 +F test/lock5.test 1f1e0fd1670deef8c0a76e8f895bd5e124133ca4
 +F test/lock6.test 83434ae8ca1d1c5e2eaf74d4e44aa24ab62b291c
  F test/lock7.test 49f1eaff1cdc491cc5dee3669f3c671d9f172431
  F test/lock_common.tcl 7ffb45accf6ee91c736df9bafe0806a44358f035
 +F test/lock_proxy.test 95be9c32d79be25cf643b4e41a0aa0e53aa21621
  F test/lookaside.test 90052e87282de256d613fcf8c9cbb845e4001d2f
- F test/main.test 3595944ab7fbb3637a499d3eff2b60c89e26adbb
 -F test/main.test bb75e406c9b64931f3dc7e7f04626633365bb22f
++F test/main.test 5c893511b64780c8b96563664f8b8d7d959911af
  F test/make-where7.tcl 05c16b5d4f5d6512881dfec560cb793915932ef9
  F test/malloc.test 21c213365f2cca95ab2d7dc078dc8525f96065f8
  F test/malloc3.test e3b32c724b5a124b57cb0ed177f675249ad0c66a
@@@ -1433,7 -1428,7 +1433,7 @@@ F tool/vdbe_profile.tcl 246d0da094856d7
  F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
  F tool/warnings.sh a98af506df552f3b3c0d904f94e4cdc4e1a6d598
  F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
- P 7bcd3ab5f0519cb9ac8f7b98827600b9eff8db85 80de1ad5eb4c3839756b6467d5b8195b4abfb1f8
- R 0601a63da78c9a4a9f03ea7945a41623
 -P c8cd7804dc905b2b20cd7c0192bcfaceaaa7e2a8
 -R ff7407a00ef53a788829701876392bcd
++P b7fa0fb3399ad2bf1e6bc4c92d43278a40f19ec4 2a8d97e7c8976df0312e1294e8c1da8b15686654
++R 3b3ae438d780007f8a2e7756894326a0
  U drh
- Z a1a7a2a2726e6248db988b26a804bb5c
 -Z e801ae846fbe7280611432828d271852
++Z cfd5f751bd276f78bb0a099814d00ed5

diff --cc manifest.uuid
index df4022b8a1018f9fab8c923fb5adc386bb2db404,a9ce1d047c27a0af6858c0b4cbf5b33aa40416c9..207934484f49754c83b5690e3b4931e8468eec05
--- 1/manifest.uuid
--- 2/manifest.uuid
+++ b/manifest.uuid
@@@ -1,1 -1,1 +1,1 @@@
- b7fa0fb3399ad2bf1e6bc4c92d43278a40f19ec4
 -2a8d97e7c8976df0312e1294e8c1da8b15686654
++82e7e31b09097b24876f69df56cdd8f4c1aad1f1

diff --cc test/main.test
index b324e11ddd97b0ea8cf666ad85a58de7b47f5d41,9346cf6ced4e5f6bd51963962225888f2ad4aaa5..e4013b7e7ccd81c2537e910a95a34ea707b39311
--- 1/test/main.test
--- 2/test/main.test
+++ b/test/main.test
@@@ -319,10 -319,10 +319,10 @@@ do_test main-3.1 
    sqlite3 db testdb
    set v [catch {execsql {SELECT * from T1 where x!!5}} msg]
    lappend v $msg
- } {1 {unrecognized token: "!!"}}
+ } {1 {unrecognized token: "!"}}
  do_test main-3.2 {
    catch {db close}
 -  foreach f [glob -nocomplain testdb/*] {forcedelete $f}
 +  catch {foreach f [glob -nocomplain testdb/*] {forcedelete $f}}
    forcedelete testdb
    sqlite3 db testdb
    set v [catch {execsql {SELECT * from T1 where ^x}} msg]