The tokenizer should never return a negative size of the next token.

Ticket #453. (CVS 1098)

FossilOrigin-Name: 4fbca3ab09596c530da7c50657f3bc9140178dd5

diff --git a/manifest b/manifest
index d6f9450600efecf6ffa05a524d2f1d83e4d2ec95..a86fd16edf936fe097b4687ce71f19128f9e0344 100644 (file)
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Add\stest\scases\sthat\scheck\sout\ssqlite_get_table()\son\squeries\swith\sresults\ssets\ncontaining\s100\scolumns.\s(CVS\s1097)
-D 2003-09-09T00:47:47
+C The\stokenizer\sshould\snever\sreturn\sa\snegative\ssize\sof\sthe\snext\stoken.\nTicket\s#453.\s(CVS\s1098)
+D 2003-09-12T02:08:15
 F Makefile.in 0cf2ffb6dc35694895e0dac488bc1259b6a4eb90
 F Makefile.linux-gcc b86a99c493a5bfb402d1d9178dcdc4bd4b32f906
 F README f1de682fbbd94899d50aca13d387d1b3fd3be2dd
@@ -56,7 +56,7 @@ F src/test1.c f9d5816610f7ec4168ab7b098d5207a5708712b6
 F src/test2.c 5014337d8576b731cce5b5a14bec4f0daf432700
 F src/test3.c 30985ebdfaf3ee1462a9b0652d3efbdc8d9798f5
 F src/threadtest.c d641a5219e718e18a1a80a50eb9bb549f451f42e
-F src/tokenize.c 74152bde07da0623aaa60fb6ab71d5af1c035546
+F src/tokenize.c 8df640021c7a18f75411a8fd25adbb76bf528e7e
 F src/trigger.c 474581eaab388233df01bb019e558af2965decbf
 F src/update.c 24260b4fda00c9726d27699a0561d53c0dccc397
 F src/util.c f16efa2d60bfd4e31ae06b07ed149557e828d294
@@ -102,7 +102,7 @@ F test/memdb.test 6ece25c7c0e6500199d3662607a3edca081abb2a
 F test/memleak.test a18e6810cae96d2f6f5136920267adbefc8e1e90
 F test/minmax.test 6d9b6d6ee34f42e2a58dffece1f76d35f446b3af
 F test/misc1.test 0b98d493b0cf55cb5f53e1f3df8107c166eecb5a
-F test/misc2.test d7bc75fae9157c4a3f4914b505713f398b3c5422
+F test/misc2.test 5818bfafd07535b0437f15c32bed983f3bd363b2
 F test/misuse.test a3aa2b18a97e4c409a1fcaff5151a4dd804a0162
 F test/notnull.test 7a08117a71e74b0321aaa937dbeb41a09d6eb1d0
 F test/null.test 5c2b57307e4b6178aae825eb65ddbee01e76b0fd
@@ -173,7 +173,7 @@ F www/speed.tcl 2f6b1155b99d39adb185f900456d1d592c4832b3
 F www/sqlite.tcl 3c83b08cf9f18aa2d69453ff441a36c40e431604
 F www/tclsqlite.tcl b9271d44dcf147a93c98f8ecf28c927307abd6da
 F www/vdbe.tcl 9b9095d4495f37697fd1935d10e14c6015e80aa1
-P f6c4908e9b5b6ac9adc4af50dc5110dbb655dae3
-R f7938d35621546ffe8cfa85d2210afbb
+P f9feedec32c975235c0d0996962033f6a63312ba
+R f36ba4593d56d9555b41bc6f7efad70c
 U drh
-Z d4864a97177572787e9b3823564d9aff
+Z 3e7100d230b7f9eaf7343d1e1cea1236

diff --git a/manifest.uuid b/manifest.uuid
index 96bd626f025d4ca7276ce4a552d4bd61442ddef4..3a04a8f396109b58784b5f243fe579784987e40f 100644 (file)
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-f9feedec32c975235c0d0996962033f6a63312ba
\ No newline at end of file
+4fbca3ab09596c530da7c50657f3bc9140178dd5
\ No newline at end of file

diff --git a/src/tokenize.c b/src/tokenize.c
index 24433a07faa82e1e0b7364e1622c40914d92d711..f99cb7131f88b9758ec7dc306dff5a6873da06ba 100644 (file)
--- a/src/tokenize.c
+++ b/src/tokenize.c
@@ -15,7 +15,7 @@
 ** individual tokens and sends those tokens one-by-one over to the
 ** parser for analysis.
 **
-** $Id: tokenize.c,v 1.62 2003/09/06 22:18:08 drh Exp $
+** $Id: tokenize.c,v 1.63 2003/09/12 02:08:15 drh Exp $
 */
 #include "sqliteInt.h"
 #include "os.h"
@@ -214,9 +214,8 @@ static const char isIdChar[] = {
 
 
 /*
-** Return the length of the token that begins at z[0].  Return
-** -1 if the token is (or might be) incomplete.  Store the token
-** type in *tokenType before returning.
+** Return the length of the token that begins at z[0]. 
+** Store the token type in *tokenType before returning.
 */
 static int sqliteGetToken(const unsigned char *z, int *tokenType){
   int i;
@@ -227,7 +226,6 @@ static int sqliteGetToken(const unsigned char *z, int *tokenType){
       return i;
     }
     case '-': {
-      if( z[1]==0 ) return -1;
       if( z[1]=='-' ){
         for(i=2; z[i] && z[i]!='\n'; i++){}
         *tokenType = TK_COMMENT;
@@ -426,7 +424,6 @@ int sqliteRunParser(Parse *pParse, const char *zSql, char **pzErrMsg){
   pParse->sLastToken.dyn = 0;
   pParse->zTail = zSql;
   while( sqlite_malloc_failed==0 && zSql[i]!=0 ){
-    
     assert( i>=0 );
     pParse->sLastToken.z = &zSql[i];
     assert( pParse->sLastToken.dyn==0 );

diff --git a/test/misc2.test b/test/misc2.test
index 60acb05c37358c8c51e2cb2e8a81d67a729907be..24b22b2e7031db7bfa37a2623e32f63514263080 100644 (file)
--- a/test/misc2.test
+++ b/test/misc2.test
@@ -13,7 +13,7 @@
 # This file implements tests for miscellanous features that were
 # left out of other test files.
 #
-# $Id: misc2.test,v 1.8 2003/08/27 22:54:32 drh Exp $
+# $Id: misc2.test,v 1.9 2003/09/12 02:08:16 drh Exp $
 
 set testdir [file dirname $argv0]
 source $testdir/tester.tcl
@@ -179,3 +179,11 @@ do_test misc2-7.4 {
   } msg]
   lappend rc $msg
 } {1 {database table is locked}}
+
+# Ticket #453.  If the SQL ended with "-", the tokenizer was calling that
+# an incomplete token, which caused problem.  The solution was to just call
+# it a minus sign.
+#
+do_test misc2-8.1 {
+  catchsql {-}
+} {1 {near "-": syntax error}}