#include "sqliteInt.h"
#include <stdlib.h>
-#define CC_X 0 /* The letter 'x' or 'X'. Start of x'01234fed' */
+ /* Character classes for tokenizing
+ **
+ ** In the sqlite3GetToken() function, a switch() on aiClass[c] is implemented
+ ** using a lookup table, whereas a switch() directly on c uses a binary search.
+ ** The lookup table is much faster. To maximize speed, and to ensure that
+ ** a lookup table is used, all of the classes need to be small integers and
+ ** all of them need to be used within the switch.
+ */
++#define CC_X 0 /* The letter 'x', or start of BLOB literal */
+ #define CC_KYWD 1 /* Alphabetics or '_'. Usable in a keyword */
+ #define CC_ID 2 /* unicode characters usable in IDs */
+ #define CC_DIGIT 3 /* Digits */
+ #define CC_DOLLAR 4 /* '$' */
+ #define CC_VARALPHA 5 /* '@', '#', ':'. Alphabetic SQL variables */
+ #define CC_VARNUM 6 /* '?'. Numeric SQL variables */
+ #define CC_SPACE 7 /* Space characters */
+ #define CC_QUOTE 8 /* '"', '\'', or '`'. String literals, quoted ids */
+ #define CC_QUOTE2 9 /* '['. [...] style quoted ids */
+ #define CC_PIPE 10 /* '|'. Bitwise OR or concatenate */
+ #define CC_MINUS 11 /* '-'. Minus or SQL-style comment */
+ #define CC_LT 12 /* '<'. Part of < or <= or <> */
+ #define CC_GT 13 /* '>'. Part of > or >= */
+ #define CC_EQ 14 /* '='. Part of = or == */
+ #define CC_BANG 15 /* '!'. Part of != */
+ #define CC_SLASH 16 /* '/'. / or c-style comment */
+ #define CC_LP 17 /* '(' */
+ #define CC_RP 18 /* ')' */
+ #define CC_SEMI 19 /* ';' */
+ #define CC_PLUS 20 /* '+' */
+ #define CC_STAR 21 /* '*' */
+ #define CC_PERCENT 22 /* '%' */
+ #define CC_COMMA 23 /* ',' */
+ #define CC_AND 24 /* '&' */
+ #define CC_TILDA 25 /* '~' */
+ #define CC_DOT 26 /* '.' */
+ #define CC_ILLEGAL 27 /* Illegal character */
+
+ static const unsigned char aiClass[] = {
+ #ifdef SQLITE_ASCII
+ /* x0 x1 x2 x3 x4 x5 x6 x7 x8 x9 xa xb xc xd xe xf */
+ /* 0x */ 27, 27, 27, 27, 27, 27, 27, 27, 27, 7, 7, 27, 7, 7, 27, 27,
+ /* 1x */ 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27,
+ /* 2x */ 7, 15, 8, 5, 4, 22, 24, 8, 17, 18, 21, 20, 23, 11, 26, 16,
+ /* 3x */ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 5, 19, 12, 14, 13, 6,
+ /* 4x */ 5, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+ /* 5x */ 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 9, 27, 27, 27, 1,
+ /* 6x */ 8, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+ /* 7x */ 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 27, 10, 27, 25, 27,
+ /* 8x */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
+ /* 9x */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
+ /* Ax */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
+ /* Bx */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
+ /* Cx */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
+ /* Dx */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
+ /* Ex */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
+ /* Fx */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2
+ #endif
+ #ifdef SQLITE_EBCDIC
+ /* x0 x1 x2 x3 x4 x5 x6 x7 x8 x9 xa xb xc xd xe xf */
+ /* 0x */ 27, 27, 27, 27, 27, 7, 27, 27, 27, 27, 27, 27, 7, 7, 27, 27,
+ /* 1x */ 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27,
+ /* 2x */ 27, 27, 27, 27, 27, 7, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27,
+ /* 3x */ 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27,
+ /* 4x */ 7, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 12, 17, 20, 10,
+ /* 5x */ 24, 27, 27, 27, 27, 27, 27, 27, 27, 27, 15, 4, 21, 18, 19, 27,
+ /* 6x */ 11, 16, 27, 27, 27, 27, 27, 27, 27, 27, 27, 23, 22, 1, 13, 7,
+ /* 7x */ 27, 27, 27, 27, 27, 27, 27, 27, 27, 8, 5, 5, 5, 8, 14, 8,
+ /* 8x */ 27, 1, 1, 1, 1, 1, 1, 1, 1, 1, 27, 27, 27, 27, 27, 27,
+ /* 9x */ 27, 1, 1, 1, 1, 1, 1, 1, 1, 1, 27, 27, 27, 27, 27, 27,
+ /* 9x */ 25, 1, 1, 1, 1, 1, 1, 0, 1, 1, 27, 27, 27, 27, 27, 27,
+ /* Bx */ 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 9, 27, 27, 27, 27, 27,
+ /* Cx */ 27, 1, 1, 1, 1, 1, 1, 1, 1, 1, 27, 27, 27, 27, 27, 27,
+ /* Dx */ 27, 1, 1, 1, 1, 1, 1, 1, 1, 1, 27, 27, 27, 27, 27, 27,
+ /* Ex */ 27, 27, 1, 1, 1, 1, 1, 0, 1, 1, 27, 27, 27, 27, 27, 27,
+ /* Fx */ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 27, 27, 27, 27, 27, 27,
+ #endif
+ };
+
/*
- ** The charMap() macro maps alphabetic characters into their
+ ** The charMap() macro maps alphabetic characters (only) into their
** lower-case ASCII equivalent. On ASCII machines, this is just
** an upper-to-lower case map. On EBCDIC machines we also need
- ** to adjust the encoding. Only alphabetic characters and underscores
- ** need to be translated.
+ ** to adjust the encoding. The mapping is only valid for alphabetics
+ ** which are the only characters for which this feature is used.
+ **
+ ** Used by keywordhash.h
*/
#ifdef SQLITE_ASCII
# define charMap(X) sqlite3UpperToLower[(unsigned char)X]
** returned. If the input is not a keyword, TK_ID is returned.
**
** The implementation of this routine was generated by a program,
--** mkkeywordhash.h, located in the tool subdirectory of the distribution.
++** mkkeywordhash.c, located in the tool subdirectory of the distribution.
** The output of the mkkeywordhash.c program is written into a file
** named keywordhash.h and then included into this source file by
** the #include below.
/*
--** Return the length of the token that begins at z[0].
++** Return the length (in bytes) of the token that begins at z[0].
** Store the token type in *tokenType before returning.
*/
int sqlite3GetToken(const unsigned char *z, int *tokenType){
int i, c;
- switch( *z ){
- case ' ': case '\t': case '\n': case '\f': case '\r': {
- switch( aiClass[*z] ){
++ switch( aiClass[*z] ){ /* Switch on the character-class of the first byte
++ ** of the token. See the comment on the CC_ defines
++ ** above. */
+ case CC_SPACE: {
testcase( z[0]==' ' );
testcase( z[0]=='\t' );
testcase( z[0]=='\n' );
if( n==0 ) *tokenType = TK_ILLEGAL;
return i;
}
++ case CC_KYWD: {
++ for(i=1; aiClass[z[i]]<=CC_KYWD; i++){}
++ if( IdChar(z[i]) ){
++ /* This token started out using characters that can appear in keywords,
++ ** but z[i] is a character not allowed within keywords, so this must
++ ** be an identifier instead */
++ i++;
++ break;
++ }
++ *tokenType = TK_ID;
++ return keywordCode((char*)z, i, tokenType);
++ }
#ifndef SQLITE_OMIT_BLOB_LITERAL
- case 'x': case 'X': {
+ case CC_X: {
testcase( z[0]=='x' ); testcase( z[0]=='X' );
if( z[1]=='\'' ){
*tokenType = TK_BLOB;
if( z[i] ) i++;
return i;
}
- /* Otherwise fall through to the next case */
- i = 1;
- break;
++ /* If it is not a BLOB literal, then it must be an ID, since no
++ ** SQL keywords start with the letter 'x'. Fall through */
}
#endif
- case CC_KYWD: {
- for(i=1; aiClass[z[i]]<=CC_KYWD; i++){}
- if( IdChar(z[i]) ){ i++; break; }
- *tokenType = TK_ID;
- return keywordCode((char*)z, i, tokenType);
- }
+ case CC_ID: {
+ i = 1;
+ break;
+ }
default: {
- if( !IdChar(*z) ){
- break;
- }
- for(i=1; IdChar(z[i]); i++){}
- *tokenType = TK_ID;
- return keywordCode((char*)z, i, tokenType);
+ *tokenType = TK_ILLEGAL;
+ return 1;
}
}
- *tokenType = TK_ILLEGAL;
- return 1;
+ while( IdChar(z[i]) ){ i++; }
+ *tokenType = TK_ID;
+ return i;
}
/*
projects / thirdparty / sqlite.git / commitdiff
