** individual tokens and sends those tokens one-by-one over to the
** parser for analysis.
**
-** $Id: tokenize.c,v 1.93 2004/10/31 02:22:49 drh Exp $
+** $Id: tokenize.c,v 1.94 2004/11/03 03:59:58 drh Exp $
*/
#include "sqliteInt.h"
#include "os.h"
#include <stdlib.h>
/*
-** This function looks up an identifier to determine if it is a
-** keyword. If it is a keyword, the token code of that keyword is
+** The sqlite3KeywordCode function looks up an identifier to determine if
+** it is a keyword. If it is a keyword, the token code of that keyword is
** returned. If the input is not a keyword, TK_ID is returned.
**
** The implementation of this routine was generated by a program,
** mkkeywordhash.c, located in the tool subdirectory of the distribution.
-** The output of the mkkeywordhash.c program was manually cut and pasted
-** into this file. When the set of keywords for SQLite changes, you
-** must modify the mkkeywordhash.c program (to add or remove keywords from
-** the data tables) then rerun that program to regenerate this function.
+** The output of the mkkeywordhash.c program is written into a file
+** named keywordhash.c and then included into this source file by
+** the #include below.
*/
-int sqlite3KeywordCode(const char *z, int n){
- static const char zText[443] =
- "ABORTABLEFTEMPORARYAFTERAISELECTHENDATABASEACHECKEYANDEFAULTRANSACTION"
- "ATURALIKELSEXCEPTRIGGEREFERENCESTATEMENTATTACHAVINGLOBEFOREIGN"
- "OREPLACEXCLUSIVEXPLAINDEXBEGINITIALLYBETWEENOTNULLIMITBYCASCADE"
- "FERRABLECASECOLLATECOMMITCONFLICTCONSTRAINTERSECTCREATECROSSDEFERRED"
- "ELETEDESCDETACHDISTINCTDROPRAGMATCHFAILFROMFULLGROUPDATEIMMEDIATE"
- "INNERESTRICTINSERTINSTEADINTOFFSETISNULLJOINORDERIGHTOUTEROLLBACK"
- "PRIMARYROWHENUNIONUNIQUEUSINGVACUUMVALUESVIEWHERE";
- static const unsigned char aHash[154] = {
- 0, 26, 82, 0, 0, 91, 90, 0, 27, 0, 0, 0, 0,
- 0, 0, 49, 0, 96, 17, 0, 0, 0, 0, 0, 0, 0,
- 0, 97, 5, 31, 0, 62, 51, 28, 58, 52, 0, 0, 60,
- 61, 0, 12, 41, 50, 0, 0, 0, 36, 63, 0, 0, 15,
- 0, 0, 0, 39, 0, 42, 0, 0, 0, 0, 78, 0, 34,
- 29, 0, 74, 71, 0, 66, 70, 37, 0, 0, 59, 0, 33,
- 0, 53, 0, 54, 0, 55, 0, 83, 72, 67, 0, 24, 0,
- 0, 79, 80, 84, 0, 0, 0, 0, 0, 0, 0, 75, 0,
- 0, 0, 0, 0, 45, 77, 35, 44, 57, 0, 0, 0, 0,
- 20, 2, 0, 38, 0, 3, 46, 93, 0, 0, 40, 0, 94,
- 0, 43, 87, 98, 0, 0, 0, 0, 0, 81, 0, 0, 0,
- 0, 10, 0, 0, 0, 0, 0, 92, 19, 0, 95,
- };
- static const unsigned char aNext[98] = {
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7,
- 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0,
- 0, 0, 0, 0, 0, 18, 22, 0, 0, 0, 0, 0, 0,
- 0, 23, 0, 16, 21, 8, 0, 32, 0, 0, 30, 0, 48,
- 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0,
- 0, 56, 0, 1, 0, 69, 64, 0, 0, 65, 0, 0, 13,
- 68, 0, 0, 76, 47, 0, 0, 0, 85, 6, 0, 89, 25,
- 4, 73, 88, 86, 0, 0, 0,
- };
- static const unsigned char aLen[98] = {
- 5, 5, 4, 4, 9, 2, 5, 5, 6, 4, 3, 8, 2,
- 4, 5, 3, 3, 7, 11, 2, 7, 4, 4, 6, 7, 10,
- 9, 6, 6, 4, 6, 3, 7, 6, 7, 9, 7, 5, 5,
- 9, 3, 7, 3, 7, 4, 5, 2, 7, 3, 10, 4, 7,
- 6, 8, 10, 2, 9, 6, 5, 8, 6, 4, 6, 8, 2,
- 4, 6, 5, 4, 4, 4, 5, 6, 9, 5, 8, 6, 7,
- 4, 2, 6, 3, 6, 4, 5, 5, 5, 8, 7, 3, 4,
- 5, 6, 5, 6, 6, 4, 5,
- };
- static const unsigned short int aOffset[98] = {
- 0, 4, 7, 10, 10, 14, 19, 23, 26, 31, 33, 35, 40,
- 42, 44, 48, 51, 53, 59, 68, 69, 75, 78, 81, 86, 92,
- 101, 110, 115, 120, 123, 125, 125, 129, 133, 139, 147, 152, 157,
- 160, 165, 169, 175, 175, 178, 181, 186, 188, 189, 193, 203, 207,
- 214, 220, 228, 235, 235, 244, 250, 255, 262, 268, 272, 278, 279,
- 286, 289, 293, 298, 302, 306, 310, 313, 319, 328, 332, 340, 346,
- 353, 356, 356, 359, 362, 368, 372, 376, 381, 385, 393, 400, 402,
- 406, 411, 417, 422, 428, 434, 437,
- };
- static const unsigned char aCode[98] = {
- TK_ABORT, TK_TABLE, TK_JOIN_KW, TK_TEMP, TK_TEMP,
- TK_OR, TK_AFTER, TK_RAISE, TK_SELECT, TK_THEN,
- TK_END, TK_DATABASE, TK_AS, TK_EACH, TK_CHECK,
- TK_KEY, TK_AND, TK_DEFAULT, TK_TRANSACTION,TK_ON,
- TK_JOIN_KW, TK_LIKE, TK_ELSE, TK_EXCEPT, TK_TRIGGER,
- TK_REFERENCES, TK_STATEMENT, TK_ATTACH, TK_HAVING, TK_GLOB,
- TK_BEFORE, TK_FOR, TK_FOREIGN, TK_IGNORE, TK_REPLACE,
- TK_EXCLUSIVE, TK_EXPLAIN, TK_INDEX, TK_BEGIN, TK_INITIALLY,
- TK_ALL, TK_BETWEEN, TK_NOT, TK_NOTNULL, TK_NULL,
- TK_LIMIT, TK_BY, TK_CASCADE, TK_ASC, TK_DEFERRABLE,
- TK_CASE, TK_COLLATE, TK_COMMIT, TK_CONFLICT, TK_CONSTRAINT,
- TK_IN, TK_INTERSECT, TK_CREATE, TK_JOIN_KW, TK_DEFERRED,
- TK_DELETE, TK_DESC, TK_DETACH, TK_DISTINCT, TK_IS,
- TK_DROP, TK_PRAGMA, TK_MATCH, TK_FAIL, TK_FROM,
- TK_JOIN_KW, TK_GROUP, TK_UPDATE, TK_IMMEDIATE, TK_JOIN_KW,
- TK_RESTRICT, TK_INSERT, TK_INSTEAD, TK_INTO, TK_OF,
- TK_OFFSET, TK_SET, TK_ISNULL, TK_JOIN, TK_ORDER,
- TK_JOIN_KW, TK_JOIN_KW, TK_ROLLBACK, TK_PRIMARY, TK_ROW,
- TK_WHEN, TK_UNION, TK_UNIQUE, TK_USING, TK_VACUUM,
- TK_VALUES, TK_VIEW, TK_WHERE,
- };
- int h, i;
- if( n<2 ) return TK_ID;
- h = (sqlite3UpperToLower[((unsigned char*)z)[0]]*5 +
- sqlite3UpperToLower[((unsigned char*)z)[n-1]]*3 +
- n) % 154;
- for(i=((int)aHash[h])-1; i>=0; i=((int)aNext[i])-1){
- if( aLen[i]==n && sqlite3StrNICmp(&zText[aOffset[i]],z,n)==0 ){
- return aCode[i];
- }
- }
- return TK_ID;
-}
+#include "keywordhash.c"
/*
struct Keyword {
char *zName; /* The keyword name */
char *zTokenType; /* Token value for this keyword */
+ int mask; /* Code this keyword if non-zero */
int id; /* Unique ID for this record */
int hash; /* Hash on the keyword */
int offset; /* Offset to start of name string */
int substrOffset; /* Offset into substrId for start of this keyword */
};
+/*
+** Define masks used to determine which keywords are allowed
+*/
+#ifdef SQLITE_OMIT_ALTER_TABLE
+# define ALTER 0
+#else
+# define ALTER 1
+#endif
+#define ALWAYS 2
+#ifdef SQLITE_OMIT_ATTACH
+# define ATTACH 0
+#else
+# define ATTACH 4
+#endif
+#ifdef SQLITE_OMIT_AUTOINCREMENT
+# define AUTOINCR 0
+#else
+# define AUTOINCR 8
+#endif
+#ifdef SQLITE_OMIT_COMPOUND_SELECT
+# define COMPOUND 0
+#else
+# define COMPOUND 16
+#endif
+#ifdef SQLITE_OMIT_CONFLICT_CLAUSE
+# define CONFLICT 0
+#else
+# define CONFLICT 32
+#endif
+#ifdef SQLITE_OMIT_EXPLAIN
+# define EXPLAIN 0
+#else
+# define EXPLAIN 64
+#endif
+#ifdef SQLITE_OMIT_FOREIGN_KEY
+# define FKEY 0
+#else
+# define FKEY 128
+#endif
+#ifdef SQLITE_OMIT_PRAGMA
+# define PRAGMA 0
+#else
+# define PRAGMA 256
+#endif
+#ifdef SQLITE_OMIT_REINDEX
+# define REINDEX 0
+#else
+# define REINDEX 512
+#endif
+#ifdef SQLITE_OMIT_TRIGGER
+# define TRIGGER 0
+#else
+# define TRIGGER 1024
+#endif
+#ifdef SQLITE_OMIT_VACUUM
+# define VACUUM 0
+#else
+# define VACUUM 2048
+#endif
+#ifdef SQLITE_OMIT_VIEW
+# define VIEW 0
+#else
+# define VIEW 4096
+#endif
+
+
/*
** These are the keywords
*/
static Keyword aKeywordTable[] = {
- { "ABORT", "TK_ABORT", },
- { "AFTER", "TK_AFTER", },
- { "ALL", "TK_ALL", },
- { "AND", "TK_AND", },
- { "AS", "TK_AS", },
- { "ASC", "TK_ASC", },
- { "ATTACH", "TK_ATTACH", },
- { "BEFORE", "TK_BEFORE", },
- { "BEGIN", "TK_BEGIN", },
- { "BETWEEN", "TK_BETWEEN", },
- { "BY", "TK_BY", },
- { "CASCADE", "TK_CASCADE", },
- { "CASE", "TK_CASE", },
- { "CHECK", "TK_CHECK", },
- { "COLLATE", "TK_COLLATE", },
- { "COMMIT", "TK_COMMIT", },
- { "CONFLICT", "TK_CONFLICT", },
- { "CONSTRAINT", "TK_CONSTRAINT", },
- { "CREATE", "TK_CREATE", },
- { "CROSS", "TK_JOIN_KW", },
- { "DATABASE", "TK_DATABASE", },
- { "DEFAULT", "TK_DEFAULT", },
- { "DEFERRED", "TK_DEFERRED", },
- { "DEFERRABLE", "TK_DEFERRABLE", },
- { "DELETE", "TK_DELETE", },
- { "DESC", "TK_DESC", },
- { "DETACH", "TK_DETACH", },
- { "DISTINCT", "TK_DISTINCT", },
- { "DROP", "TK_DROP", },
- { "END", "TK_END", },
- { "EACH", "TK_EACH", },
- { "ELSE", "TK_ELSE", },
- { "EXCEPT", "TK_EXCEPT", },
- { "EXCLUSIVE", "TK_EXCLUSIVE", },
- { "EXPLAIN", "TK_EXPLAIN", },
- { "FAIL", "TK_FAIL", },
- { "FOR", "TK_FOR", },
- { "FOREIGN", "TK_FOREIGN", },
- { "FROM", "TK_FROM", },
- { "FULL", "TK_JOIN_KW", },
- { "GLOB", "TK_GLOB", },
- { "GROUP", "TK_GROUP", },
- { "HAVING", "TK_HAVING", },
- { "IGNORE", "TK_IGNORE", },
- { "IMMEDIATE", "TK_IMMEDIATE", },
- { "IN", "TK_IN", },
- { "INDEX", "TK_INDEX", },
- { "INITIALLY", "TK_INITIALLY", },
- { "INNER", "TK_JOIN_KW", },
- { "INSERT", "TK_INSERT", },
- { "INSTEAD", "TK_INSTEAD", },
- { "INTERSECT", "TK_INTERSECT", },
- { "INTO", "TK_INTO", },
- { "IS", "TK_IS", },
- { "ISNULL", "TK_ISNULL", },
- { "JOIN", "TK_JOIN", },
- { "KEY", "TK_KEY", },
- { "LEFT", "TK_JOIN_KW", },
- { "LIKE", "TK_LIKE", },
- { "LIMIT", "TK_LIMIT", },
- { "MATCH", "TK_MATCH", },
- { "NATURAL", "TK_JOIN_KW", },
- { "NOT", "TK_NOT", },
- { "NOTNULL", "TK_NOTNULL", },
- { "NULL", "TK_NULL", },
- { "OF", "TK_OF", },
- { "OFFSET", "TK_OFFSET", },
- { "ON", "TK_ON", },
- { "OR", "TK_OR", },
- { "ORDER", "TK_ORDER", },
- { "OUTER", "TK_JOIN_KW", },
- { "PRAGMA", "TK_PRAGMA", },
- { "PRIMARY", "TK_PRIMARY", },
- { "RAISE", "TK_RAISE", },
- { "REFERENCES", "TK_REFERENCES", },
- { "REPLACE", "TK_REPLACE", },
- { "RESTRICT", "TK_RESTRICT", },
- { "RIGHT", "TK_JOIN_KW", },
- { "ROLLBACK", "TK_ROLLBACK", },
- { "ROW", "TK_ROW", },
- { "SELECT", "TK_SELECT", },
- { "SET", "TK_SET", },
- { "STATEMENT", "TK_STATEMENT", },
- { "TABLE", "TK_TABLE", },
- { "TEMP", "TK_TEMP", },
- { "TEMPORARY", "TK_TEMP", },
- { "THEN", "TK_THEN", },
- { "TRANSACTION", "TK_TRANSACTION", },
- { "TRIGGER", "TK_TRIGGER", },
- { "UNION", "TK_UNION", },
- { "UNIQUE", "TK_UNIQUE", },
- { "UPDATE", "TK_UPDATE", },
- { "USING", "TK_USING", },
- { "VACUUM", "TK_VACUUM", },
- { "VALUES", "TK_VALUES", },
- { "VIEW", "TK_VIEW", },
- { "WHEN", "TK_WHEN", },
- { "WHERE", "TK_WHERE", },
+ { "ABORT", "TK_ABORT", CONFLICT|TRIGGER },
+ { "AFTER", "TK_AFTER", TRIGGER },
+ { "ALL", "TK_ALL", ALWAYS },
+ { "ALTER", "TK_ALTER", ALTER },
+ { "AND", "TK_AND", ALWAYS },
+ { "AS", "TK_AS", ALWAYS },
+ { "ASC", "TK_ASC", ALWAYS },
+ { "ATTACH", "TK_ATTACH", ATTACH },
+ { "AUTOINCREMENT", "TK_AUTOINCR", AUTOINCR },
+ { "BEFORE", "TK_BEFORE", TRIGGER },
+ { "BEGIN", "TK_BEGIN", ALWAYS },
+ { "BETWEEN", "TK_BETWEEN", ALWAYS },
+ { "BY", "TK_BY", ALWAYS },
+ { "CASCADE", "TK_CASCADE", FKEY },
+ { "CASE", "TK_CASE", ALWAYS },
+ { "CHECK", "TK_CHECK", ALWAYS },
+ { "COLLATE", "TK_COLLATE", ALWAYS },
+ { "COMMIT", "TK_COMMIT", ALWAYS },
+ { "CONFLICT", "TK_CONFLICT", CONFLICT },
+ { "CONSTRAINT", "TK_CONSTRAINT", ALWAYS },
+ { "CREATE", "TK_CREATE", ALWAYS },
+ { "CROSS", "TK_JOIN_KW", ALWAYS },
+ { "CURRENT_DATE", "TK_CDATE", ALWAYS },
+ { "CURRENT_TIME", "TK_CTIME", ALWAYS },
+ { "CURRENT_TIMESTAMP","TK_CTIMESTAMP", ALWAYS },
+ { "DATABASE", "TK_DATABASE", ATTACH },
+ { "DEFAULT", "TK_DEFAULT", ALWAYS },
+ { "DEFERRED", "TK_DEFERRED", FKEY },
+ { "DEFERRABLE", "TK_DEFERRABLE", FKEY },
+ { "DELETE", "TK_DELETE", ALWAYS },
+ { "DESC", "TK_DESC", ALWAYS },
+ { "DETACH", "TK_DETACH", ATTACH },
+ { "DISTINCT", "TK_DISTINCT", ALWAYS },
+ { "DROP", "TK_DROP", ALWAYS },
+ { "END", "TK_END", ALWAYS },
+ { "EACH", "TK_EACH", TRIGGER },
+ { "ELSE", "TK_ELSE", ALWAYS },
+ { "EXCEPT", "TK_EXCEPT", COMPOUND },
+ { "EXCLUSIVE", "TK_EXCLUSIVE", ALWAYS },
+ { "EXPLAIN", "TK_EXPLAIN", EXPLAIN },
+ { "FAIL", "TK_FAIL", CONFLICT|TRIGGER },
+ { "FOR", "TK_FOR", TRIGGER },
+ { "FOREIGN", "TK_FOREIGN", FKEY },
+ { "FROM", "TK_FROM", ALWAYS },
+ { "FULL", "TK_JOIN_KW", ALWAYS },
+ { "GLOB", "TK_GLOB", ALWAYS },
+ { "GROUP", "TK_GROUP", ALWAYS },
+ { "HAVING", "TK_HAVING", ALWAYS },
+ { "IGNORE", "TK_IGNORE", CONFLICT|TRIGGER },
+ { "IMMEDIATE", "TK_IMMEDIATE", FKEY },
+ { "IN", "TK_IN", ALWAYS },
+ { "INDEX", "TK_INDEX", ALWAYS },
+ { "INITIALLY", "TK_INITIALLY", FKEY },
+ { "INNER", "TK_JOIN_KW", ALWAYS },
+ { "INSERT", "TK_INSERT", ALWAYS },
+ { "INSTEAD", "TK_INSTEAD", TRIGGER },
+ { "INTERSECT", "TK_INTERSECT", COMPOUND },
+ { "INTO", "TK_INTO", ALWAYS },
+ { "IS", "TK_IS", ALWAYS },
+ { "ISNULL", "TK_ISNULL", ALWAYS },
+ { "JOIN", "TK_JOIN", ALWAYS },
+ { "KEY", "TK_KEY", ALWAYS },
+ { "LEFT", "TK_JOIN_KW", ALWAYS },
+ { "LIKE", "TK_LIKE", ALWAYS },
+ { "LIMIT", "TK_LIMIT", ALWAYS },
+ { "MATCH", "TK_MATCH", ALWAYS },
+ { "NATURAL", "TK_JOIN_KW", ALWAYS },
+ { "NOT", "TK_NOT", ALWAYS },
+ { "NOTNULL", "TK_NOTNULL", ALWAYS },
+ { "NULL", "TK_NULL", ALWAYS },
+ { "OF", "TK_OF", ALWAYS },
+ { "OFFSET", "TK_OFFSET", ALWAYS },
+ { "ON", "TK_ON", ALWAYS },
+ { "OR", "TK_OR", ALWAYS },
+ { "ORDER", "TK_ORDER", ALWAYS },
+ { "OUTER", "TK_JOIN_KW", ALWAYS },
+ { "PRAGMA", "TK_PRAGMA", PRAGMA },
+ { "PRIMARY", "TK_PRIMARY", ALWAYS },
+ { "RAISE", "TK_RAISE", TRIGGER },
+ { "REFERENCES", "TK_REFERENCES", FKEY },
+ { "REINDEX", "TK_REINDEX", REINDEX },
+ { "RENAME", "TK_RENAME", ALTER },
+ { "REPLACE", "TK_REPLACE", CONFLICT },
+ { "RESTRICT", "TK_RESTRICT", FKEY },
+ { "RIGHT", "TK_JOIN_KW", ALWAYS },
+ { "ROLLBACK", "TK_ROLLBACK", ALWAYS },
+ { "ROW", "TK_ROW", TRIGGER },
+ { "SELECT", "TK_SELECT", ALWAYS },
+ { "SET", "TK_SET", ALWAYS },
+ { "STATEMENT", "TK_STATEMENT", TRIGGER },
+ { "TABLE", "TK_TABLE", ALWAYS },
+ { "TEMP", "TK_TEMP", ALWAYS },
+ { "TEMPORARY", "TK_TEMP", ALWAYS },
+ { "THEN", "TK_THEN", ALWAYS },
+ { "TRANSACTION", "TK_TRANSACTION", ALWAYS },
+ { "TRIGGER", "TK_TRIGGER", TRIGGER },
+ { "UNION", "TK_UNION", COMPOUND },
+ { "UNIQUE", "TK_UNIQUE", ALWAYS },
+ { "UPDATE", "TK_UPDATE", ALWAYS },
+ { "USING", "TK_USING", ALWAYS },
+ { "VACUUM", "TK_VACUUM", VACUUM },
+ { "VALUES", "TK_VALUES", ALWAYS },
+ { "VIEW", "TK_VIEW", VIEW },
+ { "WHEN", "TK_WHEN", ALWAYS },
+ { "WHERE", "TK_WHERE", ALWAYS },
};
/* Number of keywords */
-#define NKEYWORD (sizeof(aKeywordTable)/sizeof(aKeywordTable[0]))
+static int NKEYWORD = (sizeof(aKeywordTable)/sizeof(aKeywordTable[0]));
/* An array to map all upper-case characters into their corresponding
** lower-case character.
int nChar;
int aHash[1000]; /* 1000 is much bigger than NKEYWORD */
+ /* Remove entries from the list of keywords that have mask==0 */
+ for(i=j=0; i<NKEYWORD; i++){
+ if( aKeywordTable[i].mask==0 ) continue;
+ if( j<i ){
+ aKeywordTable[j] = aKeywordTable[i];
+ }
+ j++;
+ }
+ NKEYWORD = j;
+
/* Fill in the lengths of strings and hashes for all entries. */
for(i=0; i<NKEYWORD; i++){
Keyword *p = &aKeywordTable[i];
projects / thirdparty / sqlite.git / commitdiff
