#include <stdlib.h>
#include <stdio.h>
#include <string.h>
-#include <ctype.h>
#include "fts3_tokenizer.h"
static int simpleDelim(simple_tokenizer *t, unsigned char c){
return c<0x80 && t->delim[c];
}
+static int fts3_isalnum(int x){
+ return (x>='0' && x<='9') || (x>='A' && x<='Z') || (x>='a' && x<='z');
+}
/*
** Create a new tokenizer instance.
/* Mark non-alphanumeric ASCII characters as delimiters */
int i;
for(i=1; i<0x80; i++){
- t->delim[i] = !isalnum(i) ? -1 : 0;
+ t->delim[i] = !fts3_isalnum(i) ? -1 : 0;
}
}
** case-insensitivity.
*/
unsigned char ch = p[iStartOffset+i];
- c->pToken[i] = (char)(ch<0x80 ? tolower(ch) : ch);
+ c->pToken[i] = (char)((ch>='A' && ch<='Z') ? ch-'A'+'a' : ch);
}
*ppToken = c->pToken;
*pnBytes = n;
-C Do\snot\srun\ssome\stests\sin\snotify3.test\swith\sthe\sinmemory_journal\spermutation.\sThey\sdo\snot\spass\sas\sthe\stests\sassume\sthat\sthe\sdatabase\sschema\sis\snot\sloaded\suntil\sthe\sfirst\sinvocation\sof\s[db\seval].\sThis\sis\snot\strue\swith\sthe\sinmemory_journal\spermutation.
-D 2010-08-06T13:50:07
+-----BEGIN PGP SIGNED MESSAGE-----
+Hash: SHA1
+
+C Remove\sall\svestiges\sof\sctype.h\sfrom\sFTS3.
+D 2010-08-06T19:00:12
F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
F Makefile.in ec08dc838fd8110fe24c92e5130bcd91cbb1ff2e
F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
F ext/fts3/fts3.c 9dec342fa1cf0914da679a3b7c0d4b53a27883ba
F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe
F ext/fts3/fts3Int.h 70528ba8c33991699f96ecc64112122833cdbdb5
-F ext/fts3/fts3_expr.c f4ff02ebe854e97ac03ff00b38b728a9ab57fd4b
+F ext/fts3/fts3_expr.c 42d5697731cd30fbeabd081bb3e6d3df5531f606
F ext/fts3/fts3_hash.c 3c8f6387a4a7f5305588b203fa7c887d753e1f1c
F ext/fts3/fts3_hash.h 8331fb2206c609f9fc4c4735b9ab5ad6137c88ec
F ext/fts3/fts3_icu.c ac494aed69835008185299315403044664bda295
-F ext/fts3/fts3_porter.c 7546e4503e286a67fd4f2a82159620e3e9c7a1bc
-F ext/fts3/fts3_snippet.c bc582c38e194b48818da862f9e6f293cc44e29ee
-F ext/fts3/fts3_tokenizer.c 1a49ee3d79cbf0b9386250370d9cbfe4bb89c8ff
+F ext/fts3/fts3_porter.c 8df6f6efcc4e9e31f8bf73a4007c2e9abca1dfba
+F ext/fts3/fts3_snippet.c 2c4c921155e4b6befd272041fb903d999ac07d30
+F ext/fts3/fts3_tokenizer.c b4f2d01c24573852755bc92864816785dae39318
F ext/fts3/fts3_tokenizer.h 13ffd9fcb397fec32a05ef5cd9e0fa659bf3dbd3
-F ext/fts3/fts3_tokenizer1.c b6d86d1d750787db5c168c73da4e87670ed890a1
+F ext/fts3/fts3_tokenizer1.c 6e5cbaa588924ac578263a598e4fb9f5c9bb179d
F ext/fts3/fts3_write.c 4b21a0c6f2772b261f14e3a2e80e1e3e849268b0
F ext/fts3/mkfts3amal.tcl 252ecb7fe6467854f2aa237bf2c390b74e71f100
F ext/icu/README.txt bf8461d8cdc6b8f514c080e4e10dc3b2bbdfefa9
F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224
F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e
F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f
-P a7a15547cccb4aeb719aba19641f31390c316d8c
-R da6a7e3b792474c6501a800e8485c2ad
-U dan
-Z 24f7070a35c4e17ac7f061d90b9ace3b
+P 698fba826e40ce6414cf6c261441d68b174c1637
+R d1c3acba9d17e2a6195139457b0f8a0c
+U drh
+Z b03f5d3d02d0448f3b5b5b106b20d50e
+-----BEGIN PGP SIGNATURE-----
+Version: GnuPG v1.4.6 (GNU/Linux)
+
+iD8DBQFMXFvAoxKgR168RlERAgErAJ0aaOhg70i98cKm/RR+yqQ0GisxQgCcDQWF
+u8dUrDhb4JsjIhV8JllIlg0=
+=tPRq
+-----END PGP SIGNATURE-----