** sqliteRegisterBuildinFunctions() found at the bottom of the file.
** All other code has file scope.
**
-** $Id: func.c,v 1.156 2007/05/15 01:13:47 drh Exp $
+** $Id: func.c,v 1.157 2007/05/15 11:55:09 drh Exp $
*/
#include "sqliteInt.h"
#include <ctype.h>
case SQLITE_TEXT: {
const unsigned char *z = sqlite3_value_text(argv[0]);
if( z==0 ) return;
- for(len=0; *z; z++){ if( (0xc0&*z)!=0x80 ) len++; }
+ len = 0;
+ while( *z ){
+ len++;
+ SQLITE_SKIP_UTF8(z);
+ }
sqlite3_result_int(context, len);
break;
}
){
const unsigned char *z;
const unsigned char *z2;
- int i;
int len;
int p0type;
i64 p1, p2;
}else{
z = sqlite3_value_text(argv[0]);
if( z==0 ) return;
- for(len=0, z2=z; *z2; z2++){ if( (0xc0&*z2)!=0x80 ) len++; }
+ len = 0;
+ for(z2=z; *z2; len++){
+ SQLITE_SKIP_UTF8(z2);
+ }
}
p1 = sqlite3_value_int(argv[1]);
p2 = sqlite3_value_int(argv[2]);
p2 = len-p1;
}
if( p0type!=SQLITE_BLOB ){
- for(i=0; i<p1 && z[i]; i++){
- if( (z[i]&0xc0)==0x80 ) p1++;
+ while( *z && p1 ){
+ SQLITE_SKIP_UTF8(z);
+ p1--;
}
- while( z[i] && (z[i]&0xc0)==0x80 ){ i++; p1++; }
- for(; i<p1+p2 && z[i]; i++){
- if( (z[i]&0xc0)==0x80 ) p2++;
+ for(z2=z; *z2 && p2; p2--){
+ SQLITE_SKIP_UTF8(z2);
}
- while( z[i] && (z[i]&0xc0)==0x80 ){ i++; p2++; }
- if( p2<0 ) p2 = 0;
- sqlite3_result_text(context, (char*)&z[p1], p2, SQLITE_TRANSIENT);
+ sqlite3_result_text(context, (char*)z, z2-z, SQLITE_TRANSIENT);
}else{
if( p2<0 ) p2 = 0;
sqlite3_result_blob(context, (char*)&z[p1], p2, SQLITE_TRANSIENT);
static const struct compareInfo likeInfoAlt = { '%', '_', 0, 0 };
/*
-** X is a pointer to the first byte of a UTF-8 character. Increment
-** X so that it points to the next character. This only works right
-** if X points to a well-formed UTF-8 string.
+** Read a single UTF-8 character and return its value.
*/
-#define sqliteNextChar(X) while( (0xc0&*++(X))==0x80 ){}
-#define sqliteCharVal(X) sqlite3ReadUtf8(X)
-
+u32 sqlite3ReadUtf8(const unsigned char *z){
+ u32 c;
+ SQLITE_READ_UTF8(z, c);
+ return c;
+}
/*
** Compare two UTF-8 strings for equality where the first string can
while( (c=zPattern[1]) == matchAll || c == matchOne ){
if( c==matchOne ){
if( *zString==0 ) return 0;
- sqliteNextChar(zString);
+ SQLITE_SKIP_UTF8(zString);
}
zPattern++;
}
if( c && esc && sqlite3ReadUtf8(&zPattern[1])==esc ){
u8 const *zTemp = &zPattern[1];
- sqliteNextChar(zTemp);
+ SQLITE_SKIP_UTF8(zTemp);
c = *zTemp;
}
if( c==0 ) return 1;
if( c==matchSet ){
assert( esc==0 ); /* This is GLOB, not LIKE */
while( *zString && patternCompare(&zPattern[1],zString,pInfo,esc)==0 ){
- sqliteNextChar(zString);
+ SQLITE_SKIP_UTF8(zString);
}
return *zString!=0;
}else{
}
if( c2==0 ) return 0;
if( patternCompare(&zPattern[1],zString,pInfo,esc) ) return 1;
- sqliteNextChar(zString);
+ SQLITE_SKIP_UTF8(zString);
}
return 0;
}
}else if( !prevEscape && c==matchOne ){
if( *zString==0 ) return 0;
- sqliteNextChar(zString);
+ SQLITE_SKIP_UTF8(zString);
zPattern++;
}else if( c==matchSet ){
int prior_c = 0;
assert( esc==0 ); /* This only occurs for GLOB, not LIKE */
seen = 0;
invert = 0;
- c = sqliteCharVal(zString);
+ c = sqlite3ReadUtf8(zString);
if( c==0 ) return 0;
c2 = *++zPattern;
if( c2=='^' ){ invert = 1; c2 = *++zPattern; }
if( c==']' ) seen = 1;
c2 = *++zPattern;
}
- while( (c2 = sqliteCharVal(zPattern))!=0 && c2!=']' ){
+ while( (c2 = sqlite3ReadUtf8(zPattern))!=0 && c2!=']' ){
if( c2=='-' && zPattern[1]!=']' && zPattern[1]!=0 && prior_c>0 ){
zPattern++;
- c2 = sqliteCharVal(zPattern);
+ c2 = sqlite3ReadUtf8(zPattern);
if( c>=prior_c && c<=c2 ) seen = 1;
prior_c = 0;
}else if( c==c2 ){
}else{
prior_c = c2;
}
- sqliteNextChar(zPattern);
+ SQLITE_SKIP_UTF8(zPattern);
}
if( c2==0 || (seen ^ invert)==0 ) return 0;
- sqliteNextChar(zString);
+ SQLITE_SKIP_UTF8(zString);
zPattern++;
}else if( esc && !prevEscape && sqlite3ReadUtf8(zPattern)==esc){
prevEscape = 1;
- sqliteNextChar(zPattern);
+ SQLITE_SKIP_UTF8(zPattern);
}else{
if( noCase ){
if( sqlite3UpperToLower[c] != sqlite3UpperToLower[*zString] ) return 0;
}else{
const unsigned char *z;
for(z=zCharSet, nChar=0; *z; nChar++){
- sqliteNextChar(z);
+ SQLITE_SKIP_UTF8(z);
}
if( nChar>0 ){
azChar = sqlite3_malloc( nChar*(sizeof(char*)+1) );
aLen = (unsigned char*)&azChar[nChar];
for(z=zCharSet, nChar=0; *z; nChar++){
azChar[nChar] = z;
- sqliteNextChar(z);
+ SQLITE_SKIP_UTF8(z);
aLen[nChar] = z - azChar[nChar];
}
}
*************************************************************************
** Internal interface definitions for SQLite.
**
-** @(#) $Id: sqliteInt.h,v 1.566 2007/05/12 12:08:51 drh Exp $
+** @(#) $Id: sqliteInt.h,v 1.567 2007/05/15 11:55:09 drh Exp $
*/
#ifndef _SQLITEINT_H_
#define _SQLITEINT_H_
*/
extern int sqlite3_always_code_trigger_setup;
+/*
+** A lookup table used by the SQLITE_READ_UTF8 macro. The definition
+** is in utf.c.
+*/
+extern const unsigned char sqlite3UtfTrans1[];
+
+/*
+** Macros for reading UTF8 characters.
+**
+** SQLITE_READ_UTF8(x,c) reads a single UTF8 value out of x and writes
+** that value into c. The type of x must be unsigned char*. The type
+** of c must be unsigned int.
+**
+** SQLITE_SKIP_UTF8(x) advances x forward by one character. The type of
+** x must be unsigned char*.
+**
+** Notes On Invalid UTF-8:
+**
+** * These macros never allow a 7-bit character (0x00 through 0x7f) to
+** be encoded as a multi-byte character. Any multi-byte character that
+** attempts to encode a value between 0x00 and 0x7f is rendered as 0xfffd.
+**
+** * These macros never allow a UTF16 surragate value to be encoded.
+** If a multi-byte character attempts to encode a value between
+** 0xd800 and 0xe000 then it is rendered as 0xfffd.
+**
+** * Bytes in the range of 0x80 through 0xbf which occur as the first
+** byte of a character are interpreted as single-byte characters
+** and rendered as themselves even though they are technically
+** invalid characters.
+**
+** * These routines accept an infinite number of different UTF8 encodings
+** for unicode values 0x80 and greater. They do not change over-length
+** encodings to 0xfffd as some systems recommend.
+**
+*/
+#define SQLITE_READ_UTF8(zIn, c) { \
+ c = *(zIn++); \
+ if( c>=0xc0 ){ \
+ c = sqlite3UtfTrans1[c-0xc0]; \
+ while( (*zIn & 0xc0)==0x80 ){ \
+ c = (c<<6) + (0x3f & *(zIn++)); \
+ } \
+ if( c<0x80 \
+ || (c&0xFFFFF800)==0xD800 \
+ || (c&0xFFFFFFFE)==0xFFFE ){ c = 0xFFFD; } \
+ } \
+}
+#define SQLITE_SKIP_UTF8(zIn) { \
+ if( (*(zIn++))>=0xc0 ){ \
+ while( (*zIn & 0xc0)==0x80 ){ zIn++; } \
+ } \
+}
+
+
+
+
/*
** The SQLITE_CORRUPT_BKPT macro can be either a constant (for production
** builds) or a function call (for debugging). If it is a function call,
int sqlite3FitsIn64Bits(const char *);
int sqlite3Utf16ByteLen(const void *pData, int nChar);
int sqlite3Utf8CharLen(const char *pData, int nByte);
-int sqlite3ReadUtf8(const unsigned char *);
+u32 sqlite3ReadUtf8(const unsigned char *);
int sqlite3PutVarint(unsigned char *, u64);
int sqlite3GetVarint(const unsigned char *, u64 *);
int sqlite3GetVarint32(const unsigned char *, u32 *);
** This file contains routines used to translate between UTF-8,
** UTF-16, UTF-16BE, and UTF-16LE.
**
-** $Id: utf.c,v 1.46 2007/05/10 17:23:12 drh Exp $
+** $Id: utf.c,v 1.47 2007/05/15 11:55:09 drh Exp $
**
** Notes on UTF-8:
**
** 0xff 0xfe little-endian utf-16 follows
** 0xfe 0xff big-endian utf-16 follows
**
-**
-** Handling of malformed strings:
-**
-** SQLite accepts and processes malformed strings without an error wherever
-** possible. However this is not possible when converting between UTF-8 and
-** UTF-16.
-**
-** When converting malformed UTF-8 strings to UTF-16, one instance of the
-** replacement character U+FFFD for each byte that cannot be interpeted as
-** part of a valid unicode character.
-**
-** When converting malformed UTF-16 strings to UTF-8, one instance of the
-** replacement character U+FFFD for each pair of bytes that cannot be
-** interpeted as part of a valid unicode character.
-**
-** This file contains the following public routines:
-**
-** sqlite3VdbeMemTranslate() - Translate the encoding used by a Mem* string.
-** sqlite3VdbeMemHandleBom() - Handle byte-order-marks in UTF16 Mem* strings.
-** sqlite3Utf16ByteLen() - Calculate byte-length of a void* UTF16 string.
-** sqlite3Utf8CharLen() - Calculate char-length of a char* UTF8 string.
-** sqlite3Utf8LikeCompare() - Do a LIKE match given two UTF8 char* strings.
-**
*/
#include "sqliteInt.h"
#include <assert.h>
const int sqlite3one = 1;
/*
-** This table maps from the first byte of a UTF-8 character to the number
-** of trailing bytes expected. A value '4' indicates that the table key
-** is not a legal first byte for a UTF-8 character.
-*/
-static const u8 xtra_utf8_bytes[256] = {
-/* 0xxxxxxx */
-0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
-0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
-0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
-0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
-0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
-0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
-0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
-0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
-
-/* 10wwwwww */
-4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4,
-4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4,
-4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4,
-4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4,
-
-/* 110yyyyy */
-1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
-1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
-
-/* 1110zzzz */
-2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
-
-/* 11110yyy */
-3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4,
-};
-
-/*
-** This table maps from the number of trailing bytes in a UTF-8 character
-** to an integer constant that is effectively calculated for each character
-** read by a naive implementation of a UTF-8 character reader. The code
-** in the READ_UTF8 macro explains things best.
+** This lookup table is used to help decode the first byte of
+** a multi-byte UTF8 character.
*/
-static const int xtra_utf8_bits[] = {
- 0,
- 12416, /* (0xC0 << 6) + (0x80) */
- 925824, /* (0xE0 << 12) + (0x80 << 6) + (0x80) */
- 63447168 /* (0xF0 << 18) + (0x80 << 12) + (0x80 << 6) + 0x80 */
+const unsigned char sqlite3UtfTrans1[] = {
+ 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
+ 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f,
+ 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17,
+ 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f,
+ 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
+ 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f,
+ 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
+ 0x00, 0x01, 0x02, 0x03, 0x00, 0x01, 0x00, 0x00,
};
-/*
-** If a UTF-8 character contains N bytes extra bytes (N bytes follow
-** the initial byte so that the total character length is N+1) then
-** masking the character with utf8_mask[N] must produce a non-zero
-** result. Otherwise, we have an (illegal) overlong encoding.
-*/
-static const int utf_mask[] = {
- 0x00000000,
- 0xffffff80,
- 0xfffff800,
- 0xffff0000,
-};
-
-#define READ_UTF8(zIn, c) { \
- int xtra; \
- c = *(zIn)++; \
- xtra = xtra_utf8_bytes[c]; \
- switch( xtra ){ \
- case 4: c = (int)0xFFFD; break; \
- case 3: c = (c<<6) + *(zIn)++; \
- case 2: c = (c<<6) + *(zIn)++; \
- case 1: c = (c<<6) + *(zIn)++; \
- c -= xtra_utf8_bits[xtra]; \
- if( (utf_mask[xtra]&c)==0 \
- || (c&0xFFFFF800)==0xD800 \
- || (c&0xFFFFFFFE)==0xFFFE ){ c = 0xFFFD; } \
- } \
-}
-int sqlite3ReadUtf8(const unsigned char *z){
- int c;
- READ_UTF8(z, c);
- return c;
-}
-
-#define SKIP_UTF8(zIn) { \
- zIn += (xtra_utf8_bytes[*(u8 *)zIn] + 1); \
-}
-
#define WRITE_UTF8(zOut, c) { \
if( c<0x00080 ){ \
*zOut++ = (c&0xFF); \
if( desiredEnc==SQLITE_UTF16LE ){
/* UTF-8 -> UTF-16 Little-endian */
while( zIn<zTerm ){
- READ_UTF8(zIn, c);
+ SQLITE_READ_UTF8(zIn, c);
WRITE_UTF16LE(z, c);
}
}else{
assert( desiredEnc==SQLITE_UTF16BE );
/* UTF-8 -> UTF-16 Big-endian */
while( zIn<zTerm ){
- READ_UTF8(zIn, c);
+ SQLITE_READ_UTF8(zIn, c);
WRITE_UTF16BE(z, c);
}
}
** number of unicode characters in the first nByte of pZ (or up to
** the first 0x00, whichever comes first).
*/
-int sqlite3Utf8CharLen(const char *z, int nByte){
+int sqlite3Utf8CharLen(const char *zIn, int nByte){
int r = 0;
- const char *zTerm;
+ const u8 *z = (const u8*)zIn;
+ const u8 *zTerm;
if( nByte>=0 ){
zTerm = &z[nByte];
}else{
- zTerm = (const char *)(-1);
+ zTerm = (const u8*)(-1);
}
assert( z<=zTerm );
while( *z!=0 && z<zTerm ){
- SKIP_UTF8(z);
+ SQLITE_SKIP_UTF8(z);
r++;
}
return r;
int c;
while(1){
- READ_UTF8(zIn, c);
+ SQLITE_READ_UTF8(zIn, c);
if( c==0 ) break;
if( c!=0xfffd ){
WRITE_UTF8(zOut, c);
z = zBuf;
WRITE_UTF8(z, i);
n = z-zBuf;
+ z[0] = 0;
z = zBuf;
- READ_UTF8(z, c);
+ SQLITE_READ_UTF8(z, c);
t = i;
if( i>=0xD800 && i<=0xDFFF ) t = 0xFFFD;
if( (i&0xFFFFFFFE)==0xFFFE ) t = 0xFFFD;
z = zBuf;
WRITE_UTF16LE(z, i);
n = z-zBuf;
+ z[0] = 0;
z = zBuf;
READ_UTF16LE(z, c);
assert( c==i );
z = zBuf;
WRITE_UTF16BE(z, i);
n = z-zBuf;
+ z[0] = 0;
z = zBuf;
READ_UTF16BE(z, c);
assert( c==i );
projects / thirdparty / sqlite.git / commitdiff
