SECURITY: If the fts3 extension is used in an environment where potentially
malicious users may execute arbitrary SQL (i.e. gears), they should be
- prevented from invoking the fts3_tokenizer() function, possibly using the
- authorisation callback.
+ prevented from invoking the fts3_tokenizer() function. The
+ fts3_tokenizer() function is disabled by default. It is only enabled
+ by SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER. Do not enable it in
+ security sensitive environments.
See "Sample code" below for an example of calling the fts3_tokenizer()
function from C code.
# endif
#endif
+/*
+** This variable is set to false when running tests for which the on disk
+** structures should not be corrupt. Otherwise, true. If it is false, extra
+** assert() conditions in the fts3 code are activated - conditions that are
+** only true if it is guaranteed that the fts3 database is not corrupt.
+*/
+int sqlite3_fts3_may_be_corrupt = 1;
+
/*
** Write a 64-bit variable-length integer to memory starting at p[0].
** The length of data written will be between 1 and FTS3_VARINT_MAX bytes.
}
#define GETVARINT_STEP(v, ptr, shift, mask1, mask2, var, ret) \
- v = (v & mask1) | ( (*(ptr++)) << shift ); \
+ v = (v & mask1) | ( (*(const unsigned char*)(ptr++)) << shift ); \
if( (v & mask2)==0 ){ var = v; return ret; }
#define GETVARINT_INIT(v, ptr, shift, mask1, mask2, var, ret) \
v = (*ptr++); \
if( (v & mask2)==0 ){ var = v; return ret; }
-/*
-** Read a 64-bit variable-length integer from memory starting at p[0].
-** Return the number of bytes read, or 0 on error.
-** The value is stored in *v.
-*/
-int sqlite3Fts3GetVarint(const char *pBuf, sqlite_int64 *v){
+int sqlite3Fts3GetVarintU(const char *pBuf, sqlite_uint64 *v){
const unsigned char *p = (const unsigned char*)pBuf;
const unsigned char *pStart = p;
u32 a;
return (int)(p - pStart);
}
+/*
+** Read a 64-bit variable-length integer from memory starting at p[0].
+** Return the number of bytes read, or 0 on error.
+** The value is stored in *v.
+*/
+int sqlite3Fts3GetVarint(const char *pBuf, sqlite_int64 *v){
+ return sqlite3Fts3GetVarintU(pBuf, (sqlite3_uint64*)v);
+}
+
+/*
+** Read a 64-bit variable-length integer from memory starting at p[0] and
+** not extending past pEnd[-1].
+** Return the number of bytes read, or 0 on error.
+** The value is stored in *v.
+*/
+int sqlite3Fts3GetVarintBounded(
+ const char *pBuf,
+ const char *pEnd,
+ sqlite_int64 *v
+){
+ const unsigned char *p = (const unsigned char*)pBuf;
+ const unsigned char *pStart = p;
+ const unsigned char *pX = (const unsigned char*)pEnd;
+ u64 b = 0;
+ int shift;
+ for(shift=0; shift<=63; shift+=7){
+ u64 c = p<pX ? *p : 0;
+ p++;
+ b += (c&0x7F) << shift;
+ if( (c & 0x80)==0 ) break;
+ }
+ *v = b;
+ return (int)(p - pStart);
+}
+
/*
** Similar to sqlite3Fts3GetVarint(), except that the output is truncated to
** a non-negative 32-bit integer before it is returned.
sqlite3 *db = p->db; /* Database handle */
/* Drop the shadow tables */
- if( p->zContentTbl==0 ){
- fts3DbExec(&rc, db, "DROP TABLE IF EXISTS %Q.'%q_content'", zDb, p->zName);
- }
- fts3DbExec(&rc, db, "DROP TABLE IF EXISTS %Q.'%q_segments'", zDb,p->zName);
- fts3DbExec(&rc, db, "DROP TABLE IF EXISTS %Q.'%q_segdir'", zDb, p->zName);
- fts3DbExec(&rc, db, "DROP TABLE IF EXISTS %Q.'%q_docsize'", zDb, p->zName);
- fts3DbExec(&rc, db, "DROP TABLE IF EXISTS %Q.'%q_stat'", zDb, p->zName);
+ fts3DbExec(&rc, db,
+ "DROP TABLE IF EXISTS %Q.'%q_segments';"
+ "DROP TABLE IF EXISTS %Q.'%q_segdir';"
+ "DROP TABLE IF EXISTS %Q.'%q_docsize';"
+ "DROP TABLE IF EXISTS %Q.'%q_stat';"
+ "%s DROP TABLE IF EXISTS %Q.'%q_content';",
+ zDb, p->zName,
+ zDb, p->zName,
+ zDb, p->zName,
+ zDb, p->zName,
+ (p->zContentTbl ? "--" : ""), zDb,p->zName
+ );
/* If everything has worked, invoke fts3DisconnectMethod() to free the
** memory associated with the Fts3Table structure and return SQLITE_OK.
** memory.
*/
static char *fts3QuoteId(char const *zInput){
- int nRet;
+ sqlite3_int64 nRet;
char *zRet;
nRet = 2 + (int)strlen(zInput)*2 + 1;
- zRet = sqlite3_malloc(nRet);
+ zRet = sqlite3_malloc64(nRet);
if( zRet ){
int i;
char *z = zRet;
}
}
- aIndex = sqlite3_malloc(sizeof(struct Fts3Index) * nIndex);
+ aIndex = sqlite3_malloc64(sizeof(struct Fts3Index) * nIndex);
*apIndex = aIndex;
if( !aIndex ){
return SQLITE_NOMEM;
if( rc==SQLITE_OK ){
const char **azCol; /* Output array */
- int nStr = 0; /* Size of all column names (incl. 0x00) */
+ sqlite3_int64 nStr = 0; /* Size of all column names (incl. 0x00) */
int nCol; /* Number of table columns */
int i; /* Used to iterate through columns */
nCol = sqlite3_column_count(pStmt);
for(i=0; i<nCol; i++){
const char *zCol = sqlite3_column_name(pStmt, i);
- nStr += (int)strlen(zCol) + 1;
+ nStr += strlen(zCol) + 1;
}
/* Allocate and populate the array to return. */
- azCol = (const char **)sqlite3_malloc(sizeof(char *) * nCol + nStr);
+ azCol = (const char **)sqlite3_malloc64(sizeof(char *) * nCol + nStr);
if( azCol==0 ){
rc = SQLITE_NOMEM;
}else{
Fts3Table *p = 0; /* Pointer to allocated vtab */
int rc = SQLITE_OK; /* Return code */
int i; /* Iterator variable */
- int nByte; /* Size of allocation used for *p */
+ sqlite3_int64 nByte; /* Size of allocation used for *p */
int iCol; /* Column index */
int nString = 0; /* Bytes required to hold all column names */
int nCol = 0; /* Number of columns in the FTS table */
nName = (int)strlen(argv[2]) + 1;
nByte = sizeof(const char *) * (argc-2);
- aCol = (const char **)sqlite3_malloc(nByte);
+ aCol = (const char **)sqlite3_malloc64(nByte);
if( aCol ){
memset((void*)aCol, 0, nByte);
- azNotindexed = (char **)sqlite3_malloc(nByte);
+ azNotindexed = (char **)sqlite3_malloc64(nByte);
}
if( azNotindexed ){
memset(azNotindexed, 0, nByte);
nName + /* zName */
nDb + /* zDb */
nString; /* Space for azColumn strings */
- p = (Fts3Table*)sqlite3_malloc(nByte);
+ p = (Fts3Table*)sqlite3_malloc64(nByte);
if( p==0 ){
rc = SQLITE_NOMEM;
goto fts3_init_out;
fts3DatabasePageSize(&rc, p);
p->nNodeSize = p->nPgsz-35;
+#if defined(SQLITE_DEBUG)||defined(SQLITE_TEST)
+ p->nMergeCount = FTS3_MERGE_COUNT;
+#endif
+
/* Declare the table schema to SQLite. */
fts3DeclareVtab(&rc, p);
int iDocidLe = -1; /* Index of docid<=x constraint, if present */
int iIdx;
+ if( p->bLock ){
+ return SQLITE_ERROR;
+ }
+
/* By default use a full table scan. This is an expensive option,
** so search through the constraints to see if a more efficient
** strategy is possible.
}else{
zSql = sqlite3_mprintf("SELECT %s WHERE rowid = ?", p->zReadExprlist);
if( !zSql ) return SQLITE_NOMEM;
- rc = sqlite3_prepare_v3(p->db, zSql,-1,SQLITE_PREPARE_PERSISTENT,&pCsr->pStmt,0);
+ p->bLock++;
+ rc = sqlite3_prepare_v3(
+ p->db, zSql,-1,SQLITE_PREPARE_PERSISTENT,&pCsr->pStmt,0
+ );
+ p->bLock--;
sqlite3_free(zSql);
}
if( rc==SQLITE_OK ) pCsr->bSeekStmt = 1;
if( pCsr->isRequireSeek ){
rc = fts3CursorSeekStmt(pCsr);
if( rc==SQLITE_OK ){
+ Fts3Table *pTab = (Fts3Table*)pCsr->base.pVtab;
+ pTab->bLock++;
sqlite3_bind_int64(pCsr->pStmt, 1, pCsr->iPrevId);
pCsr->isRequireSeek = 0;
if( SQLITE_ROW==sqlite3_step(pCsr->pStmt) ){
+ pTab->bLock--;
return SQLITE_OK;
}else{
+ pTab->bLock--;
rc = sqlite3_reset(pCsr->pStmt);
if( rc==SQLITE_OK && ((Fts3Table *)pCsr->base.pVtab)->zContentTbl==0 ){
/* If no row was found and no error has occurred, then the %_content
zCsr += fts3GetVarint32(zCsr, &nSuffix);
assert( nPrefix>=0 && nSuffix>=0 );
- if( nPrefix>zCsr-zNode || nSuffix>zEnd-zCsr ){
+ if( nPrefix>zCsr-zNode || nSuffix>zEnd-zCsr || nSuffix==0 ){
rc = FTS_CORRUPT_VTAB;
goto finish_scan;
}
fts3GetVarint32(zNode, &iHeight);
rc = fts3ScanInteriorNode(zTerm, nTerm, zNode, nNode, piLeaf, piLeaf2);
- assert( !piLeaf2 || !piLeaf || rc!=SQLITE_OK || (*piLeaf<=*piLeaf2) );
+ assert_fts3_nc( !piLeaf2 || !piLeaf || rc!=SQLITE_OK || (*piLeaf<=*piLeaf2) );
if( rc==SQLITE_OK && iHeight>1 ){
char *zBlob = 0; /* Blob read from %_segments table */
rc = sqlite3Fts3ReadBlock(p, piLeaf?*piLeaf:*piLeaf2, &zBlob, &nBlob, 0);
}
if( rc==SQLITE_OK ){
- rc = fts3SelectLeaf(p, zTerm, nTerm, zBlob, nBlob, piLeaf, piLeaf2);
+ int iNewHeight = 0;
+ fts3GetVarint32(zBlob, &iNewHeight);
+ if( iNewHeight>=iHeight ){
+ rc = FTS_CORRUPT_VTAB;
+ }else{
+ rc = fts3SelectLeaf(p, zTerm, nTerm, zBlob, nBlob, piLeaf, piLeaf2);
+ }
}
sqlite3_free(zBlob);
}
}
/*
-** Value used to signify the end of an position-list. This is safe because
-** it is not possible to have a document with 2^31 terms.
+** Value used to signify the end of an position-list. This must be
+** as large or larger than any value that might appear on the
+** position-list, even a position list that has been corrupted.
*/
-#define POSITION_LIST_END 0x7fffffff
+#define POSITION_LIST_END LARGEST_INT64
/*
** This function is used to help parse position-lists. When this function is
** updated appropriately. The caller is responsible for insuring
** that there is enough space in *pp to hold the complete output.
*/
-static void fts3PoslistMerge(
+static int fts3PoslistMerge(
char **pp, /* Output buffer */
char **pp1, /* Left input list */
char **pp2 /* Right input list */
int iCol1; /* The current column index in pp1 */
int iCol2; /* The current column index in pp2 */
- if( *p1==POS_COLUMN ) fts3GetVarint32(&p1[1], &iCol1);
- else if( *p1==POS_END ) iCol1 = POSITION_LIST_END;
+ if( *p1==POS_COLUMN ){
+ fts3GetVarint32(&p1[1], &iCol1);
+ if( iCol1==0 ) return FTS_CORRUPT_VTAB;
+ }
+ else if( *p1==POS_END ) iCol1 = 0x7fffffff;
else iCol1 = 0;
- if( *p2==POS_COLUMN ) fts3GetVarint32(&p2[1], &iCol2);
- else if( *p2==POS_END ) iCol2 = POSITION_LIST_END;
+ if( *p2==POS_COLUMN ){
+ fts3GetVarint32(&p2[1], &iCol2);
+ if( iCol2==0 ) return FTS_CORRUPT_VTAB;
+ }
+ else if( *p2==POS_END ) iCol2 = 0x7fffffff;
else iCol2 = 0;
if( iCol1==iCol2 ){
*pp = p;
*pp1 = p1 + 1;
*pp2 = p2 + 1;
+ return SQLITE_OK;
}
/*
p += sqlite3Fts3PutVarint(p, iCol1);
}
- assert( *p1!=POS_END && *p1!=POS_COLUMN );
- assert( *p2!=POS_END && *p2!=POS_COLUMN );
fts3GetDeltaVarint(&p1, &iPos1); iPos1 -= 2;
fts3GetDeltaVarint(&p2, &iPos2); iPos2 -= 2;
+ if( iPos1<0 || iPos2<0 ) break;
while( 1 ){
if( iPos2==iPos1+nToken
if( *pp>=pEnd ){
*pp = 0;
}else{
- sqlite3_int64 iVal;
- *pp += sqlite3Fts3GetVarint(*pp, &iVal);
+ u64 iVal;
+ *pp += sqlite3Fts3GetVarintU(*pp, &iVal);
if( bDescIdx ){
- *pVal -= iVal;
+ *pVal = (i64)((u64)*pVal - iVal);
}else{
- *pVal += iVal;
+ *pVal = (i64)((u64)*pVal + iVal);
}
}
}
int *pbFirst, /* IN/OUT: True after first int written */
sqlite3_int64 iVal /* Write this value to the list */
){
- sqlite3_int64 iWrite;
+ sqlite3_uint64 iWrite;
if( bDescIdx==0 || *pbFirst==0 ){
- iWrite = iVal - *piPrev;
+ assert_fts3_nc( *pbFirst==0 || iVal>=*piPrev );
+ iWrite = (u64)iVal - (u64)*piPrev;
}else{
- iWrite = *piPrev - iVal;
+ assert_fts3_nc( *piPrev>=iVal );
+ iWrite = (u64)*piPrev - (u64)iVal;
}
assert( *pbFirst || *piPrev==0 );
- assert( *pbFirst==0 || iWrite>0 );
+ assert_fts3_nc( *pbFirst==0 || iWrite>0 );
*pp += sqlite3Fts3PutVarint(*pp, iWrite);
*piPrev = iVal;
*pbFirst = 1;
** Using this makes it easier to write code that can merge doclists that are
** sorted in either ascending or descending order.
*/
-#define DOCID_CMP(i1, i2) ((bDescDoclist?-1:1) * (i1-i2))
+/* #define DOCID_CMP(i1, i2) ((bDescDoclist?-1:1) * (i64)((u64)i1-i2)) */
+#define DOCID_CMP(i1, i2) ((bDescDoclist?-1:1) * (i1>i2?1:((i1==i2)?0:-1)))
/*
** This function does an "OR" merge of two doclists (output contains all
char *a2, int n2, /* Second doclist */
char **paOut, int *pnOut /* OUT: Malloc'd doclist */
){
+ int rc = SQLITE_OK;
sqlite3_int64 i1 = 0;
sqlite3_int64 i2 = 0;
sqlite3_int64 iPrev = 0;
** A symetric argument may be made if the doclists are in descending
** order.
*/
- aOut = sqlite3_malloc(n1+n2+FTS3_VARINT_MAX-1);
+ aOut = sqlite3_malloc64((i64)n1+n2+FTS3_VARINT_MAX-1+FTS3_BUFFER_PADDING);
if( !aOut ) return SQLITE_NOMEM;
p = aOut;
if( p2 && p1 && iDiff==0 ){
fts3PutDeltaVarint3(&p, bDescDoclist, &iPrev, &bFirstOut, i1);
- fts3PoslistMerge(&p, &p1, &p2);
+ rc = fts3PoslistMerge(&p, &p1, &p2);
+ if( rc ) break;
fts3GetDeltaVarint3(&p1, pEnd1, bDescDoclist, &i1);
fts3GetDeltaVarint3(&p2, pEnd2, bDescDoclist, &i2);
}else if( !p2 || (p1 && iDiff<0) ){
fts3PoslistCopy(&p, &p2);
fts3GetDeltaVarint3(&p2, pEnd2, bDescDoclist, &i2);
}
+
+ assert( (p-aOut)<=((p1?(p1-a1):n1)+(p2?(p2-a2):n2)+FTS3_VARINT_MAX-1) );
}
+ if( rc!=SQLITE_OK ){
+ sqlite3_free(aOut);
+ p = aOut = 0;
+ }else{
+ assert( (p-aOut)<=n1+n2+FTS3_VARINT_MAX-1 );
+ memset(&aOut[(p-aOut)], 0, FTS3_BUFFER_PADDING);
+ }
*paOut = aOut;
*pnOut = (int)(p-aOut);
- assert( *pnOut<=n1+n2+FTS3_VARINT_MAX-1 );
- return SQLITE_OK;
+ return rc;
}
/*
assert( nDist>0 );
if( bDescDoclist ){
- aOut = sqlite3_malloc(*pnRight + FTS3_VARINT_MAX);
+ aOut = sqlite3_malloc64((sqlite3_int64)*pnRight + FTS3_VARINT_MAX);
if( aOut==0 ) return SQLITE_NOMEM;
}else{
aOut = aRight;
pTS->anOutput[0] = nDoclist;
if( pTS->aaOutput[0] ){
memcpy(pTS->aaOutput[0], aDoclist, nDoclist);
+ memset(&pTS->aaOutput[0][nDoclist], 0, FTS3_VARINT_MAX);
}else{
return SQLITE_NOMEM;
}
){
if( (pCsr->nSegment%16)==0 ){
Fts3SegReader **apNew;
- int nByte = (pCsr->nSegment + 16)*sizeof(Fts3SegReader*);
- apNew = (Fts3SegReader **)sqlite3_realloc(pCsr->apSegment, nByte);
+ sqlite3_int64 nByte = (pCsr->nSegment + 16)*sizeof(Fts3SegReader*);
+ apNew = (Fts3SegReader **)sqlite3_realloc64(pCsr->apSegment, nByte);
if( !apNew ){
sqlite3Fts3SegReaderFree(pNew);
return SQLITE_NOMEM;
** Fts3SegReaderPending might segfault, as the data structures used by
** fts4aux are not completely populated. So it's easiest to filter these
** calls out here. */
- if( iLevel<0 && p->aIndex ){
+ if( iLevel<0 && p->aIndex && p->iPrevLangid==iLangid ){
Fts3SegReader *pSeg = 0;
rc = sqlite3Fts3SegReaderPending(p, iIndex, zTerm, nTerm, isPrefix||isScan, &pSeg);
if( rc==SQLITE_OK && pSeg ){
/* If zTerm is not NULL, and this segment is not stored entirely on its
** root node, the range of leaves scanned can be reduced. Do this. */
- if( iStartBlock && zTerm ){
+ if( iStartBlock && zTerm && zRoot ){
sqlite3_int64 *pi = (isPrefix ? &iLeavesEndBlock : 0);
rc = fts3SelectLeaf(p, zTerm, nTerm, zRoot, nRoot, &iStartBlock, pi);
if( rc!=SQLITE_OK ) goto finished;
int rc;
Fts3Cursor *pCsr = (Fts3Cursor *)pCursor;
if( pCsr->eSearch==FTS3_DOCID_SEARCH || pCsr->eSearch==FTS3_FULLSCAN_SEARCH ){
+ Fts3Table *pTab = (Fts3Table*)pCursor->pVtab;
+ pTab->bLock++;
if( SQLITE_ROW!=sqlite3_step(pCsr->pStmt) ){
pCsr->isEof = 1;
rc = sqlite3_reset(pCsr->pStmt);
pCsr->iPrevId = sqlite3_column_int64(pCsr->pStmt, 0);
rc = SQLITE_OK;
}
+ pTab->bLock--;
}else{
rc = fts3EvalNext((Fts3Cursor *)pCursor);
}
return rc;
}
-/*
-** The following are copied from sqliteInt.h.
-**
-** Constants for the largest and smallest possible 64-bit signed integers.
-** These macros are designed to work correctly on both 32-bit and 64-bit
-** compilers.
-*/
-#ifndef SQLITE_AMALGAMATION
-# define LARGEST_INT64 (0xffffffff|(((sqlite3_int64)0x7fffffff)<<32))
-# define SMALLEST_INT64 (((sqlite3_int64)-1) - LARGEST_INT64)
-#endif
-
/*
** If the numeric type of argument pVal is "integer", then return it
** converted to a 64-bit signed integer. Otherwise, return a copy of
UNUSED_PARAMETER(idxStr);
UNUSED_PARAMETER(nVal);
+ if( p->bLock ){
+ return SQLITE_ERROR;
+ }
+
eSearch = (idxNum & 0x0000FFFF);
assert( eSearch>=0 && eSearch<=(FTS3_FULLTEXT_SEARCH+p->nColumn) );
assert( p->pSegments==0 );
);
}
if( zSql ){
- rc = sqlite3_prepare_v3(p->db,zSql,-1,SQLITE_PREPARE_PERSISTENT,&pCsr->pStmt,0);
+ p->bLock++;
+ rc = sqlite3_prepare_v3(
+ p->db,zSql,-1,SQLITE_PREPARE_PERSISTENT,&pCsr->pStmt,0
+ );
+ p->bLock--;
sqlite3_free(zSql);
}else{
rc = SQLITE_NOMEM;
int rc = SQLITE_OK;
UNUSED_PARAMETER(iSavepoint);
assert( ((Fts3Table *)pVtab)->inTransaction );
- assert( ((Fts3Table *)pVtab)->mxSavepoint < iSavepoint );
+ assert( ((Fts3Table *)pVtab)->mxSavepoint <= iSavepoint );
TESTONLY( ((Fts3Table *)pVtab)->mxSavepoint = iSavepoint );
if( ((Fts3Table *)pVtab)->bIgnoreSavepoint==0 ){
rc = fts3SyncMethod(pVtab);
Fts3Table *p = (Fts3Table*)pVtab;
UNUSED_PARAMETER(iSavepoint);
assert( p->inTransaction );
- assert( p->mxSavepoint >= iSavepoint );
TESTONLY( p->mxSavepoint = iSavepoint );
sqlite3Fts3PendingTermsClear(p);
return SQLITE_OK;
}
static const sqlite3_module fts3Module = {
- /* iVersion */ 2,
+ /* iVersion */ 3,
/* xCreate */ fts3CreateMethod,
/* xConnect */ fts3ConnectMethod,
/* xBestIndex */ fts3BestIndexMethod,
#ifdef SQLITE_TEST
if( rc==SQLITE_OK ){
- rc = sqlite3Fts3ExprInitTestInterface(db);
+ rc = sqlite3Fts3ExprInitTestInterface(db, pHash);
}
#endif
return rc;
}
+#ifndef SQLITE_DISABLE_FTS4_DEFERRED
/*
** This function is called on each phrase after the position lists for
** any deferred tokens have been loaded into memory. It updates the phrases
return SQLITE_OK;
}
+#endif /* SQLITE_DISABLE_FTS4_DEFERRED */
/*
** Maximum number of tokens a phrase may have to be considered for the
int bIncrOk = (bOptOk
&& pCsr->bDesc==pTab->bDescIdx
&& p->nToken<=MAX_INCR_PHRASE_TOKENS && p->nToken>0
-#ifdef SQLITE_TEST
+#if defined(SQLITE_DEBUG) || defined(SQLITE_TEST)
&& pTab->bNoIncrDoclist==0
#endif
);
u8 *pbEof
){
char *pIter; /* Used to iterate through aAll */
- char *pEnd = &pDL->aAll[pDL->nAll]; /* 1 byte past end of aAll */
+ char *pEnd; /* 1 byte past end of aAll */
if( pDL->pNextDocid ){
pIter = pDL->pNextDocid;
+ assert( pDL->aAll!=0 || pIter==0 );
}else{
pIter = pDL->aAll;
}
- if( pIter>=pEnd ){
+ if( pIter==0 || pIter>=(pEnd = pDL->aAll + pDL->nAll) ){
/* We have already reached the end of this doclist. EOF. */
*pbEof = 1;
}else{
if( bEof==0 ){
int nList = 0;
int nByte = a[p->nToken-1].nList;
- char *aDoclist = sqlite3_malloc(nByte+1);
+ char *aDoclist = sqlite3_malloc(nByte+FTS3_BUFFER_PADDING);
if( !aDoclist ) return SQLITE_NOMEM;
memcpy(aDoclist, a[p->nToken-1].pList, nByte+1);
+ memset(&aDoclist[nByte], 0, FTS3_BUFFER_PADDING);
for(i=0; i<(p->nToken-1); i++){
if( a[i].bIgnore==0 ){
rc = sqlite3Fts3SelectDoctotal(p, &pStmt);
if( rc!=SQLITE_OK ) return rc;
a = sqlite3_column_blob(pStmt, 0);
- assert( a );
-
- pEnd = &a[sqlite3_column_bytes(pStmt, 0)];
- a += sqlite3Fts3GetVarint(a, &nDoc);
- while( a<pEnd ){
- a += sqlite3Fts3GetVarint(a, &nByte);
+ testcase( a==0 ); /* If %_stat.value set to X'' */
+ if( a ){
+ pEnd = &a[sqlite3_column_bytes(pStmt, 0)];
+ a += sqlite3Fts3GetVarintBounded(a, pEnd, &nDoc);
+ while( a<pEnd ){
+ a += sqlite3Fts3GetVarintBounded(a, pEnd, &nByte);
+ }
}
if( nDoc==0 || nByte==0 ){
sqlite3_reset(pStmt);
if( rc==SQLITE_OK && nToken>1 && pTab->bFts4 ){
Fts3TokenAndCost *aTC;
Fts3Expr **apOr;
- aTC = (Fts3TokenAndCost *)sqlite3_malloc(
+ aTC = (Fts3TokenAndCost *)sqlite3_malloc64(
sizeof(Fts3TokenAndCost) * nToken
+ sizeof(Fts3Expr *) * nOr * 2
);
&& (pExpr->pParent==0 || pExpr->pParent->eType!=FTSQUERY_NEAR)
){
Fts3Expr *p;
- int nTmp = 0; /* Bytes of temp space */
+ sqlite3_int64 nTmp = 0; /* Bytes of temp space */
char *aTmp; /* Temp space for PoslistNearMerge() */
/* Allocate temporary working space. */
nTmp += p->pRight->pPhrase->doclist.nList;
}
nTmp += p->pPhrase->doclist.nList;
- aTmp = sqlite3_malloc(nTmp*2);
+ aTmp = sqlite3_malloc64(nTmp*2);
if( !aTmp ){
*pRc = SQLITE_NOMEM;
res = 0;
** found in Fts3Expr.pPhrase->doclist.pList for each of the phrase
** expression nodes.
*/
-static void fts3EvalUpdateCounts(Fts3Expr *pExpr){
+static void fts3EvalUpdateCounts(Fts3Expr *pExpr, int nCol){
if( pExpr ){
Fts3Phrase *pPhrase = pExpr->pPhrase;
if( pPhrase && pPhrase->doclist.pList ){
int iCol = 0;
char *p = pPhrase->doclist.pList;
- assert( *p );
- while( 1 ){
+ do{
u8 c = 0;
int iCnt = 0;
while( 0xFE & (*p | c) ){
if( *p==0x00 ) break;
p++;
p += fts3GetVarint32(p, &iCol);
- }
+ }while( iCol<nCol );
}
- fts3EvalUpdateCounts(pExpr->pLeft);
- fts3EvalUpdateCounts(pExpr->pRight);
+ fts3EvalUpdateCounts(pExpr->pLeft, nCol);
+ fts3EvalUpdateCounts(pExpr->pRight, nCol);
}
}
for(p=pRoot; p; p=p->pLeft){
Fts3Expr *pE = (p->eType==FTSQUERY_PHRASE?p:p->pRight);
assert( pE->aMI==0 );
- pE->aMI = (u32 *)sqlite3_malloc(pTab->nColumn * 3 * sizeof(u32));
+ pE->aMI = (u32 *)sqlite3_malloc64(pTab->nColumn * 3 * sizeof(u32));
if( !pE->aMI ) return SQLITE_NOMEM;
memset(pE->aMI, 0, pTab->nColumn * 3 * sizeof(u32));
}
);
if( rc==SQLITE_OK && pCsr->isEof==0 ){
- fts3EvalUpdateCounts(pRoot);
+ fts3EvalUpdateCounts(pRoot, pTab->nColumn);
}
}
*/
#define FTS3_VARINT_MAX 10
+#define FTS3_BUFFER_PADDING 8
+
/*
** FTS4 virtual tables may maintain multiple indexes - one index of all terms
** in the document set and zero or more prefix indexes. All indexes are stored
#define POS_COLUMN (1) /* Column-list terminator */
#define POS_END (0) /* Position-list terminator */
+/*
+** The assert_fts3_nc() macro is similar to the assert() macro, except that it
+** is used for assert() conditions that are true only if it can be
+** guranteed that the database is not corrupt.
+*/
+#if defined(SQLITE_DEBUG) || defined(SQLITE_TEST)
+extern int sqlite3_fts3_may_be_corrupt;
+# define assert_fts3_nc(x) assert(sqlite3_fts3_may_be_corrupt || (x))
+#else
+# define assert_fts3_nc(x) assert(x)
+#endif
+
/*
** This section provides definitions to allow the
** FTS3 extension to be compiled outside of the
# define TESTONLY(X)
#endif
+#define LARGEST_INT64 (0xffffffff|(((i64)0x7fffffff)<<32))
+#define SMALLEST_INT64 (((i64)-1) - LARGEST_INT64)
+
#endif /* SQLITE_AMALGAMATION */
#ifdef SQLITE_DEBUG
char *zLanguageid; /* languageid=xxx option, or NULL */
int nAutoincrmerge; /* Value configured by 'automerge' */
u32 nLeafAdd; /* Number of leaf blocks added this trans */
+ int bLock; /* Used to prevent recursive content= tbls */
/* Precompiled statements used by the implementation. Each of these
** statements is run and reset within a single virtual table API call.
int mxSavepoint; /* Largest valid xSavepoint integer */
#endif
-#ifdef SQLITE_TEST
+#if defined(SQLITE_DEBUG) || defined(SQLITE_TEST)
/* True to disable the incremental doclist optimization. This is controled
** by special insert command 'test-no-incr-doclist'. */
int bNoIncrDoclist;
+
+ /* Number of segments in a level */
+ int nMergeCount;
#endif
};
+/* Macro to find the number of segments to merge */
+#if defined(SQLITE_DEBUG) || defined(SQLITE_TEST)
+# define MergeCount(P) ((P)->nMergeCount)
+#else
+# define MergeCount(P) FTS3_MERGE_COUNT
+#endif
+
/*
** When the core wants to read from the virtual table, it creates a
** virtual table cursor (an instance of the following structure) using
void sqlite3Fts3ErrMsg(char**,const char*,...);
int sqlite3Fts3PutVarint(char *, sqlite3_int64);
int sqlite3Fts3GetVarint(const char *, sqlite_int64 *);
+int sqlite3Fts3GetVarintU(const char *, sqlite_uint64 *);
+int sqlite3Fts3GetVarintBounded(const char*,const char*,sqlite3_int64*);
int sqlite3Fts3GetVarint32(const char *, int *);
int sqlite3Fts3VarintLen(sqlite3_uint64);
void sqlite3Fts3Dequote(char *);
);
void sqlite3Fts3ExprFree(Fts3Expr *);
#ifdef SQLITE_TEST
-int sqlite3Fts3ExprInitTestInterface(sqlite3 *db);
+int sqlite3Fts3ExprInitTestInterface(sqlite3 *db, Fts3Hash*);
int sqlite3Fts3InitTerm(sqlite3 *db);
#endif
char const *zFts3; /* Name of fts3 table */
int nDb; /* Result of strlen(zDb) */
int nFts3; /* Result of strlen(zFts3) */
- int nByte; /* Bytes of space to allocate here */
+ sqlite3_int64 nByte; /* Bytes of space to allocate here */
int rc; /* value returned by declare_vtab() */
Fts3auxTable *p; /* Virtual table object to return */
if( rc!=SQLITE_OK ) return rc;
nByte = sizeof(Fts3auxTable) + sizeof(Fts3Table) + nDb + nFts3 + 2;
- p = (Fts3auxTable *)sqlite3_malloc(nByte);
+ p = (Fts3auxTable *)sqlite3_malloc64(nByte);
if( !p ) return SQLITE_NOMEM;
memset(p, 0, nByte);
static int fts3auxGrowStatArray(Fts3auxCursor *pCsr, int nSize){
if( nSize>pCsr->nStat ){
struct Fts3auxColstats *aNew;
- aNew = (struct Fts3auxColstats *)sqlite3_realloc(pCsr->aStat,
+ aNew = (struct Fts3auxColstats *)sqlite3_realloc64(pCsr->aStat,
sizeof(struct Fts3auxColstats) * nSize
);
if( aNew==0 ) return SQLITE_NOMEM;
assert( (iEq==0 && iGe==-1) || (iEq==-1 && iGe==0) );
if( zStr ){
pCsr->filter.zTerm = sqlite3_mprintf("%s", zStr);
- pCsr->filter.nTerm = sqlite3_value_bytes(apVal[0]);
if( pCsr->filter.zTerm==0 ) return SQLITE_NOMEM;
+ pCsr->filter.nTerm = (int)strlen(pCsr->filter.zTerm);
}
}
if( iLe>=0 ){
pCsr->zStop = sqlite3_mprintf("%s", sqlite3_value_text(apVal[iLe]));
- pCsr->nStop = sqlite3_value_bytes(apVal[iLe]);
if( pCsr->zStop==0 ) return SQLITE_NOMEM;
+ pCsr->nStop = (int)strlen(pCsr->zStop);
}
if( iLangid>=0 ){
0, /* xRename */
0, /* xSavepoint */
0, /* xRelease */
- 0 /* xRollbackTo */
+ 0, /* xRollbackTo */
};
int rc; /* Return code */
** zero the memory before returning a pointer to it. If unsuccessful,
** return NULL.
*/
-static void *fts3MallocZero(int nByte){
- void *pRet = sqlite3_malloc(nByte);
+static void *fts3MallocZero(sqlite3_int64 nByte){
+ void *pRet = sqlite3_malloc64(nByte);
if( pRet ) memset(pRet, 0, nByte);
return pRet;
}
if( rc==SQLITE_OK ){
const char *zToken;
int nToken = 0, iStart = 0, iEnd = 0, iPosition = 0;
- int nByte; /* total space to allocate */
+ sqlite3_int64 nByte; /* total space to allocate */
rc = pModule->xNext(pCursor, &zToken, &nToken, &iStart, &iEnd, &iPosition);
if( rc==SQLITE_OK ){
** Enlarge a memory allocation. If an out-of-memory allocation occurs,
** then free the old allocation.
*/
-static void *fts3ReallocOrFree(void *pOrig, int nNew){
- void *pRet = sqlite3_realloc(pOrig, nNew);
+static void *fts3ReallocOrFree(void *pOrig, sqlite3_int64 nNew){
+ void *pRet = sqlite3_realloc64(pOrig, nNew);
if( !pRet ){
sqlite3_free(pOrig);
}
int nConsumed = 0;
pParse->nNest++;
rc = fts3ExprParse(pParse, zInput+1, nInput-1, ppExpr, &nConsumed);
- if( rc==SQLITE_OK && !*ppExpr ){ rc = SQLITE_DONE; }
*pnConsumed = (int)(zInput - z) + 1 + nConsumed;
return rc;
}else if( *zInput==')' ){
if( rc==SQLITE_OK ){
if( (eType==FTSQUERY_AND || eType==FTSQUERY_OR) ){
Fts3Expr **apLeaf;
- apLeaf = (Fts3Expr **)sqlite3_malloc(sizeof(Fts3Expr *) * nMaxDepth);
+ apLeaf = (Fts3Expr **)sqlite3_malloc64(sizeof(Fts3Expr *) * nMaxDepth);
if( 0==apLeaf ){
rc = SQLITE_NOMEM;
}else{
#include <stdio.h>
-/*
-** Function to query the hash-table of tokenizers (see README.tokenizers).
-*/
-static int queryTestTokenizer(
- sqlite3 *db,
- const char *zName,
- const sqlite3_tokenizer_module **pp
-){
- int rc;
- sqlite3_stmt *pStmt;
- const char zSql[] = "SELECT fts3_tokenizer(?)";
-
- *pp = 0;
- rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0);
- if( rc!=SQLITE_OK ){
- return rc;
- }
-
- sqlite3_bind_text(pStmt, 1, zName, -1, SQLITE_STATIC);
- if( SQLITE_ROW==sqlite3_step(pStmt) ){
- if( sqlite3_column_type(pStmt, 0)==SQLITE_BLOB ){
- memcpy((void *)pp, sqlite3_column_blob(pStmt, 0), sizeof(*pp));
- }
- }
-
- return sqlite3_finalize(pStmt);
-}
-
/*
** Return a pointer to a buffer containing a text representation of the
** expression passed as the first argument. The buffer is obtained from
**
** SELECT fts3_exprtest('simple', 'Bill col2:Bloggs', 'col1', 'col2');
*/
-static void fts3ExprTest(
+static void fts3ExprTestCommon(
+ int bRebalance,
sqlite3_context *context,
int argc,
sqlite3_value **argv
){
- sqlite3_tokenizer_module const *pModule = 0;
sqlite3_tokenizer *pTokenizer = 0;
int rc;
char **azCol = 0;
int ii;
Fts3Expr *pExpr;
char *zBuf = 0;
- sqlite3 *db = sqlite3_context_db_handle(context);
+ Fts3Hash *pHash = (Fts3Hash*)sqlite3_user_data(context);
+ const char *zTokenizer = 0;
+ char *zErr = 0;
if( argc<3 ){
sqlite3_result_error(context,
return;
}
- rc = queryTestTokenizer(db,
- (const char *)sqlite3_value_text(argv[0]), &pModule);
- if( rc==SQLITE_NOMEM ){
- sqlite3_result_error_nomem(context);
- goto exprtest_out;
- }else if( !pModule ){
- sqlite3_result_error(context, "No such tokenizer module", -1);
- goto exprtest_out;
- }
-
- rc = pModule->xCreate(0, 0, &pTokenizer);
- assert( rc==SQLITE_NOMEM || rc==SQLITE_OK );
- if( rc==SQLITE_NOMEM ){
- sqlite3_result_error_nomem(context);
- goto exprtest_out;
+ zTokenizer = (const char*)sqlite3_value_text(argv[0]);
+ rc = sqlite3Fts3InitTokenizer(pHash, zTokenizer, &pTokenizer, &zErr);
+ if( rc!=SQLITE_OK ){
+ if( rc==SQLITE_NOMEM ){
+ sqlite3_result_error_nomem(context);
+ }else{
+ sqlite3_result_error(context, zErr, -1);
+ }
+ sqlite3_free(zErr);
+ return;
}
- pTokenizer->pModule = pModule;
zExpr = (const char *)sqlite3_value_text(argv[1]);
nExpr = sqlite3_value_bytes(argv[1]);
nCol = argc-2;
- azCol = (char **)sqlite3_malloc(nCol*sizeof(char *));
+ azCol = (char **)sqlite3_malloc64(nCol*sizeof(char *));
if( !azCol ){
sqlite3_result_error_nomem(context);
goto exprtest_out;
azCol[ii] = (char *)sqlite3_value_text(argv[ii+2]);
}
- if( sqlite3_user_data(context) ){
+ if( bRebalance ){
char *zDummy = 0;
rc = sqlite3Fts3ExprParse(
pTokenizer, 0, azCol, 0, nCol, nCol, zExpr, nExpr, &pExpr, &zDummy
sqlite3Fts3ExprFree(pExpr);
exprtest_out:
- if( pModule && pTokenizer ){
- rc = pModule->xDestroy(pTokenizer);
+ if( pTokenizer ){
+ rc = pTokenizer->pModule->xDestroy(pTokenizer);
}
sqlite3_free(azCol);
}
+static void fts3ExprTest(
+ sqlite3_context *context,
+ int argc,
+ sqlite3_value **argv
+){
+ fts3ExprTestCommon(0, context, argc, argv);
+}
+static void fts3ExprTestRebalance(
+ sqlite3_context *context,
+ int argc,
+ sqlite3_value **argv
+){
+ fts3ExprTestCommon(1, context, argc, argv);
+}
+
/*
** Register the query expression parser test function fts3_exprtest()
** with database connection db.
*/
-int sqlite3Fts3ExprInitTestInterface(sqlite3* db){
+int sqlite3Fts3ExprInitTestInterface(sqlite3 *db, Fts3Hash *pHash){
int rc = sqlite3_create_function(
- db, "fts3_exprtest", -1, SQLITE_UTF8, 0, fts3ExprTest, 0, 0
+ db, "fts3_exprtest", -1, SQLITE_UTF8, (void*)pHash, fts3ExprTest, 0, 0
);
if( rc==SQLITE_OK ){
rc = sqlite3_create_function(db, "fts3_exprtest_rebalance",
- -1, SQLITE_UTF8, (void *)1, fts3ExprTest, 0, 0
+ -1, SQLITE_UTF8, (void*)pHash, fts3ExprTestRebalance, 0, 0
);
}
return rc;
/*
** Malloc and Free functions
*/
-static void *fts3HashMalloc(int n){
- void *p = sqlite3_malloc(n);
+static void *fts3HashMalloc(sqlite3_int64 n){
+ void *p = sqlite3_malloc64(n);
if( p ){
memset(p, 0, n);
}
if( argc>0 ){
n = strlen(argv[0])+1;
}
- p = (IcuTokenizer *)sqlite3_malloc(sizeof(IcuTokenizer)+n);
+ p = (IcuTokenizer *)sqlite3_malloc64(sizeof(IcuTokenizer)+n);
if( !p ){
return SQLITE_NOMEM;
}
nInput = strlen(zInput);
}
nChar = nInput+1;
- pCsr = (IcuCursor *)sqlite3_malloc(
+ pCsr = (IcuCursor *)sqlite3_malloc64(
sizeof(IcuCursor) + /* IcuCursor */
((nChar+3)&~3) * sizeof(UChar) + /* IcuCursor.aChar[] */
(nChar+1) * sizeof(int) /* IcuCursor.aOffset[] */
/*
** Allocate a two-slot MatchinfoBuffer object.
*/
-static MatchinfoBuffer *fts3MIBufferNew(int nElem, const char *zMatchinfo){
+static MatchinfoBuffer *fts3MIBufferNew(size_t nElem, const char *zMatchinfo){
MatchinfoBuffer *pRet;
- int nByte = sizeof(u32) * (2*nElem + 1) + sizeof(MatchinfoBuffer);
- int nStr = (int)strlen(zMatchinfo);
+ sqlite3_int64 nByte = sizeof(u32) * (2*(sqlite3_int64)nElem + 1)
+ + sizeof(MatchinfoBuffer);
+ sqlite3_int64 nStr = strlen(zMatchinfo);
- pRet = sqlite3_malloc(nByte + nStr+1);
+ pRet = sqlite3_malloc64(nByte + nStr+1);
if( pRet ){
memset(pRet, 0, nByte);
pRet->aMatchinfo[0] = (u8*)(&pRet->aMatchinfo[1]) - (u8*)pRet;
- pRet->aMatchinfo[1+nElem] = pRet->aMatchinfo[0] + sizeof(u32)*(nElem+1);
- pRet->nElem = nElem;
+ pRet->aMatchinfo[1+nElem] = pRet->aMatchinfo[0]
+ + sizeof(u32)*((int)nElem+1);
+ pRet->nElem = (int)nElem;
pRet->zMatchinfo = ((char*)pRet) + nByte;
memcpy(pRet->zMatchinfo, zMatchinfo, nStr+1);
pRet->aRef[0] = 1;
aOut = &p->aMatchinfo[p->nElem+2];
xRet = fts3MIBufferFree;
}else{
- aOut = (u32*)sqlite3_malloc(p->nElem * sizeof(u32));
+ aOut = (u32*)sqlite3_malloc64(p->nElem * sizeof(u32));
if( aOut ){
xRet = sqlite3_free;
if( p->bGlobal ) memcpy(aOut, &p->aMatchinfo[1], p->nElem*sizeof(u32));
char *pCsr = pPhrase->pTail;
int iCsr = pPhrase->iTail;
- while( iCsr<(iStart+pIter->nSnippet) ){
+ while( iCsr<(iStart+pIter->nSnippet) && iCsr>=iStart ){
int j;
- u64 mPhrase = (u64)1 << i;
+ u64 mPhrase = (u64)1 << (i%64);
u64 mPos = (u64)1 << (iCsr - iStart);
- assert( iCsr>=iStart );
+ assert( iCsr>=iStart && (iCsr - iStart)<=64 );
+ assert( i>=0 );
if( (mCover|mCovered)&mPhrase ){
iScore++;
}else{
int iFirst = 0;
pPhrase->pList = pCsr;
fts3GetDeltaPosition(&pCsr, &iFirst);
- assert( iFirst>=0 );
- pPhrase->pHead = pCsr;
- pPhrase->pTail = pCsr;
- pPhrase->iHead = iFirst;
- pPhrase->iTail = iFirst;
+ if( iFirst<0 ){
+ rc = FTS_CORRUPT_VTAB;
+ }else{
+ pPhrase->pHead = pCsr;
+ pPhrase->pTail = pCsr;
+ pPhrase->iHead = iFirst;
+ pPhrase->iTail = iFirst;
+ }
}else{
assert( rc!=SQLITE_OK || (
pPhrase->pList==0 && pPhrase->pHead==0 && pPhrase->pTail==0
int rc; /* Return Code */
int nList; /* Number of phrases in expression */
SnippetIter sIter; /* Iterates through snippet candidates */
- int nByte; /* Number of bytes of space to allocate */
+ sqlite3_int64 nByte; /* Number of bytes of space to allocate */
int iBestScore = -1; /* Best snippet score found so far */
int i; /* Loop counter */
** the required space using malloc().
*/
nByte = sizeof(SnippetPhrase) * nList;
- sIter.aPhrase = (SnippetPhrase *)sqlite3_malloc(nByte);
+ sIter.aPhrase = (SnippetPhrase *)sqlite3_malloc64(nByte);
if( !sIter.aPhrase ){
return SQLITE_NOMEM;
}
/* Set the *pmSeen output variable. */
for(i=0; i<nList; i++){
if( sIter.aPhrase[i].pHead ){
- *pmSeen |= (u64)1 << i;
+ *pmSeen |= (u64)1 << (i%64);
}
}
** appended data.
*/
if( pStr->n+nAppend+1>=pStr->nAlloc ){
- int nAlloc = pStr->nAlloc+nAppend+100;
- char *zNew = sqlite3_realloc(pStr->z, nAlloc);
+ sqlite3_int64 nAlloc = pStr->nAlloc+(sqlite3_int64)nAppend+100;
+ char *zNew = sqlite3_realloc64(pStr->z, nAlloc);
if( !zNew ){
return SQLITE_NOMEM;
}
for(nLeft=0; !(hlmask & ((u64)1 << nLeft)); nLeft++);
for(nRight=0; !(hlmask & ((u64)1 << (nSnippet-1-nRight))); nRight++);
+ assert( (nSnippet-1-nRight)<=63 && (nSnippet-1-nRight)>=0 );
nDesired = (nLeft-nRight)/2;
/* Ideally, the start of the snippet should be pushed forward in the
/*
** This function gathers 'y' or 'b' data for a single phrase.
*/
-static void fts3ExprLHits(
+static int fts3ExprLHits(
Fts3Expr *pExpr, /* Phrase expression node */
MatchInfo *p /* Matchinfo context */
){
if( *pIter!=0x01 ) break;
pIter++;
pIter += fts3GetVarint32(pIter, &iCol);
+ if( iCol>=p->nCol ) return FTS_CORRUPT_VTAB;
}
+ return SQLITE_OK;
}
/*
** Gather the results for matchinfo directives 'y' and 'b'.
*/
-static void fts3ExprLHitGather(
+static int fts3ExprLHitGather(
Fts3Expr *pExpr,
MatchInfo *p
){
+ int rc = SQLITE_OK;
assert( (pExpr->pLeft==0)==(pExpr->pRight==0) );
if( pExpr->bEof==0 && pExpr->iDocid==p->pCursor->iPrevId ){
if( pExpr->pLeft ){
- fts3ExprLHitGather(pExpr->pLeft, p);
- fts3ExprLHitGather(pExpr->pRight, p);
+ rc = fts3ExprLHitGather(pExpr->pLeft, p);
+ if( rc==SQLITE_OK ) rc = fts3ExprLHitGather(pExpr->pRight, p);
}else{
- fts3ExprLHits(pExpr, p);
+ rc = fts3ExprLHits(pExpr, p);
}
}
+ return rc;
}
/*
return SQLITE_ERROR;
}
-static int fts3MatchinfoSize(MatchInfo *pInfo, char cArg){
- int nVal; /* Number of integers output by cArg */
+static size_t fts3MatchinfoSize(MatchInfo *pInfo, char cArg){
+ size_t nVal; /* Number of integers output by cArg */
switch( cArg ){
case FTS3_MATCHINFO_NDOC:
Fts3Table *pTab,
sqlite3_stmt **ppStmt,
sqlite3_int64 *pnDoc,
- const char **paLen
+ const char **paLen,
+ const char **ppEnd
){
sqlite3_stmt *pStmt;
const char *a;
+ const char *pEnd;
sqlite3_int64 nDoc;
+ int n;
+
if( !*ppStmt ){
int rc = sqlite3Fts3SelectDoctotal(pTab, ppStmt);
pStmt = *ppStmt;
assert( sqlite3_data_count(pStmt)==1 );
+ n = sqlite3_column_bytes(pStmt, 0);
a = sqlite3_column_blob(pStmt, 0);
- a += sqlite3Fts3GetVarint(a, &nDoc);
- if( nDoc==0 ) return FTS_CORRUPT_VTAB;
- *pnDoc = (u32)nDoc;
+ if( a==0 ){
+ return FTS_CORRUPT_VTAB;
+ }
+ pEnd = a + n;
+ a += sqlite3Fts3GetVarintBounded(a, pEnd, &nDoc);
+ if( nDoc<=0 || a>pEnd ){
+ return FTS_CORRUPT_VTAB;
+ }
+ *pnDoc = nDoc;
if( paLen ) *paLen = a;
+ if( ppEnd ) *ppEnd = pEnd;
return SQLITE_OK;
}
int i;
int iCol;
int nToken = 0;
+ int rc = SQLITE_OK;
/* Allocate and populate the array of LcsIterator objects. The array
** contains one element for each matchable phrase in the query.
**/
- aIter = sqlite3_malloc(sizeof(LcsIterator) * pCsr->nPhrase);
+ aIter = sqlite3_malloc64(sizeof(LcsIterator) * pCsr->nPhrase);
if( !aIter ) return SQLITE_NOMEM;
memset(aIter, 0, sizeof(LcsIterator) * pCsr->nPhrase);
(void)fts3ExprIterate(pCsr->pExpr, fts3MatchinfoLcsCb, (void*)aIter);
int nLive = 0; /* Number of iterators in aIter not at EOF */
for(i=0; i<pInfo->nPhrase; i++){
- int rc;
LcsIterator *pIt = &aIter[i];
rc = sqlite3Fts3EvalPhrasePoslist(pCsr, pIt->pExpr, iCol, &pIt->pRead);
- if( rc!=SQLITE_OK ) return rc;
+ if( rc!=SQLITE_OK ) goto matchinfo_lcs_out;
if( pIt->pRead ){
pIt->iPos = pIt->iPosOffset;
- fts3LcsIteratorAdvance(&aIter[i]);
+ fts3LcsIteratorAdvance(pIt);
+ if( pIt->pRead==0 ){
+ rc = FTS_CORRUPT_VTAB;
+ goto matchinfo_lcs_out;
+ }
nLive++;
}
}
pInfo->aMatchinfo[iCol] = nLcs;
}
+ matchinfo_lcs_out:
sqlite3_free(aIter);
- return SQLITE_OK;
+ return rc;
}
/*
case FTS3_MATCHINFO_NDOC:
if( bGlobal ){
sqlite3_int64 nDoc = 0;
- rc = fts3MatchinfoSelectDoctotal(pTab, &pSelect, &nDoc, 0);
+ rc = fts3MatchinfoSelectDoctotal(pTab, &pSelect, &nDoc, 0, 0);
pInfo->aMatchinfo[0] = (u32)nDoc;
}
break;
if( bGlobal ){
sqlite3_int64 nDoc; /* Number of rows in table */
const char *a; /* Aggregate column length array */
+ const char *pEnd; /* First byte past end of length array */
- rc = fts3MatchinfoSelectDoctotal(pTab, &pSelect, &nDoc, &a);
+ rc = fts3MatchinfoSelectDoctotal(pTab, &pSelect, &nDoc, &a, &pEnd);
if( rc==SQLITE_OK ){
int iCol;
for(iCol=0; iCol<pInfo->nCol; iCol++){
u32 iVal;
sqlite3_int64 nToken;
a += sqlite3Fts3GetVarint(a, &nToken);
+ if( a>pEnd ){
+ rc = SQLITE_CORRUPT_VTAB;
+ break;
+ }
iVal = (u32)(((u32)(nToken&0xffffffff)+nDoc/2)/nDoc);
pInfo->aMatchinfo[iCol] = iVal;
}
if( rc==SQLITE_OK ){
int iCol;
const char *a = sqlite3_column_blob(pSelectDocsize, 0);
+ const char *pEnd = a + sqlite3_column_bytes(pSelectDocsize, 0);
for(iCol=0; iCol<pInfo->nCol; iCol++){
sqlite3_int64 nToken;
- a += sqlite3Fts3GetVarint(a, &nToken);
+ a += sqlite3Fts3GetVarintBounded(a, pEnd, &nToken);
+ if( a>pEnd ){
+ rc = SQLITE_CORRUPT_VTAB;
+ break;
+ }
pInfo->aMatchinfo[iCol] = (u32)nToken;
}
}
case FTS3_MATCHINFO_LHITS_BM:
case FTS3_MATCHINFO_LHITS: {
- int nZero = fts3MatchinfoSize(pInfo, zArg[i]) * sizeof(u32);
+ size_t nZero = fts3MatchinfoSize(pInfo, zArg[i]) * sizeof(u32);
memset(pInfo->aMatchinfo, 0, nZero);
- fts3ExprLHitGather(pCsr->pExpr, pInfo);
+ rc = fts3ExprLHitGather(pCsr->pExpr, pInfo);
break;
}
if( rc!=SQLITE_OK ) break;
if( bGlobal ){
if( pCsr->pDeferred ){
- rc = fts3MatchinfoSelectDoctotal(pTab, &pSelect, &pInfo->nDoc, 0);
+ rc = fts3MatchinfoSelectDoctotal(pTab, &pSelect, &pInfo->nDoc,0,0);
if( rc!=SQLITE_OK ) break;
}
rc = fts3ExprIterate(pExpr, fts3ExprGlobalHitsCb,(void*)pInfo);
** initialize those elements that are constant for every row.
*/
if( pCsr->pMIBuffer==0 ){
- int nMatchinfo = 0; /* Number of u32 elements in match-info */
+ size_t nMatchinfo = 0; /* Number of u32 elements in match-info */
int i; /* Used to iterate through zArg */
/* Determine the number of phrases in the query */
return;
}
+ /* Limit the snippet length to 64 tokens. */
+ if( nToken<-64 ) nToken = -64;
+ if( nToken>+64 ) nToken = +64;
+
for(nSnippet=1; 1; nSnippet++){
int iSnip; /* Loop counter 0..nSnippet-1 */
nTerm = pExpr->pPhrase->nToken;
if( pList ){
fts3GetDeltaPosition(&pList, &iPos);
- assert( iPos>=0 );
+ assert_fts3_nc( iPos>=0 );
}
for(iTerm=0; iTerm<nTerm; iTerm++){
if( rc!=SQLITE_OK ) goto offsets_out;
/* Allocate the array of TermOffset iterators. */
- sCtx.aTerm = (TermOffset *)sqlite3_malloc(sizeof(TermOffset)*nToken);
+ sCtx.aTerm = (TermOffset *)sqlite3_malloc64(sizeof(TermOffset)*nToken);
if( 0==sCtx.aTerm ){
rc = SQLITE_NOMEM;
goto offsets_out;
/* All offsets for this column have been gathered. */
rc = SQLITE_DONE;
}else{
- assert( iCurrent<=iMinPos );
+ assert_fts3_nc( iCurrent<=iMinPos );
if( 0==(0xFE&*pTerm->pList) ){
pTerm->pList = 0;
}else{
char const *zFts3; /* Name of fts3 table */
int nDb; /* Result of strlen(zDb) */
int nFts3; /* Result of strlen(zFts3) */
- int nByte; /* Bytes of space to allocate here */
+ sqlite3_int64 nByte; /* Bytes of space to allocate here */
int rc; /* value returned by declare_vtab() */
- Fts3termTable *p; /* Virtual table object to return */
+ Fts3termTable *p; /* Virtual table object to return */
int iIndex = 0;
UNUSED_PARAMETER(pCtx);
if( rc!=SQLITE_OK ) return rc;
nByte = sizeof(Fts3termTable) + sizeof(Fts3Table) + nDb + nFts3 + 2;
- p = (Fts3termTable *)sqlite3_malloc(nByte);
+ p = (Fts3termTable *)sqlite3_malloc64(nByte);
if( !p ) return SQLITE_NOMEM;
- memset(p, 0, nByte);
+ memset(p, 0, (size_t)nByte);
p->pFts3Tab = (Fts3Table *)&p[1];
p->pFts3Tab->zDb = (char *)&p->pFts3Tab[1];
0, /* xRename */
0, /* xSavepoint */
0, /* xRelease */
- 0 /* xRollbackTo */
+ 0, /* xRollbackTo */
};
int rc; /* Return code */
}else{
/* Advance to the end of the token */
const char *pToken = p;
- int nToken;
+ sqlite3_int64 nToken;
while( p<pEnd && testIsTokenChar(*p) ) p++;
- nToken = (int)(p-pToken);
+ nToken = (sqlite3_int64)(p-pToken);
/* Copy the token into the buffer */
if( nToken>pCsr->nBuffer ){
sqlite3_free(pCsr->aBuffer);
- pCsr->aBuffer = sqlite3_malloc(nToken);
+ pCsr->aBuffer = sqlite3_malloc64(nToken);
}
if( pCsr->aBuffer==0 ){
rc = SQLITE_NOMEM;
pCsr->iInput = (int)(p - pCsr->aInput);
*ppToken = pCsr->aBuffer;
- *pnBytes = nToken;
+ *pnBytes = (int)nToken;
*piStartOffset = (int)(pToken - pCsr->aInput);
*piEndOffset = (int)(p - pCsr->aInput);
*piPosition = pCsr->iToken;
** End of tokenizer code.
**************************************************************************/
+/*
+** sqlite3_fts3_may_be_corrupt BOOLEAN
+**
+** Set or clear the global "may-be-corrupt" flag. Return the old value.
+*/
+static int SQLITE_TCLAPI fts3_may_be_corrupt(
+ void * clientData,
+ Tcl_Interp *interp,
+ int objc,
+ Tcl_Obj *CONST objv[]
+){
+ int bOld = sqlite3_fts3_may_be_corrupt;
+
+ if( objc!=2 && objc!=1 ){
+ Tcl_WrongNumArgs(interp, 1, objv, "?BOOLEAN?");
+ return TCL_ERROR;
+ }
+ if( objc==2 ){
+ int bNew;
+ if( Tcl_GetBooleanFromObj(interp, objv[1], &bNew) ) return TCL_ERROR;
+ sqlite3_fts3_may_be_corrupt = bNew;
+ }
+
+ Tcl_SetObjResult(interp, Tcl_NewIntObj(bOld));
+ return TCL_OK;
+}
+
int Sqlitetestfts3_Init(Tcl_Interp *interp){
Tcl_CreateObjCommand(interp, "fts3_near_match", fts3_near_match_cmd, 0, 0);
Tcl_CreateObjCommand(interp,
Tcl_CreateObjCommand(
interp, "fts3_test_tokenizer", fts3_test_tokenizer_cmd, 0, 0
);
-
Tcl_CreateObjCommand(
interp, "fts3_test_varint", fts3_test_varint_cmd, 0, 0
);
+ Tcl_CreateObjCommand(
+ interp, "sqlite3_fts3_may_be_corrupt", fts3_may_be_corrupt, 0, 0
+ );
return TCL_OK;
}
#endif /* SQLITE_ENABLE_FTS3 || SQLITE_ENABLE_FTS4 */
nByte += (int)(strlen(argv[i]) + 1);
}
- *pazDequote = azDequote = sqlite3_malloc(sizeof(char *)*argc + nByte);
+ *pazDequote = azDequote = sqlite3_malloc64(sizeof(char *)*argc + nByte);
if( azDequote==0 ){
rc = SQLITE_NOMEM;
}else{
if( idxNum==1 ){
const char *zByte = (const char *)sqlite3_value_text(apVal[0]);
int nByte = sqlite3_value_bytes(apVal[0]);
- pCsr->zInput = sqlite3_malloc(nByte+1);
+ pCsr->zInput = sqlite3_malloc64(nByte+1);
if( pCsr->zInput==0 ){
rc = SQLITE_NOMEM;
}else{
0, /* xRename */
0, /* xSavepoint */
0, /* xRelease */
- 0 /* xRollbackTo */
+ 0, /* xRollbackTo */
};
int rc; /* Return code */
return isEnabled;
}
+/*
+** The real sqlite3_value_frombind() implementation was not added
+** until version 3.28.0 of the SQLite core. This fake version facilitates
+** testing.
+*/
+static int bFrombindTrue = 0;
+static int sqlite3_value_frombind(sqlite3_value *NotUsed){
+ (void)NotUsed;
+ return bFrombindTrue;
+}
+
/*
** Implementation of the SQL scalar function for accessing the underlying
** hash table. This function may be called as follows:
nName = sqlite3_value_bytes(argv[0])+1;
if( argc==2 ){
- if( fts3TokenizerEnabled(context) ){
+ if( fts3TokenizerEnabled(context) || sqlite3_value_frombind(argv[1]) ){
void *pOld;
int n = sqlite3_value_bytes(argv[1]);
if( zName==0 || n!=sizeof(pPtr) ){
return;
}
}
- sqlite3_result_blob(context, (void *)&pPtr, sizeof(pPtr), SQLITE_TRANSIENT);
+ if( fts3TokenizerEnabled(context) || sqlite3_value_frombind(argv[0]) ){
+ sqlite3_result_blob(context, (void *)&pPtr, sizeof(pPtr), SQLITE_TRANSIENT);
+ }
}
int sqlite3Fts3IsIdChar(char c){
int iArg = 0;
z = &z[n+1];
while( z<zEnd && (NULL!=(z = (char *)sqlite3Fts3NextToken(z, &n))) ){
- int nNew = sizeof(char *)*(iArg+1);
- char const **aNew = (const char **)sqlite3_realloc((void *)aArg, nNew);
+ sqlite3_int64 nNew = sizeof(char *)*(iArg+1);
+ char const **aNew = (const char **)sqlite3_realloc64((void *)aArg, nNew);
if( !aNew ){
sqlite3_free(zCopy);
sqlite3_free((void *)aArg);
sqlite3_bind_text(pStmt, 1, zName, -1, SQLITE_STATIC);
if( SQLITE_ROW==sqlite3_step(pStmt) ){
- if( sqlite3_column_type(pStmt, 0)==SQLITE_BLOB ){
+ if( sqlite3_column_type(pStmt, 0)==SQLITE_BLOB
+ && sqlite3_column_bytes(pStmt, 0)==sizeof(*pp)
+ ){
memcpy((void *)pp, sqlite3_column_blob(pStmt, 0), sizeof(*pp));
}
}
UNUSED_PARAMETER(argv);
/* Test the query function */
+ bFrombindTrue = 1;
sqlite3Fts3SimpleTokenizerModule(&p1);
rc = queryTokenizer(db, "simple", &p2);
assert( rc==SQLITE_OK );
assert( rc==SQLITE_OK );
assert( p2==p1 );
}
+ bFrombindTrue = 0;
sqlite3_result_text(context, "ok", -1, SQLITE_STATIC);
}
){
int rc = SQLITE_OK;
void *p = (void *)pHash;
- const int any = SQLITE_ANY;
+ const int any = SQLITE_UTF8|SQLITE_DIRECTONLY;
#ifdef SQLITE_TEST
char *zTest = 0;
struct unicode_tokenizer {
sqlite3_tokenizer base;
- int bRemoveDiacritic;
+ int eRemoveDiacritic;
int nException;
int *aiException;
};
int *aNew; /* New aiException[] array */
int nNew; /* Number of valid entries in array aNew[] */
- aNew = sqlite3_realloc(p->aiException, (p->nException+nEntry)*sizeof(int));
+ aNew = sqlite3_realloc64(p->aiException,(p->nException+nEntry)*sizeof(int));
if( aNew==0 ) return SQLITE_NOMEM;
nNew = p->nException;
pNew = (unicode_tokenizer *) sqlite3_malloc(sizeof(unicode_tokenizer));
if( pNew==NULL ) return SQLITE_NOMEM;
memset(pNew, 0, sizeof(unicode_tokenizer));
- pNew->bRemoveDiacritic = 1;
+ pNew->eRemoveDiacritic = 1;
for(i=0; rc==SQLITE_OK && i<nArg; i++){
const char *z = azArg[i];
int n = (int)strlen(z);
if( n==19 && memcmp("remove_diacritics=1", z, 19)==0 ){
- pNew->bRemoveDiacritic = 1;
+ pNew->eRemoveDiacritic = 1;
}
else if( n==19 && memcmp("remove_diacritics=0", z, 19)==0 ){
- pNew->bRemoveDiacritic = 0;
+ pNew->eRemoveDiacritic = 0;
+ }
+ else if( n==19 && memcmp("remove_diacritics=2", z, 19)==0 ){
+ pNew->eRemoveDiacritic = 2;
}
else if( n>=11 && memcmp("tokenchars=", z, 11)==0 ){
rc = unicodeAddExceptions(pNew, 1, &z[11], n-11);
/* Grow the output buffer if required. */
if( (zOut-pCsr->zToken)>=(pCsr->nAlloc-4) ){
- char *zNew = sqlite3_realloc(pCsr->zToken, pCsr->nAlloc+64);
+ char *zNew = sqlite3_realloc64(pCsr->zToken, pCsr->nAlloc+64);
if( !zNew ) return SQLITE_NOMEM;
zOut = &zNew[zOut - pCsr->zToken];
pCsr->zToken = zNew;
/* Write the folded case of the last character read to the output */
zEnd = z;
- iOut = sqlite3FtsUnicodeFold((int)iCode, p->bRemoveDiacritic);
+ iOut = sqlite3FtsUnicodeFold((int)iCode, p->eRemoveDiacritic);
if( iOut ){
WRITE_UTF8(zOut, iOut);
}
/*
-** 2012 May 25
+** 2012-05-25
**
** The author disclaims copyright to this source code. In place of
** a legal notice, here is a blessing:
** E"). The resuls of passing a codepoint that corresponds to an
** uppercase letter are undefined.
*/
-static int remove_diacritic(int c){
+static int remove_diacritic(int c, int bComplex){
unsigned short aDia[] = {
0, 1797, 1848, 1859, 1891, 1928, 1940, 1995,
2024, 2040, 2060, 2110, 2168, 2206, 2264, 2286,
2344, 2383, 2472, 2488, 2516, 2596, 2668, 2732,
2782, 2842, 2894, 2954, 2984, 3000, 3028, 3336,
- 3456, 3696, 3712, 3728, 3744, 3896, 3912, 3928,
- 3968, 4008, 4040, 4106, 4138, 4170, 4202, 4234,
- 4266, 4296, 4312, 4344, 4408, 4424, 4472, 4504,
- 6148, 6198, 6264, 6280, 6360, 6429, 6505, 6529,
- 61448, 61468, 61534, 61592, 61642, 61688, 61704, 61726,
- 61784, 61800, 61836, 61880, 61914, 61948, 61998, 62122,
- 62154, 62200, 62218, 62302, 62364, 62442, 62478, 62536,
- 62554, 62584, 62604, 62640, 62648, 62656, 62664, 62730,
- 62924, 63050, 63082, 63274, 63390,
+ 3456, 3696, 3712, 3728, 3744, 3766, 3832, 3896,
+ 3912, 3928, 3944, 3968, 4008, 4040, 4056, 4106,
+ 4138, 4170, 4202, 4234, 4266, 4296, 4312, 4344,
+ 4408, 4424, 4442, 4472, 4488, 4504, 6148, 6198,
+ 6264, 6280, 6360, 6429, 6505, 6529, 61448, 61468,
+ 61512, 61534, 61592, 61610, 61642, 61672, 61688, 61704,
+ 61726, 61784, 61800, 61816, 61836, 61880, 61896, 61914,
+ 61948, 61998, 62062, 62122, 62154, 62184, 62200, 62218,
+ 62252, 62302, 62364, 62410, 62442, 62478, 62536, 62554,
+ 62584, 62604, 62640, 62648, 62656, 62664, 62730, 62766,
+ 62830, 62890, 62924, 62974, 63032, 63050, 63082, 63118,
+ 63182, 63242, 63274, 63310, 63368, 63390,
};
- char aChar[] = {
- '\0', 'a', 'c', 'e', 'i', 'n', 'o', 'u', 'y', 'y', 'a', 'c',
- 'd', 'e', 'e', 'g', 'h', 'i', 'j', 'k', 'l', 'n', 'o', 'r',
- 's', 't', 'u', 'u', 'w', 'y', 'z', 'o', 'u', 'a', 'i', 'o',
- 'u', 'g', 'k', 'o', 'j', 'g', 'n', 'a', 'e', 'i', 'o', 'r',
- 'u', 's', 't', 'h', 'a', 'e', 'o', 'y', '\0', '\0', '\0', '\0',
- '\0', '\0', '\0', '\0', 'a', 'b', 'd', 'd', 'e', 'f', 'g', 'h',
- 'h', 'i', 'k', 'l', 'l', 'm', 'n', 'p', 'r', 'r', 's', 't',
- 'u', 'v', 'w', 'w', 'x', 'y', 'z', 'h', 't', 'w', 'y', 'a',
- 'e', 'i', 'o', 'u', 'y',
+#define HIBIT ((unsigned char)0x80)
+ unsigned char aChar[] = {
+ '\0', 'a', 'c', 'e', 'i', 'n',
+ 'o', 'u', 'y', 'y', 'a', 'c',
+ 'd', 'e', 'e', 'g', 'h', 'i',
+ 'j', 'k', 'l', 'n', 'o', 'r',
+ 's', 't', 'u', 'u', 'w', 'y',
+ 'z', 'o', 'u', 'a', 'i', 'o',
+ 'u', 'u'|HIBIT, 'a'|HIBIT, 'g', 'k', 'o',
+ 'o'|HIBIT, 'j', 'g', 'n', 'a'|HIBIT, 'a',
+ 'e', 'i', 'o', 'r', 'u', 's',
+ 't', 'h', 'a', 'e', 'o'|HIBIT, 'o',
+ 'o'|HIBIT, 'y', '\0', '\0', '\0', '\0',
+ '\0', '\0', '\0', '\0', 'a', 'b',
+ 'c'|HIBIT, 'd', 'd', 'e'|HIBIT, 'e', 'e'|HIBIT,
+ 'f', 'g', 'h', 'h', 'i', 'i'|HIBIT,
+ 'k', 'l', 'l'|HIBIT, 'l', 'm', 'n',
+ 'o'|HIBIT, 'p', 'r', 'r'|HIBIT, 'r', 's',
+ 's'|HIBIT, 't', 'u', 'u'|HIBIT, 'v', 'w',
+ 'w', 'x', 'y', 'z', 'h', 't',
+ 'w', 'y', 'a', 'a'|HIBIT, 'a'|HIBIT, 'a'|HIBIT,
+ 'e', 'e'|HIBIT, 'e'|HIBIT, 'i', 'o', 'o'|HIBIT,
+ 'o'|HIBIT, 'o'|HIBIT, 'u', 'u'|HIBIT, 'u'|HIBIT, 'y',
};
unsigned int key = (((unsigned int)c)<<3) | 0x00000007;
}
}
assert( key>=aDia[iRes] );
- return ((c > (aDia[iRes]>>3) + (aDia[iRes]&0x07)) ? c : (int)aChar[iRes]);
+ if( bComplex==0 && (aChar[iRes] & 0x80) ) return c;
+ return (c > (aDia[iRes]>>3) + (aDia[iRes]&0x07)) ? c : ((int)aChar[iRes] & 0x7F);
}
unsigned int mask1 = 0x000361F8;
if( c<768 || c>817 ) return 0;
return (c < 768+32) ?
- (mask0 & (1 << (c-768))) :
- (mask1 & (1 << (c-768-32)));
+ (mask0 & ((unsigned int)1 << (c-768))) :
+ (mask1 & ((unsigned int)1 << (c-768-32)));
}
** The results are undefined if the value passed to this function
** is less than zero.
*/
-int sqlite3FtsUnicodeFold(int c, int bRemoveDiacritic){
+int sqlite3FtsUnicodeFold(int c, int eRemoveDiacritic){
/* Each entry in the following array defines a rule for folding a range
** of codepoints to lower case. The rule applies to a range of nRange
** codepoints starting at codepoint iCode.
assert( ret>0 );
}
- if( bRemoveDiacritic ) ret = remove_diacritic(ret);
+ if( eRemoveDiacritic ){
+ ret = remove_diacritic(ret, eRemoveDiacritic==2);
+ }
}
else if( c>=66560 && c<66600 ){
#include <string.h>
#include <assert.h>
#include <stdlib.h>
-
+#include <stdio.h>
#define FTS_MAX_APPENDABLE_HEIGHT 16
#endif
/*
-** The two values that may be meaningfully bound to the :1 parameter in
+** The values that may be meaningfully bound to the :1 parameter in
** statements SQL_REPLACE_STAT and SQL_SELECT_STAT.
*/
#define FTS_STAT_DOCTOTAL 0
** returns zero rows. */
/* 28 */ "SELECT level, count(*) AS cnt FROM %Q.'%q_segdir' "
" GROUP BY level HAVING cnt>=?"
- " ORDER BY (level %% 1024) ASC LIMIT 1",
+ " ORDER BY (level %% 1024) ASC, 2 DESC LIMIT 1",
/* Estimate the upper limit on the number of leaf nodes in a new segment
** created by merging the oldest :2 segments from absolute level :1. See
pStmt = p->aStmt[eStmt];
if( !pStmt ){
+ int f = SQLITE_PREPARE_PERSISTENT|SQLITE_PREPARE_NO_VTAB;
char *zSql;
if( eStmt==SQL_CONTENT_INSERT ){
zSql = sqlite3_mprintf(azSql[eStmt], p->zDb, p->zName, p->zWriteExprlist);
}else if( eStmt==SQL_SELECT_CONTENT_BY_ROWID ){
+ f &= ~SQLITE_PREPARE_NO_VTAB;
zSql = sqlite3_mprintf(azSql[eStmt], p->zReadExprlist);
}else{
zSql = sqlite3_mprintf(azSql[eStmt], p->zDb, p->zName);
if( !zSql ){
rc = SQLITE_NOMEM;
}else{
- rc = sqlite3_prepare_v3(p->db, zSql, -1, SQLITE_PREPARE_PERSISTENT,
- &pStmt, NULL);
+ rc = sqlite3_prepare_v3(p->db, zSql, -1, f, &pStmt, NULL);
sqlite3_free(zSql);
assert( rc==SQLITE_OK || pStmt==0 );
p->aStmt[eStmt] = pStmt;
int iLevel /* Level of segments */
){
sqlite3_int64 iBase; /* First absolute level for iLangid/iIndex */
- assert( iLangid>=0 );
+ assert_fts3_nc( iLangid>=0 );
assert( p->nIndex>0 );
assert( iIndex>=0 && iIndex<p->nIndex );
assert( !p || p->iLastDocid<=iDocid );
if( !p || p->iLastDocid!=iDocid ){
- sqlite3_int64 iDelta = iDocid - (p ? p->iLastDocid : 0);
+ u64 iDelta = (u64)iDocid - (u64)(p ? p->iLastDocid : 0);
if( p ){
assert( p->nData<p->nSpace );
assert( p->aData[p->nData]==0 );
** segment and allocate (newly freed) index 0 at level iLevel. Otherwise,
** if iNext is less than FTS3_MERGE_COUNT, allocate index iNext.
*/
- if( iNext>=FTS3_MERGE_COUNT ){
+ if( iNext>=MergeCount(p) ){
fts3LogMerge(16, getAbsoluteLevel(p, iLangid, iIndex, iLevel));
rc = fts3SegmentMerge(p, iLangid, iIndex, iLevel);
*piIdx = 0;
}
*paBlob = aByte;
}
+ }else if( rc==SQLITE_ERROR ){
+ rc = FTS_CORRUPT_VTAB;
}
return rc;
/* If iCurrentBlock>=iLeafEndBlock, this is an EOF condition. All leaf
** blocks have already been traversed. */
- assert( pReader->iCurrentBlock<=pReader->iLeafEndBlock );
+#ifdef CORRUPT_DB
+ assert( pReader->iCurrentBlock<=pReader->iLeafEndBlock || CORRUPT_DB );
+#endif
if( pReader->iCurrentBlock>=pReader->iLeafEndBlock ){
return SQLITE_OK;
}
pNext += fts3GetVarint32(pNext, &nSuffix);
if( nSuffix<=0
|| (&pReader->aNode[pReader->nNode] - pNext)<nSuffix
- || nPrefix>pReader->nTermAlloc
+ || nPrefix>pReader->nTerm
){
return FTS_CORRUPT_VTAB;
}
}else{
rc = fts3SegReaderRequire(pReader, p, FTS3_VARINT_MAX);
if( rc==SQLITE_OK ){
- sqlite3_int64 iDelta;
- pReader->pOffsetList = p + sqlite3Fts3GetVarint(p, &iDelta);
+ u64 iDelta;
+ pReader->pOffsetList = p + sqlite3Fts3GetVarintU(p, &iDelta);
if( pTab->bDescIdx ){
- pReader->iDocid -= iDelta;
+ pReader->iDocid = (i64)((u64)pReader->iDocid - iDelta);
}else{
- pReader->iDocid += iDelta;
+ pReader->iDocid = (i64)((u64)pReader->iDocid + iDelta);
}
}
}
}
- return SQLITE_OK;
+ return rc;
}
Fts3SegReader *pReader; /* Newly allocated SegReader object */
int nExtra = 0; /* Bytes to allocate segment root node */
- assert( iStartLeaf<=iEndLeaf );
+ assert( zRoot!=0 || nRoot==0 );
+#ifdef CORRUPT_DB
+ assert( zRoot!=0 || CORRUPT_DB );
+#endif
+
if( iStartLeaf==0 ){
+ if( iEndLeaf!=0 ) return FTS_CORRUPT_VTAB;
nExtra = nRoot + FTS3_NODE_PADDING;
}
pReader->aNode = (char *)&pReader[1];
pReader->rootOnly = 1;
pReader->nNode = nRoot;
- memcpy(pReader->aNode, zRoot, nRoot);
+ if( nRoot ) memcpy(pReader->aNode, zRoot, nRoot);
memset(&pReader->aNode[nRoot], 0, FTS3_NODE_PADDING);
}else{
pReader->iCurrentBlock = iStartLeaf-1;
}
if( nElem>0 ){
- int nByte = sizeof(Fts3SegReader) + (nElem+1)*sizeof(Fts3HashElem *);
- pReader = (Fts3SegReader *)sqlite3_malloc(nByte);
+ sqlite3_int64 nByte;
+ nByte = sizeof(Fts3SegReader) + (nElem+1)*sizeof(Fts3HashElem *);
+ pReader = (Fts3SegReader *)sqlite3_malloc64(nByte);
if( !pReader ){
rc = SQLITE_NOMEM;
}else{
sqlite3_bind_blob(pStmt, 2, z, n, SQLITE_STATIC);
sqlite3_step(pStmt);
rc = sqlite3_reset(pStmt);
+ sqlite3_bind_null(pStmt, 2);
}
return rc;
}
sqlite3_bind_blob(pStmt, 6, zRoot, nRoot, SQLITE_STATIC);
sqlite3_step(pStmt);
rc = sqlite3_reset(pStmt);
+ sqlite3_bind_null(pStmt, 6);
}
return rc;
}
nPrefix = fts3PrefixCompress(pTree->zTerm, pTree->nTerm, zTerm, nTerm);
nSuffix = nTerm-nPrefix;
+ /* If nSuffix is zero or less, then zTerm/nTerm must be a prefix of
+ ** pWriter->zTerm/pWriter->nTerm. i.e. must be equal to or less than when
+ ** compared with BINARY collation. This indicates corruption. */
+ if( nSuffix<=0 ) return FTS_CORRUPT_VTAB;
+
nReq += sqlite3Fts3VarintLen(nPrefix)+sqlite3Fts3VarintLen(nSuffix)+nSuffix;
if( nReq<=p->nNodeSize || !pTree->zTerm ){
nPrefix = fts3PrefixCompress(pWriter->zTerm, pWriter->nTerm, zTerm, nTerm);
nSuffix = nTerm-nPrefix;
+ /* If nSuffix is zero or less, then zTerm/nTerm must be a prefix of
+ ** pWriter->zTerm/pWriter->nTerm. i.e. must be equal to or less than when
+ ** compared with BINARY collation. This indicates corruption. */
+ if( nSuffix<=0 ) return FTS_CORRUPT_VTAB;
+
/* Figure out how many bytes are required by this new entry */
nReq = sqlite3Fts3VarintLen(nPrefix) + /* varint containing prefix size */
sqlite3Fts3VarintLen(nSuffix) + /* varint containing suffix size */
int rc;
/* The current leaf node is full. Write it out to the database. */
+ if( pWriter->iFree==LARGEST_INT64 ) return FTS_CORRUPT_VTAB;
rc = fts3WriteSegment(p, pWriter->iFree++, pWriter->aData, nData);
if( rc!=SQLITE_OK ) return rc;
p->nLeafAdd++;
/* Append the prefix-compressed term and doclist to the buffer. */
nData += sqlite3Fts3PutVarint(&pWriter->aData[nData], nPrefix);
nData += sqlite3Fts3PutVarint(&pWriter->aData[nData], nSuffix);
+ assert( nSuffix>0 );
memcpy(&pWriter->aData[nData], &zTerm[nPrefix], nSuffix);
nData += nSuffix;
nData += sqlite3Fts3PutVarint(&pWriter->aData[nData], nDoclist);
+ assert( nDoclist>0 );
memcpy(&pWriter->aData[nData], aDoclist, nDoclist);
pWriter->nData = nData + nDoclist;
pWriter->zTerm = zNew;
}
assert( pWriter->zTerm==pWriter->zMalloc );
+ assert( nTerm>0 );
memcpy(pWriter->zTerm, zTerm, nTerm);
}else{
pWriter->zTerm = (char *)zTerm;
nList -= (int)(p - pList);
pList = p;
- if( nList==0 ){
+ if( nList<=0 ){
break;
}
p = &pList[1];
p += fts3GetVarint32(p, &iCurrent);
}
- if( bZero && &pList[nList]!=pEnd ){
+ if( bZero && (pEnd - &pList[nList])>0){
memset(&pList[nList], 0, pEnd - &pList[nList]);
}
*ppList = pList;
pMsr->aBuffer = pNew;
}
+ assert( nList>0 );
memcpy(pMsr->aBuffer, pList, nList);
return SQLITE_OK;
}
** doclist. */
sqlite3_int64 iDelta;
if( p->bDescIdx && nDoclist>0 ){
- iDelta = iPrev - iDocid;
+ if( iPrev<=iDocid ) return FTS_CORRUPT_VTAB;
+ iDelta = (i64)((u64)iPrev - (u64)iDocid);
}else{
- iDelta = iDocid - iPrev;
+ if( nDoclist>0 && iPrev>=iDocid ) return FTS_CORRUPT_VTAB;
+ iDelta = (i64)((u64)iDocid - (u64)iPrev);
}
- assert( iDelta>0 || (nDoclist==0 && iDelta==iDocid) );
- assert( nDoclist>0 || iDelta==iDocid );
nByte = sqlite3Fts3VarintLen(iDelta) + (isRequirePos?nList+1:0);
if( nDoclist+nByte>pCsr->nBuffer ){
if( rc!=SQLITE_OK ) goto finished;
assert( csr.nSegment>0 );
- assert( iNewLevel>=getAbsoluteLevel(p, iLangid, iIndex, 0) );
- assert( iNewLevel<getAbsoluteLevel(p, iLangid, iIndex,FTS3_SEGDIR_MAXLEVEL) );
+ assert_fts3_nc( iNewLevel>=getAbsoluteLevel(p, iLangid, iIndex, 0) );
+ assert_fts3_nc(
+ iNewLevel<getAbsoluteLevel(p, iLangid, iIndex,FTS3_SEGDIR_MAXLEVEL)
+ );
memset(&filter, 0, sizeof(Fts3SegFilter));
filter.flags = FTS3_SEGMENT_REQUIRE_POS;
csr.zTerm, csr.nTerm, csr.aDoclist, csr.nDoclist);
}
if( rc!=SQLITE_OK ) goto finished;
- assert( pWriter || bIgnoreEmpty );
+ assert_fts3_nc( pWriter || bIgnoreEmpty );
if( iLevel!=FTS3_SEGCURSOR_PENDING ){
rc = fts3DeleteSegdir(
const char *zBuf, /* The BLOB containing the varints */
int nBuf /* size of the BLOB */
){
- int i, j;
- UNUSED_PARAMETER(nBuf);
- for(i=j=0; i<N; i++){
- sqlite3_int64 x;
- j += sqlite3Fts3GetVarint(&zBuf[j], &x);
- assert(j<=nBuf);
- a[i] = (u32)(x & 0xffffffff);
+ int i = 0;
+ if( nBuf && (zBuf[nBuf-1]&0x80)==0 ){
+ int j;
+ for(i=j=0; i<N && j<nBuf; i++){
+ sqlite3_int64 x;
+ j += sqlite3Fts3GetVarint(&zBuf[j], &x);
+ a[i] = (u32)(x & 0xffffffff);
+ }
}
+ while( i<N ) a[i++] = 0;
}
/*
int rc; /* Result code from subfunctions */
if( *pRC ) return;
- pBlob = sqlite3_malloc( 10*p->nColumn );
+ pBlob = sqlite3_malloc64( 10*(sqlite3_int64)p->nColumn );
if( pBlob==0 ){
*pRC = SQLITE_NOMEM;
return;
const int nStat = p->nColumn+2;
if( *pRC ) return;
- a = sqlite3_malloc( (sizeof(u32)+10)*nStat );
+ a = sqlite3_malloc64( (sizeof(u32)+10)*(sqlite3_int64)nStat );
if( a==0 ){
*pRC = SQLITE_NOMEM;
return;
sqlite3_bind_blob(pStmt, 2, pBlob, nBlob, SQLITE_STATIC);
sqlite3_step(pStmt);
*pRC = sqlite3_reset(pStmt);
+ sqlite3_bind_null(pStmt, 2);
sqlite3_free(a);
}
int rc;
sqlite3_stmt *pAllLangid = 0;
- rc = fts3SqlStmt(p, SQL_SELECT_ALL_LANGID, &pAllLangid, 0);
+ rc = sqlite3Fts3PendingTermsFlush(p);
+ if( rc==SQLITE_OK ){
+ rc = fts3SqlStmt(p, SQL_SELECT_ALL_LANGID, &pAllLangid, 0);
+ }
if( rc==SQLITE_OK ){
int rc2;
sqlite3_bind_int(pAllLangid, 1, p->iPrevLangid);
}
sqlite3Fts3SegmentsClose(p);
- sqlite3Fts3PendingTermsClear(p);
return (rc==SQLITE_OK && bReturnDone && bSeenDone) ? SQLITE_DONE : rc;
}
}
if( rc==SQLITE_OK ){
- int nByte = sizeof(u32) * (p->nColumn+1)*3;
- aSz = (u32 *)sqlite3_malloc(nByte);
+ sqlite3_int64 nByte = sizeof(u32) * ((sqlite3_int64)p->nColumn+1)*3;
+ aSz = (u32 *)sqlite3_malloc64(nByte);
if( aSz==0 ){
rc = SQLITE_NOMEM;
}else{
){
int rc; /* Return Code */
sqlite3_stmt *pStmt = 0; /* Statement used to read %_segdir entry */
- int nByte; /* Bytes allocated at pCsr->apSegment[] */
+ sqlite3_int64 nByte; /* Bytes allocated at pCsr->apSegment[] */
/* Allocate space for the Fts3MultiSegReader.aCsr[] array */
memset(pCsr, 0, sizeof(*pCsr));
nByte = sizeof(Fts3SegReader *) * nSeg;
- pCsr->apSegment = (Fts3SegReader **)sqlite3_malloc(nByte);
+ pCsr->apSegment = (Fts3SegReader **)sqlite3_malloc64(nByte);
if( pCsr->apSegment==0 ){
rc = SQLITE_NOMEM;
}
p->iOff += fts3GetVarint32(&p->aNode[p->iOff], &nSuffix);
- if( nPrefix>p->iOff || nSuffix>p->nNode-p->iOff ){
- return SQLITE_CORRUPT_VTAB;
+ if( nPrefix>p->term.n || nSuffix>p->nNode-p->iOff || nSuffix==0 ){
+ return FTS_CORRUPT_VTAB;
}
blobGrowBuffer(&p->term, nPrefix+nSuffix, &rc);
if( rc==SQLITE_OK ){
if( p->iChild==0 ){
p->iOff += fts3GetVarint32(&p->aNode[p->iOff], &p->nDoclist);
if( (p->nNode-p->iOff)<p->nDoclist ){
- return SQLITE_CORRUPT_VTAB;
+ return FTS_CORRUPT_VTAB;
}
p->aDoclist = &p->aNode[p->iOff];
p->iOff += p->nDoclist;
}
}
- assert( p->iOff<=p->nNode );
+ assert_fts3_nc( p->iOff<=p->nNode );
return rc;
}
p->nNode = nNode;
/* Figure out if this is a leaf or an internal node. */
- if( p->aNode[0] ){
+ if( aNode && aNode[0] ){
/* An internal node. */
p->iOff = 1 + sqlite3Fts3GetVarint(&p->aNode[1], &p->iChild);
}else{
p->iOff = 1;
}
- return nodeReaderNext(p);
+ return aNode ? nodeReaderNext(p) : SQLITE_OK;
}
/*
** be added to. */
nPrefix = fts3PrefixCompress(pNode->key.a, pNode->key.n, zTerm, nTerm);
nSuffix = nTerm - nPrefix;
+ if(nSuffix<=0 ) return FTS_CORRUPT_VTAB;
nSpace = sqlite3Fts3VarintLen(nPrefix);
nSpace += sqlite3Fts3VarintLen(nSuffix) + nSuffix;
/* Node must have already been started. There must be a doclist for a
** leaf node, and there must not be a doclist for an internal node. */
assert( pNode->n>0 );
- assert( (pNode->a[0]=='\0')==(aDoclist!=0) );
+ assert_fts3_nc( (pNode->a[0]=='\0')==(aDoclist!=0) );
blobGrowBuffer(pPrev, nTerm, &rc);
if( rc!=SQLITE_OK ) return rc;
nPrefix = fts3PrefixCompress(pPrev->a, pPrev->n, zTerm, nTerm);
nSuffix = nTerm - nPrefix;
+ if( nSuffix<=0 ) return FTS_CORRUPT_VTAB;
memcpy(pPrev->a, zTerm, nTerm);
pPrev->n = nTerm;
int nCmp = MIN(nLhs, nRhs);
int res;
- res = memcmp(zLhs, zRhs, nCmp);
+ res = (nCmp ? memcmp(zLhs, zRhs, nCmp) : 0);
if( res==0 ) res = nLhs - nRhs;
return res;
pWriter->bNoLeafData = (pWriter->nLeafData==0);
nRoot = sqlite3_column_bytes(pSelect, 4);
aRoot = sqlite3_column_blob(pSelect, 4);
+ if( aRoot==0 ){
+ sqlite3_reset(pSelect);
+ return nRoot ? SQLITE_NOMEM : FTS_CORRUPT_VTAB;
+ }
}else{
return sqlite3_reset(pSelect);
}
int i;
int nHeight = (int)aRoot[0];
NodeWriter *pNode;
+ if( nHeight<1 || nHeight>FTS_MAX_APPENDABLE_HEIGHT ){
+ sqlite3_reset(pSelect);
+ return FTS_CORRUPT_VTAB;
+ }
pWriter->nLeafEst = (int)((iEnd - iStart) + 1)/FTS_MAX_APPENDABLE_HEIGHT;
pWriter->iStart = iStart;
pNode = &pWriter->aNodeWriter[nHeight];
pNode->iBlock = pWriter->iStart + pWriter->nLeafEst*nHeight;
- blobGrowBuffer(&pNode->block, MAX(nRoot, p->nNodeSize), &rc);
+ blobGrowBuffer(&pNode->block,
+ MAX(nRoot, p->nNodeSize)+FTS3_NODE_PADDING, &rc
+ );
if( rc==SQLITE_OK ){
memcpy(pNode->block.a, aRoot, nRoot);
pNode->block.n = nRoot;
+ memset(&pNode->block.a[nRoot], 0, FTS3_NODE_PADDING);
}
for(i=nHeight; i>=0 && rc==SQLITE_OK; i--){
NodeReader reader;
pNode = &pWriter->aNodeWriter[i];
- rc = nodeReaderInit(&reader, pNode->block.a, pNode->block.n);
- while( reader.aNode && rc==SQLITE_OK ) rc = nodeReaderNext(&reader);
- blobGrowBuffer(&pNode->key, reader.term.n, &rc);
- if( rc==SQLITE_OK ){
- memcpy(pNode->key.a, reader.term.a, reader.term.n);
- pNode->key.n = reader.term.n;
- if( i>0 ){
- char *aBlock = 0;
- int nBlock = 0;
- pNode = &pWriter->aNodeWriter[i-1];
- pNode->iBlock = reader.iChild;
- rc = sqlite3Fts3ReadBlock(p, reader.iChild, &aBlock, &nBlock, 0);
- blobGrowBuffer(&pNode->block, MAX(nBlock, p->nNodeSize), &rc);
- if( rc==SQLITE_OK ){
- memcpy(pNode->block.a, aBlock, nBlock);
- pNode->block.n = nBlock;
+ if( pNode->block.a){
+ rc = nodeReaderInit(&reader, pNode->block.a, pNode->block.n);
+ while( reader.aNode && rc==SQLITE_OK ) rc = nodeReaderNext(&reader);
+ blobGrowBuffer(&pNode->key, reader.term.n, &rc);
+ if( rc==SQLITE_OK ){
+ memcpy(pNode->key.a, reader.term.a, reader.term.n);
+ pNode->key.n = reader.term.n;
+ if( i>0 ){
+ char *aBlock = 0;
+ int nBlock = 0;
+ pNode = &pWriter->aNodeWriter[i-1];
+ pNode->iBlock = reader.iChild;
+ rc = sqlite3Fts3ReadBlock(p, reader.iChild, &aBlock, &nBlock, 0);
+ blobGrowBuffer(&pNode->block,
+ MAX(nBlock, p->nNodeSize)+FTS3_NODE_PADDING, &rc
+ );
+ if( rc==SQLITE_OK ){
+ memcpy(pNode->block.a, aBlock, nBlock);
+ pNode->block.n = nBlock;
+ memset(&pNode->block.a[nBlock], 0, FTS3_NODE_PADDING);
+ }
+ sqlite3_free(aBlock);
}
- sqlite3_free(aBlock);
}
}
nodeReaderRelease(&reader);
NodeReader reader; /* Reader object */
Blob prev = {0, 0, 0}; /* Previous term written to new node */
int rc = SQLITE_OK; /* Return code */
- int bLeaf = aNode[0]=='\0'; /* True for a leaf node */
+ int bLeaf; /* True for a leaf node */
+
+ if( nNode<1 ) return FTS_CORRUPT_VTAB;
+ bLeaf = aNode[0]=='\0';
/* Allocate required output space */
blobGrowBuffer(pNew, nNode, &rc);
sqlite3_bind_int(pChomp, 4, iIdx);
sqlite3_step(pChomp);
rc = sqlite3_reset(pChomp);
+ sqlite3_bind_null(pChomp, 2);
}
}
sqlite3_bind_blob(pReplace, 2, pHint->a, pHint->n, SQLITE_STATIC);
sqlite3_step(pReplace);
rc = sqlite3_reset(pReplace);
+ sqlite3_bind_null(pReplace, 2);
}
return rc;
const int nHint = pHint->n;
int i;
- i = pHint->n-2;
+ i = pHint->n-1;
+ if( (pHint->a[i] & 0x80) ) return FTS_CORRUPT_VTAB;
while( i>0 && (pHint->a[i-1] & 0x80) ) i--;
+ if( i==0 ) return FTS_CORRUPT_VTAB;
+ i--;
while( i>0 && (pHint->a[i-1] & 0x80) ) i--;
pHint->n = i;
i += sqlite3Fts3GetVarint(&pHint->a[i], piAbsLevel);
i += fts3GetVarint32(&pHint->a[i], pnInput);
+ assert( i<=nHint );
if( i!=nHint ) return FTS_CORRUPT_VTAB;
return SQLITE_OK;
rc = fts3IncrmergeHintPop(&hint, &iHintAbsLevel, &nHintSeg);
if( nSeg<0 || (iAbsLevel % nMod) >= (iHintAbsLevel % nMod) ){
+ /* Based on the scan in the block above, it is known that there
+ ** are no levels with a relative level smaller than that of
+ ** iAbsLevel with more than nSeg segments, or if nSeg is -1,
+ ** no levels with more than nMin segments. Use this to limit the
+ ** value of nHintSeg to avoid a large memory allocation in case the
+ ** merge-hint is corrupt*/
iAbsLevel = iHintAbsLevel;
- nSeg = nHintSeg;
+ nSeg = MIN(MAX(nMin,nSeg), nHintSeg);
bUseHint = 1;
bDirtyHint = 1;
}else{
/* If nSeg is less that zero, then there is no level with at least
** nMin segments and no hint in the %_stat table. No work to do.
** Exit early in this case. */
- if( nSeg<0 ) break;
+ if( nSeg<=0 ) break;
/* Open a cursor to iterate through the contents of the oldest nSeg
** indexes of absolute level iAbsLevel. If this cursor is opened using
}
if( SQLITE_OK==rc && pCsr->nSegment==nSeg
&& SQLITE_OK==(rc = sqlite3Fts3SegReaderStart(p, pCsr, pFilter))
- && SQLITE_ROW==(rc = sqlite3Fts3SegReaderStep(p, pCsr))
){
+ int bEmpty = 0;
+ rc = sqlite3Fts3SegReaderStep(p, pCsr);
+ if( rc==SQLITE_OK ){
+ bEmpty = 1;
+ }else if( rc!=SQLITE_ROW ){
+ sqlite3Fts3SegReaderFinish(pCsr);
+ break;
+ }
if( bUseHint && iIdx>0 ){
const char *zKey = pCsr->zTerm;
int nKey = pCsr->nTerm;
if( rc==SQLITE_OK && pWriter->nLeafEst ){
fts3LogMerge(nSeg, iAbsLevel);
- do {
- rc = fts3IncrmergeAppend(p, pWriter, pCsr);
- if( rc==SQLITE_OK ) rc = sqlite3Fts3SegReaderStep(p, pCsr);
- if( pWriter->nWork>=nRem && rc==SQLITE_ROW ) rc = SQLITE_OK;
- }while( rc==SQLITE_ROW );
+ if( bEmpty==0 ){
+ do {
+ rc = fts3IncrmergeAppend(p, pWriter, pCsr);
+ if( rc==SQLITE_OK ) rc = sqlite3Fts3SegReaderStep(p, pCsr);
+ if( pWriter->nWork>=nRem && rc==SQLITE_ROW ) rc = SQLITE_OK;
+ }while( rc==SQLITE_ROW );
+ }
/* Update or delete the input segments */
if( rc==SQLITE_OK ){
const char *zParam /* Nul-terminated string containing "A,B" */
){
int rc;
- int nMin = (FTS3_MERGE_COUNT / 2);
+ int nMin = (MergeCount(p) / 2);
int nMerge = 0;
const char *z = zParam;
int rc = SQLITE_OK;
sqlite3_stmt *pStmt = 0;
p->nAutoincrmerge = fts3Getint(&zParam);
- if( p->nAutoincrmerge==1 || p->nAutoincrmerge>FTS3_MERGE_COUNT ){
+ if( p->nAutoincrmerge==1 || p->nAutoincrmerge>MergeCount(p) ){
p->nAutoincrmerge = 8;
}
if( !p->bHasStat ){
i64 iDocid = 0;
i64 iCol = 0;
- i64 iPos = 0;
+ u64 iPos = 0;
pCsr += sqlite3Fts3GetVarint(pCsr, &iDocid);
while( pCsr<pEnd ){
- i64 iVal = 0;
- pCsr += sqlite3Fts3GetVarint(pCsr, &iVal);
+ u64 iVal = 0;
+ pCsr += sqlite3Fts3GetVarintU(pCsr, &iVal);
if( pCsr<pEnd ){
if( iVal==0 || iVal==1 ){
iCol = 0;
if( iVal ){
pCsr += sqlite3Fts3GetVarint(pCsr, &iCol);
}else{
- pCsr += sqlite3Fts3GetVarint(pCsr, &iVal);
- iDocid += iVal;
+ pCsr += sqlite3Fts3GetVarintU(pCsr, &iVal);
+ if( p->bDescIdx ){
+ iDocid = (i64)((u64)iDocid - iVal);
+ }else{
+ iDocid = (i64)((u64)iDocid + iVal);
+ }
}
}else{
iPos += (iVal - 2);
for(iCol=0; rc==SQLITE_OK && iCol<p->nColumn; iCol++){
if( p->abNotindexed[iCol]==0 ){
const char *zText = (const char *)sqlite3_column_text(pStmt, iCol+1);
- int nText = sqlite3_column_bytes(pStmt, iCol+1);
sqlite3_tokenizer_cursor *pT = 0;
- rc = sqlite3Fts3OpenTokenizer(p->pTokenizer, iLang, zText, nText,&pT);
+ rc = sqlite3Fts3OpenTokenizer(p->pTokenizer, iLang, zText, -1, &pT);
while( rc==SQLITE_OK ){
char const *zToken; /* Buffer containing token */
int nToken = 0; /* Number of bytes in token */
** meaningful value to insert is the text 'optimize'.
*/
static int fts3SpecialInsert(Fts3Table *p, sqlite3_value *pVal){
- int rc; /* Return Code */
+ int rc = SQLITE_ERROR; /* Return Code */
const char *zVal = (const char *)sqlite3_value_text(pVal);
int nVal = sqlite3_value_bytes(pVal);
rc = fts3DoIncrmerge(p, &zVal[6]);
}else if( nVal>10 && 0==sqlite3_strnicmp(zVal, "automerge=", 10) ){
rc = fts3DoAutoincrmerge(p, &zVal[10]);
-#ifdef SQLITE_TEST
- }else if( nVal>9 && 0==sqlite3_strnicmp(zVal, "nodesize=", 9) ){
- p->nNodeSize = atoi(&zVal[9]);
- rc = SQLITE_OK;
- }else if( nVal>11 && 0==sqlite3_strnicmp(zVal, "maxpending=", 9) ){
- p->nMaxPendingData = atoi(&zVal[11]);
- rc = SQLITE_OK;
- }else if( nVal>21 && 0==sqlite3_strnicmp(zVal, "test-no-incr-doclist=", 21) ){
- p->bNoIncrDoclist = atoi(&zVal[21]);
- rc = SQLITE_OK;
-#endif
+#if defined(SQLITE_DEBUG) || defined(SQLITE_TEST)
}else{
- rc = SQLITE_ERROR;
+ int v;
+ if( nVal>9 && 0==sqlite3_strnicmp(zVal, "nodesize=", 9) ){
+ v = atoi(&zVal[9]);
+ if( v>=24 && v<=p->nPgsz-35 ) p->nNodeSize = v;
+ rc = SQLITE_OK;
+ }else if( nVal>11 && 0==sqlite3_strnicmp(zVal, "maxpending=", 9) ){
+ v = atoi(&zVal[11]);
+ if( v>=64 && v<=FTS3_MAX_PENDING_DATA ) p->nMaxPendingData = v;
+ rc = SQLITE_OK;
+ }else if( nVal>21 && 0==sqlite3_strnicmp(zVal,"test-no-incr-doclist=",21) ){
+ p->bNoIncrDoclist = atoi(&zVal[21]);
+ rc = SQLITE_OK;
+ }else if( nVal>11 && 0==sqlite3_strnicmp(zVal,"mergecount=",11) ){
+ v = atoi(&zVal[11]);
+ if( v>=4 && v<=FTS3_MERGE_COUNT && (v&1)==0 ) p->nMergeCount = v;
+ rc = SQLITE_OK;
+ }
+#endif
}
-
return rc;
}
){
Fts3Table *p = (Fts3Table *)pVtab;
int rc = SQLITE_OK; /* Return Code */
- int isRemove = 0; /* True for an UPDATE or DELETE */
u32 *aSzIns = 0; /* Sizes of inserted documents */
u32 *aSzDel = 0; /* Sizes of deleted documents */
int nChng = 0; /* Net change in number of documents */
}
/* Allocate space to hold the change in document sizes */
- aSzDel = sqlite3_malloc( sizeof(aSzDel[0])*(p->nColumn+1)*2 );
+ aSzDel = sqlite3_malloc64(sizeof(aSzDel[0])*((sqlite3_int64)p->nColumn+1)*2);
if( aSzDel==0 ){
rc = SQLITE_NOMEM;
goto update_out;
if( sqlite3_value_type(apVal[0])!=SQLITE_NULL ){
assert( sqlite3_value_type(apVal[0])==SQLITE_INTEGER );
rc = fts3DeleteByRowid(p, apVal[0], &nChng, aSzDel);
- isRemove = 1;
}
/* If this is an INSERT or UPDATE operation, insert the new record. */
rc = FTS_CORRUPT_VTAB;
}
}
- if( rc==SQLITE_OK && (!isRemove || *pRowid!=p->iPrevDocid ) ){
+ if( rc==SQLITE_OK ){
rc = fts3PendingTermsDocid(p, 0, iLangid, *pRowid);
}
if( rc==SQLITE_OK ){
set nRange 1
set iFirst [lindex $map 0 0]
set cPrev [lindex $map 0 1]
+ set fPrev [lindex $map 0 2]
foreach m [lrange $map 1 end] {
- foreach {i c} $m {}
+ foreach {i c f} $m {}
- if {$cPrev == $c} {
+ if {$cPrev == $c && $fPrev==$f} {
for {set j [expr $iFirst+$nRange]} {$j<$i} {incr j} {
if {[info exists tl_lookup_table($j)]==0} break
}
lappend lRange [list $iFirst $nRange]
lappend aChar $cPrev
+ lappend aFlag $fPrev
set iFirst $i
set cPrev $c
+ set fPrev $f
set nRange 1
}
lappend lRange [list $iFirst $nRange]
lappend aChar $cPrev
+ lappend aFlag $fPrev
puts "/*"
puts "** If the argument is a codepoint corresponding to a lowercase letter"
puts "** E\"). The resuls of passing a codepoint that corresponds to an"
puts "** uppercase letter are undefined."
puts "*/"
- puts "static int ${::remove_diacritic}(int c)\{"
+ puts "static int ${::remove_diacritic}(int c, int bComplex)\{"
puts " unsigned short aDia\[\] = \{"
puts -nonewline " 0, "
set i 1
}
puts ""
puts " \};"
- puts " char aChar\[\] = \{"
- puts -nonewline " '\\0', "
+ puts "#define HIBIT ((unsigned char)0x80)"
+ puts " unsigned char aChar\[\] = \{"
+ puts -nonewline " '\\0', "
set i 1
- foreach c $aChar {
- set str "'$c', "
- if {$c == ""} { set str "'\\0', " }
+ foreach c $aChar f $aFlag {
+ if { $f } {
+ set str "'$c'|HIBIT, "
+ } else {
+ set str "'$c', "
+ }
+ if {$c == ""} { set str "'\\0', " }
- if {($i % 12)==0} {puts "" ; puts -nonewline " " }
+ if {($i % 6)==0} {puts "" ; puts -nonewline " " }
incr i
puts -nonewline "$str"
}
}
}
assert( key>=aDia[iRes] );
- return ((c > (aDia[iRes]>>3) + (aDia[iRes]&0x07)) ? c : (int)aChar[iRes]);}
+ if( bComplex==0 && (aChar[iRes] & 0x80) ) return c;
+ return (c > (aDia[iRes]>>3) + (aDia[iRes]&0x07)) ? c : ((int)aChar[iRes] & 0x7F);}
puts "\}"
}
set lCode [list]
foreach m $map {
- foreach {code char} $m {}
+ foreach {code char flag} $m {}
+ if {$flag} continue
if {$code && $char == ""} { lappend lCode $code }
}
set lCode [lsort -integer $lCode]
puts " if( c<$iFirst || c>$iLast ) return 0;"
puts " return (c < $iFirst+32) ?"
- puts " (mask0 & (1 << (c-$iFirst))) :"
- puts " (mask1 & (1 << (c-$iFirst-32)));"
+ puts " (mask0 & ((unsigned int)1 << (c-$iFirst))) :"
+ puts " (mask1 & ((unsigned int)1 << (c-$iFirst-32)));"
puts "\}"
}
puts "** The results are undefined if the value passed to this function"
puts "** is less than zero."
puts "*/"
- puts "int ${zFunc}\(int c, int bRemoveDiacritic)\{"
+ puts "int ${zFunc}\(int c, int eRemoveDiacritic)\{"
set liOff [tl_generate_ioff_table $lRecord]
tl_print_table_header
assert( ret>0 );
}
- if( bRemoveDiacritic ) ret = ${::remove_diacritic}(ret);
+ if( eRemoveDiacritic ){
+ ret = ${::remove_diacritic}(ret, eRemoveDiacritic==2);
+ }
}
}]
puts "\}"
}
+proc code {txt} {
+ set txt [string trimright $txt]
+ set txt [string trimleft $txt "\n"]
+ set n [expr {[string length $txt] - [string length [string trim $txt]]}]
+ set ret ""
+ foreach L [split $txt "\n"] {
+ append ret "[string range $L $n end]\n"
+ }
+ return [uplevel "subst -nocommands {$ret}"]
+}
+
+proc intarray {lInt} {
+ set ret ""
+ set n [llength $lInt]
+ for {set i 0} {$i < $n} {incr i 10} {
+ append ret "\n "
+ foreach int [lrange $lInt $i [expr $i+9]] {
+ append ret [format "%-7s" "$int, "]
+ }
+ }
+ append ret "\n "
+ set ret
+}
+
+proc categories_switch {Cvar first lSecond} {
+ upvar $Cvar C
+ set ret ""
+ append ret "case '$first':\n"
+ append ret " switch( zCat\[1\] ){\n"
+ foreach s $lSecond {
+ append ret " case '$s': aArray\[$C($first$s)\] = 1; break;\n"
+ }
+ append ret " case '*': \n"
+ foreach s $lSecond {
+ append ret " aArray\[$C($first$s)\] = 1;\n"
+ }
+ append ret " break;\n"
+ append ret " default: return 1;"
+ append ret " }\n"
+ append ret " break;\n"
+}
+
+# Argument is a list. Each element of which is itself a list of two elements:
+#
+# * the codepoint
+# * the category
+#
+# List elements are sorted in order of codepoint.
+#
+proc print_categories {lMap} {
+ set categories {
+ Cc Cf Cn Cs
+ Ll Lm Lo Lt Lu
+ Mc Me Mn
+ Nd Nl No
+ Pc Pd Pe Pf Pi Po Ps
+ Sc Sk Sm So
+ Zl Zp Zs
+
+ LC Co
+ }
+
+ for {set i 0} {$i < [llength $categories]} {incr i} {
+ set C([lindex $categories $i]) [expr 1+$i]
+ }
+
+ set caseC [categories_switch C C {c f n s o}]
+ set caseL [categories_switch C L {l m o t u C}]
+ set caseM [categories_switch C M {c e n}]
+ set caseN [categories_switch C N {d l o}]
+ set caseP [categories_switch C P {c d e f i o s}]
+ set caseS [categories_switch C S {c k m o}]
+ set caseZ [categories_switch C Z {l p s}]
+
+ set nCat [expr [llength [array names C]] + 1]
+ puts [code {
+ int sqlite3Fts5UnicodeCatParse(const char *zCat, u8 *aArray){
+ aArray[0] = 1;
+ switch( zCat[0] ){
+ $caseC
+ $caseL
+ $caseM
+ $caseN
+ $caseP
+ $caseS
+ $caseZ
+ }
+ return 0;
+ }
+ }]
+
+ set nRepeat 0
+ set first [lindex $lMap 0 0]
+ set class [lindex $lMap 0 1]
+ set prev -1
+
+ set CASE(0) "Lu"
+ set CASE(1) "Ll"
+
+ foreach m $lMap {
+ foreach {codepoint cl} $m {}
+ set codepoint [expr "0x$codepoint"]
+ if {$codepoint>=(1<<20)} continue
+
+ set bNew 0
+ if {$codepoint!=($prev+1)} {
+ set bNew 1
+ } elseif {
+ $cl==$class || ($class=="LC" && $cl==$CASE([expr $nRepeat & 0x01]))
+ } {
+ incr nRepeat
+ } elseif {$class=="Lu" && $nRepeat==1 && $cl=="Ll"} {
+ set class LC
+ incr nRepeat
+ } else {
+ set bNew 1
+ }
+ if {$bNew} {
+ lappend lEntries [list $first $class $nRepeat]
+ set nRepeat 1
+ set first $codepoint
+ set class $cl
+ }
+ set prev $codepoint
+ }
+ if {$nRepeat>0} {
+ lappend lEntries [list $first $class $nRepeat]
+ }
+
+ set aBlock [list 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0]
+ set aMap [list]
+ foreach e $lEntries {
+ foreach {cp class nRepeat} $e {}
+ set block [expr ($cp>>16)]
+ if {$block>0 && [lindex $aBlock $block]==0} {
+ for {set i 1} {$i<=$block} {incr i} {
+ if {[lindex $aBlock $i]==0} {
+ lset aBlock $i [llength $aMap]
+ }
+ }
+ }
+ lappend aMap [expr {$cp & 0xFFFF}]
+ lappend aData [expr {($nRepeat << 5) + $C($class)}]
+ }
+ for {set i 1} {$i<[llength $aBlock]} {incr i} {
+ if {[lindex $aBlock $i]==0} {
+ lset aBlock $i [llength $aMap]
+ }
+ }
+
+ set aBlockArray [intarray $aBlock]
+ set aMapArray [intarray $aMap]
+ set aDataArray [intarray $aData]
+ puts [code {
+ static u16 aFts5UnicodeBlock[] = {$aBlockArray};
+ static u16 aFts5UnicodeMap[] = {$aMapArray};
+ static u16 aFts5UnicodeData[] = {$aDataArray};
+
+ int sqlite3Fts5UnicodeCategory(u32 iCode) {
+ int iRes = -1;
+ int iHi;
+ int iLo;
+ int ret;
+ u16 iKey;
+
+ if( iCode>=(1<<20) ){
+ return 0;
+ }
+ iLo = aFts5UnicodeBlock[(iCode>>16)];
+ iHi = aFts5UnicodeBlock[1+(iCode>>16)];
+ iKey = (iCode & 0xFFFF);
+ while( iHi>iLo ){
+ int iTest = (iHi + iLo) / 2;
+ assert( iTest>=iLo && iTest<iHi );
+ if( iKey>=aFts5UnicodeMap[iTest] ){
+ iRes = iTest;
+ iLo = iTest+1;
+ }else{
+ iHi = iTest;
+ }
+ }
+
+ if( iRes<0 ) return 0;
+ if( iKey>=(aFts5UnicodeMap[iRes]+(aFts5UnicodeData[iRes]>>5)) ) return 0;
+ ret = aFts5UnicodeData[iRes] & 0x1F;
+ if( ret!=$C(LC) ) return ret;
+ return ((iKey - aFts5UnicodeMap[iRes]) & 0x01) ? $C(Ll) : $C(Lu);
+ }
+
+ void sqlite3Fts5UnicodeAscii(u8 *aArray, u8 *aAscii){
+ int i = 0;
+ int iTbl = 0;
+ while( i<128 ){
+ int bToken = aArray[ aFts5UnicodeData[iTbl] & 0x1F ];
+ int n = (aFts5UnicodeData[iTbl] >> 5) + i;
+ for(; i<128 && i<n; i++){
+ aAscii[i] = (u8)bToken;
+ }
+ iTbl++;
+ }
+ }
+ }]
+}
+
+proc print_test_categories {lMap} {
+
+ set lCP [list]
+ foreach e $lMap {
+ foreach {cp cat} $e {}
+ if {[expr 0x$cp] < (1<<20)} {
+ lappend lCP "{0x$cp, \"$cat\"}, "
+ }
+ }
+
+ set aCP "\n"
+ for {set i 0} {$i < [llength $lCP]} {incr i 4} {
+ append aCP " [join [lrange $lCP $i $i+3]]\n"
+ }
+
+
+ puts [code {
+ static int categories_test (int *piCode){
+ struct Codepoint {
+ int iCode;
+ const char *zCat;
+ } aCP[] = {$aCP};
+ int i;
+ int iCP = 0;
+
+ for(i=0; i<1000000; i++){
+ u8 aArray[40];
+ int cat = 0;
+ int c = 0;
+ memset(aArray, 0, sizeof(aArray));
+ if( aCP[iCP].iCode==i ){
+ sqlite3Fts5UnicodeCatParse(aCP[iCP].zCat, aArray);
+ iCP++;
+ }else{
+ aArray[0] = 1;
+ }
+
+ c = sqlite3Fts5UnicodeCategory((u32)i);
+ if( aArray[c]==0 ){
+ *piCode = i;
+ return 1;
+ }
+ }
+
+ return 0;
+ }
+ }]
+}
+
proc print_fold_test {zFunc mappings} {
global tl_lookup_table
proc print_fileheader {} {
puts [string trim {
/*
-** 2012 May 25
+** 2012-05-25
**
** The author disclaims copyright to this source code. In place of
** a legal notice, here is a blessing:
puts "#include <stdio.h>"
puts ""
puts "int main(int argc, char **argv)\{"
- puts " int r1, r2;"
+ puts " int r1, r2, r3;"
puts " int code;"
+ puts " r3 = 0;"
puts " r1 = isalnum_test(&code);"
puts " if( r1 ) printf(\"isalnum(): Problem with code %d\\n\",code);"
puts " else printf(\"isalnum(): test passed\\n\");"
puts " r2 = fold_test(&code);"
puts " if( r2 ) printf(\"fold(): Problem with code %d\\n\",code);"
puts " else printf(\"fold(): test passed\\n\");"
- puts " return (r1 || r2);"
+ if {$::generate_fts5_code} {
+ puts " r3 = categories_test(&code);"
+ puts " if( r3 ) printf(\"categories(): Problem with code %d\\n\",code);"
+ puts " else printf(\"categories(): test passed\\n\");"
+ }
+ puts " return (r1 || r2 || r3);"
puts "\}"
}
print_fileheader
+if {$::generate_test_code} {
+ puts "typedef unsigned short int u16;"
+ puts "typedef unsigned char u8;"
+ puts "#include <string.h>"
+}
+
# Print the isalnum() function to stdout.
#
set lRange [an_load_separator_ranges]
-print_isalnum ${function_prefix}UnicodeIsalnum $lRange
+if {$generate_fts5_code==0} {
+ print_isalnum ${function_prefix}UnicodeIsalnum $lRange
+}
# Leave a gap between the two generated C functions.
#
#
print_fold ${function_prefix}UnicodeFold
+if {$generate_fts5_code} {
+ puts ""
+ puts ""
+ print_categories [cc_load_unicodedata_text ${unicodedata.txt}]
+}
+
# Print the test routines and main() function to stdout, if -test
# was specified.
#
if {$::generate_test_code} {
- print_test_isalnum ${function_prefix}UnicodeIsalnum $lRange
+ if {$generate_fts5_code==0} {
+ print_test_isalnum ${function_prefix}UnicodeIsalnum $lRange
+ }
print_fold_test ${function_prefix}UnicodeFold $mappings
+ print_test_categories [cc_load_unicodedata_text ${unicodedata.txt}]
print_test_main
}
# character that it should be replaced with, or an empty string if the
# codepoint should simply be removed from the input. Examples:
#
-# { 224 a } (replace codepoint 224 to "a")
-# { 769 "" } (remove codepoint 769 from input)
+# { 224 a 0 } (replace codepoint 224 to "a")
+# { 769 "" 0 } (remove codepoint 769 from input)
#
# Mappings are only returned for non-upper case codepoints. It is assumed
# that the input has already been folded to lower case.
#
+# The third value in the list is always either 0 or 1. 0 if the
+# UnicodeData.txt file maps the codepoint to a single ASCII character and
+# a diacritic, or 1 if the mapping is indirect. For example, consider the
+# two entries:
+#
+# 1ECD;LATIN SMALL LETTER O WITH DOT BELOW;Ll;0;L;006F 0323;;;;N;;;1ECC;;1ECC
+# 1ED9;LATIN SMALL LETTER O WITH CIRCUMFLEX AND DOT BELOW;Ll;0;L;1ECD 0302;;;;N;;;1ED8;;1ED8
+#
+# The first codepoint is a direct mapping (as 006F is ASCII and 0323 is a
+# diacritic). The second is an indirect mapping, as it maps to the
+# first codepoint plus 0302 (a diacritic).
+#
proc rd_load_unicodedata_text {zName} {
global tl_lookup_table
set iAscii [expr "0x[lindex $character_decomposition_mapping 0]"]
set iDia [expr "0x[lindex $character_decomposition_mapping 1]"]
+ # Filter out upper-case characters, as they will be mapped to their
+ # lower-case equivalents before this data is used.
if {[info exists tl_lookup_table($iCode)]} continue
+ # Check if this is an indirect mapping. If so, set bIndirect to true
+ # and change $iAscii to the indirectly mappped ASCII character.
+ set bIndirect 0
+ if {[info exists dia($iDia)] && [info exists mapping($iAscii)]} {
+ set iAscii $mapping($iAscii)
+ set bIndirect 1
+ }
+
if { ($iAscii >= 97 && $iAscii <= 122)
|| ($iAscii >= 65 && $iAscii <= 90)
} {
- lappend lRet [list $iCode [string tolower [format %c $iAscii]]]
+ lappend lRet [list $iCode [string tolower [format %c $iAscii]] $bIndirect]
+ set mapping($iCode) $iAscii
set dia($iDia) 1
}
}
foreach d [array names dia] {
- lappend lRet [list $d ""]
+ lappend lRet [list $d "" 0]
}
set lRet [lsort -integer -index 0 $lRet]
}
}
+proc cc_load_unicodedata_text {zName} {
+ set fd [open $zName]
+ set lField {
+ code
+ character_name
+ general_category
+ canonical_combining_classes
+ bidirectional_category
+ character_decomposition_mapping
+ decimal_digit_value
+ digit_value
+ numeric_value
+ mirrored
+ unicode_1_name
+ iso10646_comment_field
+ uppercase_mapping
+ lowercase_mapping
+ titlecase_mapping
+ }
+ set lRet [list]
+
+ while { ![eof $fd] } {
+ set line [gets $fd]
+ if {$line == ""} continue
+
+ set fields [split $line ";"]
+ if {[llength $fields] != [llength $lField]} { error "parse error: $line" }
+ foreach $lField $fields {}
+
+ lappend lRet [list $code $general_category]
+ }
+
+ close $fd
+ set lRet
+}
+
-C Fix\sa\sbuffer\soverread\sthat\scould\soccur\swhen\srunning\sfts5\sprefix\squeries\sinside\sa\stransaction.
-D 2019-09-03T18:36:11.001
+C Import\sall\sFTS3/4\senhancements\sand\sfixes\sthat\sexist\son\sthe\slatest\strunk\n(3.31.0-beta)\sthat\sdo\snot\srequire\sextensive\schange\sto\sthe\sSQLite\score\ninto\sthe\s3.22\sbranch.\s\sBasically,\sthe\s3.31.0\sFTS3\ssources\sare\scopied\sinto\n3.22.0,\swith\sminor\schanges\sto\swork\saround\score\senhancements\sthat\sare\snot\navailable\sin\s3.22.0.
+D 2020-01-17T14:18:48.465
F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
F Makefile.in 38f84f301cbef443b2d269f67a74b8cc536469831f70df7c3e912acc04932cc2
F ext/fts2/mkfts2amal.tcl 974d5d438cb3f7c4a652639262f82418c1e4cff0
F ext/fts3/README.content fdc666a70d5257a64fee209f97cf89e0e6e32b51
F ext/fts3/README.syntax a19711dc5458c20734b8e485e75fb1981ec2427a
-F ext/fts3/README.tokenizers e0a8b81383ea60d0334d274fadf305ea14a8c314
+F ext/fts3/README.tokenizers b92bdeb8b46503f0dd301d364efc5ef59ef9fa8e2758b8e742f39fa93a2e422d
F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d
-F ext/fts3/fts3.c 5adba0be37f75eb71fe6f38a784f40ca4c2455900b948e1a80e05d5061a30979
+F ext/fts3/fts3.c b3192f3596592b11f964ec57a46629946d07cb762c54690487d90315384a1f4b
F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe
-F ext/fts3/fts3Int.h eb2502000148e80913b965db3e59f29251266d0a
-F ext/fts3/fts3_aux.c 9edc3655fcb287f0467d0a4b886a01c6185fe9f1
-F ext/fts3/fts3_expr.c dfd571a24412779ac01f25c01d888c6ef7b2d0ef
-F ext/fts3/fts3_hash.c 29b986e43f4e9dd40110eafa377dc0d63c422c60
+F ext/fts3/fts3Int.h f091030b976045e7df91af2337935952b477cdbd9f48058c44c965684484cb50
+F ext/fts3/fts3_aux.c 489e1e383868263207aef8ece4ab0a53053e4d7cf93ec48db79736140a5fb2e9
+F ext/fts3/fts3_expr.c b132af223e90e35b9f9efa9fe63d6ae737d34153a3b6066736086df8abc78a1f
+F ext/fts3/fts3_hash.c 8b6e31bfb0844c27dc6092c2620bdb1fca17ed613072db057d96952c6bdb48b7
F ext/fts3/fts3_hash.h 39cf6874dc239d6b4e30479b1975fe5b22a3caaf
-F ext/fts3/fts3_icu.c deb46f7020d87ea7a14a433fb7a7f4bef42a9652
+F ext/fts3/fts3_icu.c 305ce7fb6036484085b5556a9c8e62acdc7763f0f4cdf5fd538212a9f3720116
F ext/fts3/fts3_porter.c 3565faf04b626cddf85f03825e86056a4562c009
-F ext/fts3/fts3_snippet.c 68ae118b0f834ea53d2b89e4087fc0f0b8c4ee4e
-F ext/fts3/fts3_term.c 88c55a6fa1a51ab494e33dced0401a6c28791fd7
-F ext/fts3/fts3_test.c 79f2a7fbb3f672fa032e5a432ca274ea3ee93c34
-F ext/fts3/fts3_tokenize_vtab.c a27593ab19657166f6fa5ec073b678cc29a75860
-F ext/fts3/fts3_tokenizer.c a22bf311a71f3efa9d7012d8cc48fc9b0f3dace7
+F ext/fts3/fts3_snippet.c 052b35ad746349ffb53820379bacdb23ff3ac60d3cc13d986e56d42822ef5a9a
+F ext/fts3/fts3_term.c b529d6cceecd85a8256c4275012ac739744b38008b55f56184f35839110d0fab
+F ext/fts3/fts3_test.c 73b16e229e517c1b1f0fb8e1046182a4e5dbc8dbe6eea8a5d4353fcce7dbbf39
+F ext/fts3/fts3_tokenize_vtab.c 8ac0608b94c36980dcfaaaf63c95aba988e0e086a08b4c31563c1fb601458bd2
+F ext/fts3/fts3_tokenizer.c 3a9c6a2551b0dd7b8f70f193df4ec263b9f5541f93c4ca737358f2457817d9c1
F ext/fts3/fts3_tokenizer.h 64c6ef6c5272c51ebe60fc607a896e84288fcbc3
F ext/fts3/fts3_tokenizer1.c 5c98225a53705e5ee34824087478cf477bdb7004
-F ext/fts3/fts3_unicode.c 525a3bd9a7564603c5c061b7de55403a565307758a94600e8a2f6b00d1c40d9d
-F ext/fts3/fts3_unicode2.c cc04fc672bfd42b1e650398cb0bf71f64f9aae032cfe75bbcfe75b9cf966029c
-F ext/fts3/fts3_write.c d8c64541a9c1f139336951e814c69ee7783bc2125383fd6de8ec461d4d6975a8
+F ext/fts3/fts3_unicode.c 4b9af6151c29b35ed09574937083cece7c31e911f69615e168a39677569b684d
+F ext/fts3/fts3_unicode2.c 416eb7e1e81142703520d284b768ca2751d40e31fa912cae24ba74860532bf0f
+F ext/fts3/fts3_write.c 6f9dd5d774003ea81b8b32daa7d0819f9aaf01bf2b5f33498a69aab096094ed3
F ext/fts3/fts3speed.tcl b54caf6a18d38174f1a6e84219950d85e98bb1e9
F ext/fts3/mkfts3amal.tcl 252ecb7fe6467854f2aa237bf2c390b74e71f100
F ext/fts3/tool/fts3cov.sh c331d006359456cf6f8f953e37f2b9c7d568f3863f00bb5f7eb87fea4ac01b73
F ext/fts3/tool/fts3view.c 202801a2056995b763864d60c2dee744d46f1677
F ext/fts3/unicode/CaseFolding.txt 8c678ca52ecc95e16bc7afc2dbf6fc9ffa05db8c
F ext/fts3/unicode/UnicodeData.txt cd07314edb62d49fde34debdaf92fa2aa69011e7
-F ext/fts3/unicode/mkunicode.tcl ab0543a3b2399092ea2dd75df1bef333405b0d7f6b8c4951a0fbb60e780cb69f
-F ext/fts3/unicode/parseunicode.tcl da577d1384810fb4e2b209bf3313074353193e95
+F ext/fts3/unicode/mkunicode.tcl bf7fcaa6d68e6d38223467983785d054f1cff4d9e3905dd51f6ed8801bb590d5
+F ext/fts3/unicode/parseunicode.tcl a981bd6466d12dd17967515801c3ff23f74a281be1a03cf1e6f52a6959fc77eb
F ext/fts5/extract_api_docs.tcl a36e54ec777172ddd3f9a88daf593b00848368e0
F ext/fts5/fts5.h 62f3e33ceeb9a428db139f9c012186b371da1cc7
F ext/fts5/fts5Int.h eda28e3a0a5d87c412e8355fe35da875b04cb389908c8eb0d867ad662adbc491
F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
F src/select.c bebe7cce45d899d2237c76bce059d525abf5b861f2fce92f6b53914a961c01ba
F src/shell.c.in 4e1bcf8c70b8fb97c7cbaca6602e2a291d7fe17eff23a5de003d6fabd87f27d1
-F src/sqlite.h.in 959deaad89679e31d7f68fda668b0c5d1f592fffed7a9c1740fb8ded4e4e754a
+F src/sqlite.h.in c050d0749910f9ae101f126f5adbfe15c692e105a21ee5cec7e98a608d0cd847
F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
F src/sqlite3ext.h 99189e7611eb0bf98f21c7835dc74730a84e2e809c98e1e31c33896dee7a2849
F src/sqliteInt.h 9c70315598b34810a83e4894455acb18e95cf63ce4e6cbb451ac2d17eabc2544
F test/fts3defer3.test dd53fc13223c6d8264a98244e9b19abd35ed71cd
F test/fts3drop.test 1b906e293d6773812587b3dc458cb9e8f3f0c297
F test/fts3e.test 1f6c6ac9cc8b772ca256e6b22aaeed50c9350851
-F test/fts3expr.test 9466627007804d855bf9df2a0cfb3dac23686fdc
+F test/fts3expr.test ebae205a7a89446c32583bcd492dcb817b9f6b31819bb4dde2583bb99c77e526
F test/fts3expr2.test 18da930352e5693eaa163a3eacf96233b7290d1a
F test/fts3expr3.test c4d4a7d6327418428c96e0a3a1137c251b8dfbf8
F test/fts3expr4.test c39a15d676b14fc439d9bf845aa7bddcf4a74dc3
F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 8452fd549966de12da35110bf4c87a2f34ade30ef44b6ed8252b43dc19fa830d
-Q +b3fa58dd7403dbd4d2e9f3ae23d7d1337830d6fef2aa2f137ac5174de0d5828e
-R 6e7cd0bb05f4e071bb761d7993471717
+P 68b898381ac2942965a3dbd416a45ddf813d6df7ea160f500ae4978e44a3a050
+R 875ae95303236d12f9a2fe1f9b260baa
U drh
-Z a550bb8619b8e3d46b4ec8c730c74128
+Z 7916b9cfdd1d58ce6d6b2a3acbbaec70
-68b898381ac2942965a3dbd416a45ddf813d6df7ea160f500ae4978e44a3a050
\ No newline at end of file
+cbcbb1e5a98bde5a40e90faf8467b4f761d2ad7414cf4423571bdb8ba38a81d5
\ No newline at end of file
** </dl>
*/
#define SQLITE_PREPARE_PERSISTENT 0x01
+#define SQLITE_PREPARE_NO_VTAB 0x04
/*
** CAPI3REF: Compiling An SQL Statement
** to [sqlite3_create_function()], [sqlite3_create_function16()], or
** [sqlite3_create_function_v2()].
*/
-#define SQLITE_DETERMINISTIC 0x800
+#define SQLITE_DETERMINISTIC 0x000000800
+#define SQLITE_DIRECTONLY 0x000080000
/*
** CAPI3REF: Deprecated Functions
} {1 {Usage: fts3_exprtest(tokenizer, expr, col1, ...}}
do_test fts3expr-5.2 {
catchsql { SELECT fts3_exprtest('doesnotexist', 'a b', 'c') }
-} {1 {No such tokenizer module}}
+} {1 {unknown tokenizer: doesnotexist}}
do_test fts3expr-5.3 {
catchsql { SELECT fts3_exprtest('simple', 'a b OR', 'c') }
} {1 {Error parsing expression}}
projects / thirdparty / sqlite.git / commitdiff
