return SQLITE_NOMEM;
}
- rc = sqlite3Fts3ExprParse(p->pTokenizer, p->azColumn, p->bHasStat,
- p->nColumn, iCol, zQuery, -1, &pCsr->pExpr
+ pCsr->iLangid = 0;
+ if( nVal==2 ) pCsr->iLangid = sqlite3_value_int(apVal[1]);
+
+ rc = sqlite3Fts3ExprParse(p->pTokenizer, pCsr->iLangid,
+ p->azColumn, p->bHasStat, p->nColumn, iCol, zQuery, -1, &pCsr->pExpr
);
if( rc!=SQLITE_OK ){
if( rc==SQLITE_ERROR ){
return rc;
}
- pCsr->iLangid = 0;
- if( nVal==2 ) pCsr->iLangid = sqlite3_value_int(apVal[1]);
-
rc = sqlite3Fts3ReadLock(p);
if( rc!=SQLITE_OK ) return rc;
void sqlite3Fts3Matchinfo(sqlite3_context *, Fts3Cursor *, const char *);
/* fts3_expr.c */
-int sqlite3Fts3ExprParse(sqlite3_tokenizer *,
+int sqlite3Fts3ExprParse(sqlite3_tokenizer *, int,
char **, int, int, int, const char *, int, Fts3Expr **
);
void sqlite3Fts3ExprFree(Fts3Expr *);
int sqlite3Fts3InitTerm(sqlite3 *db);
#endif
+int sqlite3Fts3OpenTokenizer(sqlite3_tokenizer *, int, const char *, int,
+ sqlite3_tokenizer_cursor **
+);
+
/* fts3_aux.c */
int sqlite3Fts3InitAux(sqlite3 *db);
typedef struct ParseContext ParseContext;
struct ParseContext {
sqlite3_tokenizer *pTokenizer; /* Tokenizer module */
+ int iLangid; /* Language id used with tokenizer */
const char **azCol; /* Array of column names for fts3 table */
int bFts4; /* True to allow FTS4-only syntax */
int nCol; /* Number of entries in azCol[] */
return pRet;
}
+int sqlite3Fts3OpenTokenizer(
+ sqlite3_tokenizer *pTokenizer,
+ int iLangid,
+ const char *z,
+ int n,
+ sqlite3_tokenizer_cursor **ppCsr
+){
+ sqlite3_tokenizer_module const *pModule = pTokenizer->pModule;
+ sqlite3_tokenizer_cursor *pCsr = 0;
+ int rc;
+
+ rc = pModule->xOpen(pTokenizer, z, n, &pCsr);
+ assert( rc==SQLITE_OK || pCsr==0 );
+ if( rc==SQLITE_OK ){
+ pCsr->pTokenizer = pTokenizer;
+ if( pModule->iVersion>=1 ){
+ rc = pModule->xLanguageid(pCsr, iLangid);
+ if( rc!=SQLITE_OK ){
+ pModule->xClose(pCsr);
+ pCsr = 0;
+ }
+ }
+ }
+ *ppCsr = pCsr;
+ return rc;
+}
+
/*
** Extract the next token from buffer z (length n) using the tokenizer
Fts3Expr *pRet = 0;
int nConsumed = 0;
- rc = pModule->xOpen(pTokenizer, z, n, &pCursor);
+ rc = sqlite3Fts3OpenTokenizer(pTokenizer, pParse->iLangid, z, n, &pCursor);
if( rc==SQLITE_OK ){
const char *zToken;
int nToken, iStart, iEnd, iPosition;
int nByte; /* total space to allocate */
- pCursor->pTokenizer = pTokenizer;
rc = pModule->xNext(pCursor, &zToken, &nToken, &iStart, &iEnd, &iPosition);
-
if( rc==SQLITE_OK ){
nByte = sizeof(Fts3Expr) + sizeof(Fts3Phrase) + nToken;
pRet = (Fts3Expr *)fts3MallocZero(nByte);
** appends buffer zTemp to buffer p, and fills in the Fts3Expr and Fts3Phrase
** structures.
*/
- rc = pModule->xOpen(pTokenizer, zInput, nInput, &pCursor);
+ rc = sqlite3Fts3OpenTokenizer(
+ pTokenizer, pParse->iLangid, zInput, nInput, &pCursor);
if( rc==SQLITE_OK ){
int ii;
- pCursor->pTokenizer = pTokenizer;
for(ii=0; rc==SQLITE_OK; ii++){
const char *zByte;
int nByte, iBegin, iEnd, iPos;
*/
int sqlite3Fts3ExprParse(
sqlite3_tokenizer *pTokenizer, /* Tokenizer module */
+ int iLangid, /* Language id for tokenizer */
char **azCol, /* Array of column names for fts3 table */
int bFts4, /* True to allow FTS4-only syntax */
int nCol, /* Number of entries in azCol[] */
int nParsed;
int rc;
ParseContext sParse;
+
+ memset(&sParse, 0, sizeof(ParseContext));
sParse.pTokenizer = pTokenizer;
+ sParse.iLangid = iLangid;
sParse.azCol = (const char **)azCol;
sParse.nCol = nCol;
sParse.iDefaultCol = iDefaultCol;
- sParse.nNest = 0;
sParse.bFts4 = bFts4;
if( z==0 ){
*ppExpr = 0;
}
rc = sqlite3Fts3ExprParse(
- pTokenizer, azCol, 0, nCol, nCol, zExpr, nExpr, &pExpr
+ pTokenizer, 0, azCol, 0, nCol, nCol, zExpr, nExpr, &pExpr
);
if( rc!=SQLITE_OK && rc!=SQLITE_NOMEM ){
sqlite3_result_error(context, "Error parsing expression", -1);
*/
static int fts3SnippetShift(
Fts3Table *pTab, /* FTS3 table snippet comes from */
+ int iLangid, /* Language id to use in tokenizing */
int nSnippet, /* Number of tokens desired for snippet */
const char *zDoc, /* Document text to extract snippet from */
int nDoc, /* Size of buffer zDoc in bytes */
/* Open a cursor on zDoc/nDoc. Check if there are (nSnippet+nDesired)
** or more tokens in zDoc/nDoc.
*/
- rc = pMod->xOpen(pTab->pTokenizer, zDoc, nDoc, &pC);
+ rc = sqlite3Fts3OpenTokenizer(pTab->pTokenizer, iLangid, zDoc, nDoc, &pC);
if( rc!=SQLITE_OK ){
return rc;
}
- pC->pTokenizer = pTab->pTokenizer;
while( rc==SQLITE_OK && iCurrent<(nSnippet+nDesired) ){
const char *ZDUMMY; int DUMMY1, DUMMY2, DUMMY3;
rc = pMod->xNext(pC, &ZDUMMY, &DUMMY1, &DUMMY2, &DUMMY3, &iCurrent);
/* Open a token cursor on the document. */
pMod = (sqlite3_tokenizer_module *)pTab->pTokenizer->pModule;
- rc = pMod->xOpen(pTab->pTokenizer, zDoc, nDoc, &pC);
+ rc = sqlite3Fts3OpenTokenizer(pTab->pTokenizer, pCsr->iLangid, zDoc,nDoc,&pC);
if( rc!=SQLITE_OK ){
return rc;
}
- pC->pTokenizer = pTab->pTokenizer;
while( rc==SQLITE_OK ){
int iBegin; /* Offset in zDoc of start of token */
if( !isShiftDone ){
int n = nDoc - iBegin;
- rc = fts3SnippetShift(pTab, nSnippet, &zDoc[iBegin], n, &iPos, &hlmask);
+ rc = fts3SnippetShift(
+ pTab, pCsr->iLangid, nSnippet, &zDoc[iBegin], n, &iPos, &hlmask
+ );
isShiftDone = 1;
/* Now that the shift has been done, check if the initial "..." are
}
/* Initialize a tokenizer iterator to iterate through column iCol. */
- rc = pMod->xOpen(pTab->pTokenizer, zDoc, nDoc, &pC);
+ rc = sqlite3Fts3OpenTokenizer(pTab->pTokenizer, pCsr->iLangid,
+ zDoc, nDoc, &pC
+ );
if( rc!=SQLITE_OK ) goto offsets_out;
- pC->pTokenizer = pTab->pTokenizer;
rc = pMod->xNext(pC, &ZDUMMY, &NDUMMY, &iStart, &iEnd, &iCurrent);
while( rc==SQLITE_OK ){
** This file is not part of the production FTS code. It is only used for
** testing. It contains a Tcl command that can be used to test if a document
** matches an FTS NEAR expression.
+**
+** As of March 2012, it also contains a version 1 tokenizer used for testing
+** that the sqlite3_tokenizer_module.xLanguage() method is invoked correctly.
*/
#include <tcl.h>
return TCL_OK;
}
+/**************************************************************************
+** Beginning of test tokenizer code.
+**
+** For language 0, this tokenizer is similar to the default 'simple'
+** tokenizer. For other languages L, the following:
+**
+** * Odd numbered languages are case-sensitive. Even numbered
+** languages are not.
+**
+** * Language ids 100 or greater are considered an error.
+**
+** The implementation assumes that the input contains only ASCII characters
+** (i.e. those that may be encoded in UTF-8 using a single byte).
+*/
+typedef struct test_tokenizer {
+ sqlite3_tokenizer base;
+} test_tokenizer;
+
+typedef struct test_tokenizer_cursor {
+ sqlite3_tokenizer_cursor base;
+ const char *aInput; /* Input being tokenized */
+ int nInput; /* Size of the input in bytes */
+ int iInput; /* Current offset in aInput */
+ int iToken; /* Index of next token to be returned */
+ char *aBuffer; /* Buffer containing current token */
+ int nBuffer; /* Number of bytes allocated at pToken */
+ int iLangid; /* Configured language id */
+} test_tokenizer_cursor;
+
+static int testTokenizerCreate(
+ int argc, const char * const *argv,
+ sqlite3_tokenizer **ppTokenizer
+){
+ test_tokenizer *pNew;
+
+ pNew = sqlite3_malloc(sizeof(test_tokenizer));
+ if( !pNew ) return SQLITE_NOMEM;
+ memset(pNew, 0, sizeof(test_tokenizer));
+
+ *ppTokenizer = (sqlite3_tokenizer *)pNew;
+ return SQLITE_OK;
+}
+
+static int testTokenizerDestroy(sqlite3_tokenizer *pTokenizer){
+ test_tokenizer *p = (test_tokenizer *)pTokenizer;
+ sqlite3_free(p);
+ return SQLITE_OK;
+}
+
+static int testTokenizerOpen(
+ sqlite3_tokenizer *pTokenizer, /* The tokenizer */
+ const char *pInput, int nBytes, /* String to be tokenized */
+ sqlite3_tokenizer_cursor **ppCursor /* OUT: Tokenization cursor */
+){
+ int rc = SQLITE_OK; /* Return code */
+ test_tokenizer_cursor *pCsr; /* New cursor object */
+
+ UNUSED_PARAMETER(pTokenizer);
+
+ pCsr = (test_tokenizer_cursor *)sqlite3_malloc(sizeof(test_tokenizer_cursor));
+ if( pCsr==0 ){
+ rc = SQLITE_NOMEM;
+ }else{
+ memset(pCsr, 0, sizeof(test_tokenizer_cursor));
+ pCsr->aInput = pInput;
+ if( nBytes<0 ){
+ pCsr->nInput = strlen(pInput);
+ }else{
+ pCsr->nInput = nBytes;
+ }
+ }
+
+ *ppCursor = (sqlite3_tokenizer_cursor *)pCsr;
+ return rc;
+}
+
+static int testTokenizerClose(sqlite3_tokenizer_cursor *pCursor){
+ test_tokenizer_cursor *pCsr = (test_tokenizer_cursor *)pCursor;
+ sqlite3_free(pCsr->aBuffer);
+ sqlite3_free(pCsr);
+ return SQLITE_OK;
+}
+
+static int testIsTokenChar(char c){
+ return (c>='a' && c<='z') || (c>='A' && c<='Z');
+}
+static int testTolower(char c){
+ char ret = c;
+ if( ret>='A' && ret<='Z') ret = ret - ('A'-'a');
+ return ret;
+}
+
+static int testTokenizerNext(
+ sqlite3_tokenizer_cursor *pCursor, /* Cursor returned by testTokenizerOpen */
+ const char **ppToken, /* OUT: *ppToken is the token text */
+ int *pnBytes, /* OUT: Number of bytes in token */
+ int *piStartOffset, /* OUT: Starting offset of token */
+ int *piEndOffset, /* OUT: Ending offset of token */
+ int *piPosition /* OUT: Position integer of token */
+){
+ test_tokenizer_cursor *pCsr = (test_tokenizer_cursor *)pCursor;
+ int rc = SQLITE_OK;
+ const char *p;
+ const char *pEnd;
+
+ p = &pCsr->aInput[pCsr->iInput];
+ pEnd = &pCsr->aInput[pCsr->nInput];
+
+ /* Skip past any white-space */
+ assert( p<=pEnd );
+ while( p<pEnd && testIsTokenChar(*p)==0 ) p++;
+
+ if( p==pEnd ){
+ rc = SQLITE_DONE;
+ }else{
+ /* Advance to the end of the token */
+ const char *pToken = p;
+ int nToken;
+ while( p<pEnd && testIsTokenChar(*p) ) p++;
+ nToken = p-pToken;
+
+ /* Copy the token into the buffer */
+ if( nToken>pCsr->nBuffer ){
+ sqlite3_free(pCsr->aBuffer);
+ pCsr->aBuffer = sqlite3_malloc(nToken);
+ }
+ if( pCsr->aBuffer==0 ){
+ rc = SQLITE_NOMEM;
+ }else{
+ int i;
+
+ if( pCsr->iLangid & 0x00000001 ){
+ for(i=0; i<nToken; i++) pCsr->aBuffer[i] = pToken[i];
+ }else{
+ for(i=0; i<nToken; i++) pCsr->aBuffer[i] = testTolower(pToken[i]);
+ }
+ pCsr->iToken++;
+ pCsr->iInput = p - pCsr->aInput;
+
+ *ppToken = pCsr->aBuffer;
+ *pnBytes = nToken;
+ *piStartOffset = pToken - pCsr->aInput;
+ *piEndOffset = p - pCsr->aInput;
+ *piPosition = pCsr->iToken;
+ }
+ }
+
+ return rc;
+}
+
+static int testTokenizerLanguage(
+ sqlite3_tokenizer_cursor *pCursor,
+ int iLangid
+){
+ int rc = SQLITE_OK;
+ test_tokenizer_cursor *pCsr = (test_tokenizer_cursor *)pCursor;
+ pCsr->iLangid = iLangid;
+ if( pCsr->iLangid>=100 ){
+ rc = SQLITE_ERROR;
+ }
+ return rc;
+}
+
+static int fts3_test_tokenizer_cmd(
+ ClientData clientData,
+ Tcl_Interp *interp,
+ int objc,
+ Tcl_Obj *CONST objv[]
+){
+ static const sqlite3_tokenizer_module testTokenizerModule = {
+ 1,
+ testTokenizerCreate,
+ testTokenizerDestroy,
+ testTokenizerOpen,
+ testTokenizerClose,
+ testTokenizerNext,
+ testTokenizerLanguage
+ };
+ const sqlite3_tokenizer_module *pPtr = &testTokenizerModule;
+ if( objc!=1 ){
+ Tcl_WrongNumArgs(interp, 1, objv, "");
+ return TCL_ERROR;
+ }
+ Tcl_SetObjResult(interp, Tcl_NewByteArrayObj(
+ (const unsigned char *)&pPtr, sizeof(sqlite3_tokenizer_module *)
+ ));
+ return TCL_OK;
+}
+
+/*
+** End of tokenizer code.
+**************************************************************************/
+
int Sqlitetestfts3_Init(Tcl_Interp *interp){
Tcl_CreateObjCommand(interp, "fts3_near_match", fts3_near_match_cmd, 0, 0);
Tcl_CreateObjCommand(interp,
"fts3_configure_incr_load", fts3_configure_incr_load_cmd, 0, 0
);
+ Tcl_CreateObjCommand(
+ interp, "fts3_test_tokenizer", fts3_test_tokenizer_cmd, 0, 0
+ );
return TCL_OK;
}
#endif /* ifdef SQLITE_TEST */
goto finish;
}
pTokenizer->pModule = p;
- if( SQLITE_OK!=p->xOpen(pTokenizer, zInput, nInput, &pCsr) ){
+ if( sqlite3Fts3OpenTokenizer(pTokenizer, 0, zInput, nInput, &pCsr) ){
zErr = "error in xOpen()";
goto finish;
}
- pCsr->pTokenizer = pTokenizer;
while( SQLITE_OK==p->xNext(pCsr, &zToken, &nToken, &iStart, &iEnd, &iPos) ){
Tcl_ListObjAppendElement(0, pRet, Tcl_NewIntObj(iPos));
struct sqlite3_tokenizer_module {
/*
- ** Structure version. Should always be set to 0.
+ ** Structure version. Should always be set to 0 or 1.
*/
int iVersion;
int *piEndOffset, /* OUT: Byte offset of end of token in input buffer */
int *piPosition /* OUT: Number of tokens returned before this one */
);
+
+ /***********************************************************************
+ ** Methods below this point are only available if iVersion>=1.
+ */
+
+ /*
+ ** Configure the language id of a tokenizer cursor.
+ */
+ int (*xLanguageid)(sqlite3_tokenizer_cursor *pCsr, int iLangid);
};
struct sqlite3_tokenizer {
*/
static int fts3PendingTermsAdd(
Fts3Table *p, /* Table into which text will be inserted */
+ int iLangid, /* Language id to use */
const char *zText, /* Text of document to be inserted */
int iCol, /* Column into which text is being inserted */
u32 *pnWord /* OUT: Number of tokens inserted */
return SQLITE_OK;
}
- rc = pModule->xOpen(pTokenizer, zText, -1, &pCsr);
+ rc = sqlite3Fts3OpenTokenizer(pTokenizer, iLangid, zText, -1, &pCsr);
if( rc!=SQLITE_OK ){
return rc;
}
- pCsr->pTokenizer = pTokenizer;
xNext = pModule->xNext;
while( SQLITE_OK==rc
** Argument apVal is the same as the similarly named argument passed to
** fts3InsertData(). Parameter iDocid is the docid of the new row.
*/
-static int fts3InsertTerms(Fts3Table *p, sqlite3_value **apVal, u32 *aSz){
+static int fts3InsertTerms(
+ Fts3Table *p,
+ int iLangid,
+ sqlite3_value **apVal,
+ u32 *aSz
+){
int i; /* Iterator variable */
for(i=2; i<p->nColumn+2; i++){
const char *zText = (const char *)sqlite3_value_text(apVal[i]);
- int rc = fts3PendingTermsAdd(p, zText, i-2, &aSz[i-2]);
+ int rc = fts3PendingTermsAdd(p, iLangid, zText, i-2, &aSz[i-2]);
if( rc!=SQLITE_OK ){
return rc;
}
if( rc==SQLITE_OK ){
if( SQLITE_ROW==sqlite3_step(pSelect) ){
int i;
- rc = fts3PendingTermsDocid(p,
- langidFromSelect(p, pSelect),
- sqlite3_column_int64(pSelect, 0)
- );
+ int iLangid = langidFromSelect(p, pSelect);
+ rc = fts3PendingTermsDocid(p, iLangid, sqlite3_column_int64(pSelect, 0));
for(i=1; rc==SQLITE_OK && i<=p->nColumn; i++){
const char *zText = (const char *)sqlite3_column_text(pSelect, i);
- rc = fts3PendingTermsAdd(p, zText, -1, &aSz[i-1]);
+ rc = fts3PendingTermsAdd(p, iLangid, zText, -1, &aSz[i-1]);
aSz[p->nColumn] += sqlite3_column_bytes(pSelect, i);
}
if( rc!=SQLITE_OK ){
while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){
int iCol;
- rc = fts3PendingTermsDocid(p,
- langidFromSelect(p, pStmt), sqlite3_column_int64(pStmt, 0)
- );
+ int iLangid = langidFromSelect(p, pStmt);
+ rc = fts3PendingTermsDocid(p, iLangid, sqlite3_column_int64(pStmt, 0));
aSz[p->nColumn] = 0;
for(iCol=0; rc==SQLITE_OK && iCol<p->nColumn; iCol++){
const char *z = (const char *) sqlite3_column_text(pStmt, iCol+1);
- rc = fts3PendingTermsAdd(p, z, iCol, &aSz[iCol]);
+ rc = fts3PendingTermsAdd(p, iLangid, z, iCol, &aSz[iCol]);
aSz[p->nColumn] += sqlite3_column_bytes(pStmt, iCol+1);
}
if( p->bHasDocsize ){
const char *zText = (const char *)sqlite3_column_text(pCsr->pStmt, i+1);
sqlite3_tokenizer_cursor *pTC = 0;
- rc = pModule->xOpen(pT, zText, -1, &pTC);
+ rc = sqlite3Fts3OpenTokenizer(pT, pCsr->iLangid, zText, -1, &pTC);
while( rc==SQLITE_OK ){
char const *zToken; /* Buffer containing token */
int nToken; /* Number of bytes in token */
int iDum1, iDum2; /* Dummy variables */
int iPos; /* Position of token in zText */
- pTC->pTokenizer = pT;
rc = pModule->xNext(pTC, &zToken, &nToken, &iDum1, &iDum2, &iPos);
for(pDef=pCsr->pDeferred; pDef && rc==SQLITE_OK; pDef=pDef->pNext){
Fts3PhraseToken *pPT = pDef->pToken;
/* If this is an INSERT or UPDATE operation, insert the new record. */
if( nArg>1 && rc==SQLITE_OK ){
+ int iLangid = sqlite3_value_int(apVal[2 + p->nColumn + 2]);
if( bInsertDone==0 ){
rc = fts3InsertData(p, apVal, pRowid);
if( rc==SQLITE_CONSTRAINT && p->zContentTbl==0 ){
}
}
if( rc==SQLITE_OK && (!isRemove || *pRowid!=p->iPrevDocid ) ){
- rc = fts3PendingTermsDocid(p,
- sqlite3_value_int(apVal[2 + p->nColumn + 2]),
- *pRowid
- );
+ rc = fts3PendingTermsDocid(p, iLangid, *pRowid);
}
if( rc==SQLITE_OK ){
assert( p->iPrevDocid==*pRowid );
- rc = fts3InsertTerms(p, apVal, aSzIns);
+ rc = fts3InsertTerms(p, iLangid, apVal, aSzIns);
}
if( p->bHasDocsize ){
fts3InsertDocsize(&rc, p, aSzIns);
-C Fix\sproblems\swith\scombining\scontent=\sand\slanguageid=\sin\sa\ssingle\sfts4\stable.
-D 2012-03-02T19:53:02.350
+C Add\sthe\sxLanguageid\smethod\sto\ssqlite3_fts3_tokenizer\sversions\s1\sand\sgreater.
+D 2012-03-03T18:46:41.456
F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
F Makefile.in 3f79a373e57c3b92dabf76f40b065e719d31ac34
F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
F ext/fts3/README.syntax a19711dc5458c20734b8e485e75fb1981ec2427a
F ext/fts3/README.tokenizers 998756696647400de63d5ba60e9655036cb966e9
F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d
-F ext/fts3/fts3.c fd89caa4169520c32cf46ca5a62df6dd48201422
+F ext/fts3/fts3.c fcda9a9ff7ccfb9fe4388d36063e3405a652e15f
F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe
-F ext/fts3/fts3Int.h 521d300f2af4e741f53c4e2dd540275fb64533eb
+F ext/fts3/fts3Int.h d1d7f964ddee067bcd16a6af4ba7ecf66220056d
F ext/fts3/fts3_aux.c 72de4cb43db7bfc2f68fbda04b7d8095ae9a6239
-F ext/fts3/fts3_expr.c f5df26bddf46a5916b2a5f80c4027996e92b7b15
+F ext/fts3/fts3_expr.c dbc7ba4c3a6061adde0f38ed8e9b349568299551
F ext/fts3/fts3_hash.c 8dd2d06b66c72c628c2732555a32bc0943114914
F ext/fts3/fts3_hash.h 8331fb2206c609f9fc4c4735b9ab5ad6137c88ec
F ext/fts3/fts3_icu.c 6c8f395cdf9e1e3afa7fadb7e523dbbf381c6dfa
F ext/fts3/fts3_porter.c b7e5276f9f0a5fc7018b6fa55ce0f31f269ef881
-F ext/fts3/fts3_snippet.c 1f9ee6a8e0e242649645968dcec4deb253d86c2a
+F ext/fts3/fts3_snippet.c c9e126c20760988aa7c43c6ea1379db34738282e
F ext/fts3/fts3_term.c d3466cf99432291be08e379d89645462431809d6
-F ext/fts3/fts3_test.c 24fa13f330db011500acb95590da9eee24951894
-F ext/fts3/fts3_tokenizer.c 9ff7ec66ae3c5c0340fa081958e64f395c71a106
-F ext/fts3/fts3_tokenizer.h 13ffd9fcb397fec32a05ef5cd9e0fa659bf3dbd3
+F ext/fts3/fts3_test.c a026412a41450a014ccb7abdd5efaa7c9711d49e
+F ext/fts3/fts3_tokenizer.c 3da7254a9881f7e270ab28e2004e0d22b3212bce
+F ext/fts3/fts3_tokenizer.h 66dec98e365854b6cd2d54f1a96bb6d428fc5a68
F ext/fts3/fts3_tokenizer1.c 0dde8f307b8045565cf63797ba9acfaff1c50c68
-F ext/fts3/fts3_write.c 35b98a42f9bbdd28af1b1f3bb0c09ff07090a764
+F ext/fts3/fts3_write.c f87bb2d27d31cb7a7bf306747079095393c9d073
F ext/fts3/fts3speed.tcl b54caf6a18d38174f1a6e84219950d85e98bb1e9
F ext/fts3/mkfts3amal.tcl 252ecb7fe6467854f2aa237bf2c390b74e71f100
F ext/icu/README.txt bf8461d8cdc6b8f514c080e4e10dc3b2bbdfefa9
F test/fts3sort.test 95be0b19d7e41c44b29014f13ea8bddd495fd659
F test/fts4aa.test 6e7f90420b837b2c685f3bcbe84c868492d40a68
F test/fts4content.test 17b2360f7d1a9a7e5aa8022783f5c5731b6dfd4f
-F test/fts4langid.test be989b5cddcd7596b87232af193f6c4560a34272
+F test/fts4langid.test fabdd5a8db0fa00292e0704809f566e3fb6dba3a
F test/func.test 6c5ce11e3a0021ca3c0649234e2d4454c89110ca
F test/func2.test 772d66227e4e6684b86053302e2d74a2500e1e0f
F test/func3.test 001021e5b88bd02a3b365a5c5fd8f6f49d39744a
F test/pagesize.test 1dd51367e752e742f58e861e65ed7390603827a0
F test/pcache.test 065aa286e722ab24f2e51792c1f093bf60656b16
F test/pcache2.test a83efe2dec0d392f814bfc998def1d1833942025
-F test/permutations.test fa6f0e5f13fe0b1d3f7a7613179b7f7b20028184
+F test/permutations.test 2b5a1b64a8e5114757457fbce9010387d1fe7682
F test/pragma.test f6111ded4d56b79436a60a757d62f3c96a9cf3f5
F test/pragma2.test 3a55f82b954242c642f8342b17dffc8b47472947
F test/printf.test ec9870c4dce8686a37818e0bf1aba6e6a1863552
F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f
F tool/warnings-clang.sh 9f406d66e750e8ac031c63a9ef3248aaa347ef2a
F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381
-P 181bc35731f19c3e4497ba3338c209918d34ea69
-R acfb20f690a18ac8d67e116ae8c76f7d
+P 22491e7bc38aee43819b888e04241cb6a6ef73a3
+R b3e8f4cc7fcd5d7bfa4510a8d2434f16
U dan
-Z b771fbbc3f5dd49e39970a5f917fc8b7
+Z 8fb02bd9c9b9481fcea2986d10daaba1
-22491e7bc38aee43819b888e04241cb6a6ef73a3
\ No newline at end of file
+f8e9c445dd358c40e5a7bf3756b9f291909dbea7
\ No newline at end of file
# 3.* - Tests with content= tables. Both where there is a real
# underlying content table and where there is not.
#
-#
# 4.* - Test that if one is provided, the tokenizer xLanguage method
# is called to configure the tokenizer before tokenizing query
# or document text.
or_merge_lists [rowid_list "zero one"] [rowid_list "one two"]
}
+#-------------------------------------------------------------------------
+# Test cases 4.*
+#
+proc build_multilingual_db_2 {db} {
+ $db eval {
+ CREATE VIRTUAL TABLE t4 USING fts4(
+ tokenize=testtokenizer,
+ languageid=lid
+ );
+ }
+ for {set i 0} {$i < 50} {incr i} {
+ execsql {
+ INSERT INTO t4(docid, content, lid) VALUES($i, 'The Quick Brown Fox', $i)
+ }
+ }
+}
+do_test 4.1.0 {
+ reset_db
+ set ptr [fts3_test_tokenizer]
+ execsql { SELECT fts3_tokenizer('testtokenizer', $ptr) }
+ build_multilingual_db_2 db
+} {}
+do_execsql_test 4.1.1 {
+ SELECT docid FROM t4 WHERE t4 MATCH 'quick';
+} {0}
+do_execsql_test 4.1.2 {
+ SELECT docid FROM t4 WHERE t4 MATCH 'quick' AND lid=1;
+} {}
+do_execsql_test 4.1.3 {
+ SELECT docid FROM t4 WHERE t4 MATCH 'Quick' AND lid=1;
+} {1}
+for {set i 0} {$i < 50} {incr i} {
+ do_execsql_test 4.1.4.$i {
+ SELECT count(*) FROM t4 WHERE t4 MATCH 'fox' AND lid=$i;
+ } [expr 0==($i%2)]
+}
+do_catchsql_test 4.1.5 {
+ INSERT INTO t4(content, lid) VALUES('hello world', 101)
+} {1 {SQL logic error or missing database}}
finish_test
-
fts3aux1.test fts3comp1.test fts3auto.test
fts4aa.test fts4content.test
fts3conf.test fts3prefix.test fts3fault2.test fts3corrupt.test
- fts3corrupt2.test
- fts3first.test
+ fts3corrupt2.test fts3first.test fts4langid.test
}
projects / thirdparty / sqlite.git / commitdiff
