} \
}
+#define SKIP_UTF8(zIn) { \
+ if( ((unsigned char)(*(zIn++)))>=0xc0 ){ \
+ while( (((unsigned char)*zIn) & 0xc0)==0x80 ){ zIn++; } \
+ } \
+}
+
#endif /* ifndef SQLITE_AMALGAMATION */
typedef struct Unicode61Tokenizer Unicode61Tokenizer;
typedef struct TrigramTokenizer TrigramTokenizer;
struct TrigramTokenizer {
int bFold; /* True to fold to lower-case */
+ int iFoldParam; /* Parameter to pass to Fts5UnicodeFold() */
};
/*
}else{
pNew->bFold = (zArg[0]=='0');
}
+ }else if( 0==sqlite3_stricmp(azArg[i], "remove_diacritics") ){
+ if( (zArg[0]!='0' && zArg[0]!='1' && zArg[0]!='2') || zArg[1] ){
+ rc = SQLITE_ERROR;
+ }else{
+ pNew->iFoldParam = (zArg[0]!='0') ? 2 : 0;
+ }
}else{
rc = SQLITE_ERROR;
}
}
+
+ if( pNew->iFoldParam!=0 && pNew->bFold==0 ){
+ rc = SQLITE_ERROR;
+ }
+
if( rc!=SQLITE_OK ){
fts5TriDelete((Fts5Tokenizer*)pNew);
pNew = 0;
TrigramTokenizer *p = (TrigramTokenizer*)pTok;
int rc = SQLITE_OK;
char aBuf[32];
+ char *zOut = aBuf;
+ int ii;
const unsigned char *zIn = (const unsigned char*)pText;
const unsigned char *zEof = &zIn[nText];
u32 iCode;
+ int aStart[3]; /* Input offset of each character in aBuf[] */
UNUSED_PARAM(unusedFlags);
- while( 1 ){
- char *zOut = aBuf;
- int iStart = zIn - (const unsigned char*)pText;
- const unsigned char *zNext;
-
- READ_UTF8(zIn, zEof, iCode);
- if( iCode==0 ) break;
- zNext = zIn;
- if( zIn<zEof ){
- if( p->bFold ) iCode = sqlite3Fts5UnicodeFold(iCode, 0);
- WRITE_UTF8(zOut, iCode);
+
+ /* Populate aBuf[] with the characters for the first trigram. */
+ for(ii=0; ii<3; ii++){
+ do {
+ aStart[ii] = zIn - (const unsigned char*)pText;
READ_UTF8(zIn, zEof, iCode);
- if( iCode==0 ) break;
- }else{
- break;
- }
- if( zIn<zEof ){
- if( p->bFold ) iCode = sqlite3Fts5UnicodeFold(iCode, 0);
- WRITE_UTF8(zOut, iCode);
+ if( iCode==0 ) return SQLITE_OK;
+ if( p->bFold ) iCode = sqlite3Fts5UnicodeFold(iCode, p->iFoldParam);
+ }while( iCode==0 );
+ WRITE_UTF8(zOut, iCode);
+ }
+
+ /* At the start of each iteration of this loop:
+ **
+ ** aBuf: Contains 3 characters. The 3 characters of the next trigram.
+ ** zOut: Points to the byte following the last character in aBuf.
+ ** aStart[3]: Contains the byte offset in the input text corresponding
+ ** to the start of each of the three characters in the buffer.
+ */
+ assert( zIn<=zEof );
+ while( 1 ){
+ int iNext; /* Start of character following current tri */
+ const char *z1;
+
+ /* Read characters from the input up until the first non-diacritic */
+ do {
+ iNext = zIn - (const unsigned char*)pText;
READ_UTF8(zIn, zEof, iCode);
if( iCode==0 ) break;
- if( p->bFold ) iCode = sqlite3Fts5UnicodeFold(iCode, 0);
- WRITE_UTF8(zOut, iCode);
- }else{
- break;
- }
- rc = xToken(pCtx, 0, aBuf, zOut-aBuf, iStart, iStart + zOut-aBuf);
- if( rc!=SQLITE_OK ) break;
- zIn = zNext;
+ if( p->bFold ) iCode = sqlite3Fts5UnicodeFold(iCode, p->iFoldParam);
+ }while( iCode==0 );
+
+ /* Pass the current trigram back to fts5 */
+ rc = xToken(pCtx, 0, aBuf, zOut-aBuf, aStart[0], iNext);
+ if( iCode==0 || rc!=SQLITE_OK ) break;
+
+ /* Remove the first character from buffer aBuf[]. Append the character
+ ** with codepoint iCode. */
+ z1 = aBuf;
+ SKIP_UTF8(z1);
+ memmove(aBuf, z1, zOut - z1);
+ zOut -= (z1 - aBuf);
+ WRITE_UTF8(zOut, iCode);
+
+ /* Update the aStart[] array */
+ aStart[0] = aStart[1];
+ aStart[1] = aStart[2];
+ aStart[2] = iNext;
}
return rc;
){
if( xCreate==fts5TriCreate ){
TrigramTokenizer *p = (TrigramTokenizer*)pTok;
- return p->bFold ? FTS5_PATTERN_LIKE : FTS5_PATTERN_GLOB;
+ if( p->iFoldParam==0 ){
+ return p->bFold ? FTS5_PATTERN_LIKE : FTS5_PATTERN_GLOB;
+ }
}
return FTS5_PATTERN_NONE;
}
--- /dev/null
+# 2023 October 24
+#
+# The author disclaims copyright to this source code. In place of
+# a legal notice, here is a blessing:
+#
+# May you do good and not evil.
+# May you find forgiveness for yourself and forgive others.
+# May you share freely, never taking more than you give.
+#
+#*************************************************************************
+#
+# Tests for the fts5 "trigram" tokenizer.
+#
+
+source [file join [file dirname [info script]] fts5_common.tcl]
+ifcapable !fts5 { finish_test ; return }
+set ::testprefix fts5trigram2
+
+do_execsql_test 1.0 "
+ CREATE VIRTUAL TABLE t1 USING fts5(y, tokenize='trigram remove_diacritics 1');
+ INSERT INTO t1 VALUES('abc\u0303defghijklm');
+ INSERT INTO t1 VALUES('a\u0303b\u0303c\u0303defghijklm');
+"
+
+do_execsql_test 1.1 {
+ SELECT highlight(t1, 0, '(', ')') FROM t1('abc');
+} [list \
+ "(abc\u0303)defghijklm" \
+ "(a\u0303b\u0303c\u0303)defghijklm" \
+]
+
+do_execsql_test 1.2 {
+ SELECT highlight(t1, 0, '(', ')') FROM t1('bcde');
+} [list \
+ "a(bc\u0303de)fghijklm" \
+ "a\u0303(b\u0303c\u0303de)fghijklm" \
+]
+
+do_execsql_test 1.3 {
+ SELECT highlight(t1, 0, '(', ')') FROM t1('cdef');
+} [list \
+ "ab(c\u0303def)ghijklm" \
+ "a\u0303b\u0303(c\u0303def)ghijklm" \
+]
+
+do_execsql_test 1.4 {
+ SELECT highlight(t1, 0, '(', ')') FROM t1('def');
+} [list \
+ "abc\u0303(def)ghijklm" \
+ "a\u0303b\u0303c\u0303(def)ghijklm" \
+]
+
+
+#-------------------------------------------------------------------------
+do_catchsql_test 2.0 {
+ CREATE VIRTUAL TABLE t2 USING fts5(
+ z, tokenize='trigram case_sensitive 1 remove_diacritics 1'
+ );
+} {1 {error in tokenizer constructor}}
+
+do_execsql_test 2.1 {
+ CREATE VIRTUAL TABLE t2 USING fts5(
+ z, tokenize='trigram case_sensitive 0 remove_diacritics 1'
+ );
+}
+do_execsql_test 2.2 "
+ INSERT INTO t2 VALUES('\u00E3bcdef');
+ INSERT INTO t2 VALUES('b\u00E3cdef');
+ INSERT INTO t2 VALUES('bc\u00E3def');
+ INSERT INTO t2 VALUES('bcd\u00E3ef');
+"
+
+do_execsql_test 2.3 {
+ SELECT highlight(t2, 0, '(', ')') FROM t2('abc');
+} "(\u00E3bc)def"
+do_execsql_test 2.4 {
+ SELECT highlight(t2, 0, '(', ')') FROM t2('bac');
+} "(b\u00E3c)def"
+do_execsql_test 2.5 {
+ SELECT highlight(t2, 0, '(', ')') FROM t2('bca');
+} "(bc\u00E3)def"
+do_execsql_test 2.6 "
+ SELECT highlight(t2, 0, '(', ')') FROM t2('\u00E3bc');
+" "(\u00E3bc)def"
+
+#-------------------------------------------------------------------------
+do_execsql_test 3.0 {
+ CREATE VIRTUAL TABLE t3 USING fts5(
+ z, tokenize='trigram remove_diacritics 1'
+ );
+} {}
+do_execsql_test 3.1 "
+ INSERT INTO t3 VALUES ('\u0303abc\u0303');
+"
+do_execsql_test 3.2 {
+ SELECT highlight(t3, 0, '(', ')') FROM t3('abc');
+} "\u0303(abc\u0303)"
+
+#-------------------------------------------------------------------------
+do_execsql_test 4.0 {
+ CREATE VIRTUAL TABLE t4 USING fts5(z, tokenize=trigram);
+} {}
+
+breakpoint
+do_execsql_test 4.1 {
+ INSERT INTO t4 VALUES('ABCD');
+} {}
+
+finish_test
-C One\smore\stweak\sto\stool/srctree-check.tcl\sso\sthat\sa\scomplete\sbuild\scan\sbe\naccomplished\sfrom\sa\sread-only\ssource\stree.
-D 2023-11-02T13:10:16.285
+C Add\sthe\s"remove_diacritics"\soption\sto\sthe\sfts5\strigram\stokenizer.
+D 2023-11-02T17:31:06.153
F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
F ext/fts5/fts5_tcl.c b1445cbe69908c411df8084a10b2485500ac70a9c747cdc8cda175a3da59d8ae
F ext/fts5/fts5_test_mi.c 08c11ec968148d4cb4119d96d819f8c1f329812c568bac3684f5464be177d3ee
F ext/fts5/fts5_test_tok.c 3cb0a9b508b30d17ef025ccddd26ae3dc8ddffbe76c057616e59a9aa85d36f3b
-F ext/fts5/fts5_tokenize.c 5e251efb0f1af99a25ed50010ba6b1ad1250aca5921af1988fdcabe5ebc3cb43
+F ext/fts5/fts5_tokenize.c 5a895f3bf366a87cb8c0812445736248c236e7ea664a10125021662acd6b0ba4
F ext/fts5/fts5_unicode2.c eca63dbc797f8ff0572e97caf4631389c0ab900d6364861b915bdd4735973f00
F ext/fts5/fts5_varint.c e64d2113f6e1bfee0032972cffc1207b77af63319746951bf1d09885d1dadf80
F ext/fts5/fts5_vocab.c aed56169ae5c1aa9b8189c779ffeef04ed516d3c712c06914e6d91a6759f4e4a
F ext/fts5/test/fts5tok2.test dcacb32d4a2a3f0dd3215d4a3987f78ae4be21a2
F ext/fts5/test/fts5tokenizer.test ac3c9112b263a639fb0508ae73a3ee886bf4866d2153771a8e8a20c721305a43
F ext/fts5/test/fts5trigram.test 6c4e37864f3e7d90673db5563d9736d7e40080ab94d10ebdffa94c1b77941da0
+F ext/fts5/test/fts5trigram2.test 9fe4207f8a4241747aff1005258b564958588d21bfd240d6cd4c2e955d31c156
F ext/fts5/test/fts5ubsan.test 783d5a8d13ebfa169e634940228db54540780e3ba7a87ad1e4510e61440bf64b
F ext/fts5/test/fts5umlaut.test a42fe2fe6387c40c49ab27ccbd070e1ae38e07f38d05926482cc0bccac9ad602
F ext/fts5/test/fts5unicode.test 17056f4efe6b0a5d4f41fdf7a7dc9af2873004562eaa899d40633b93dc95f5a9
F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 58eb5440d75fb13c1a089cb935de6fb94b4322e10e349db8f11f37b9a2fda2fc
-R 18054cea9bc23d801f864db646273e7d
-U drh
-Z 72328c40506425c913ea7fab75e23b19
+P cba9f0601ca995ac6952aa3a83f7264f6d25aaaa2ea36b19e90cbf591077de72
+R 91c6db43050bc1c48384df29a062813f
+U dan
+Z e9e46724e46ba97d1a7a1bb71106fcd0
# Remove this line to create a well-formed Fossil manifest.
-cba9f0601ca995ac6952aa3a83f7264f6d25aaaa2ea36b19e90cbf591077de72
\ No newline at end of file
+83da80135b6105f47d1de560232449562ae8ac176c8011a6f75589f62bc9b1db
\ No newline at end of file
projects / thirdparty / sqlite.git / commitdiff
