Change the name of the "unicode" tokenizer to "unicode61" to emphasize that the case...

FossilOrigin-Name: 8f3e60aa2253f21bcee5d03982cfdd7f16c00060

diff --git a/ext/fts3/README.tokenizers b/ext/fts3/README.tokenizers
index e06803acc0599d5636cb062b29e36dc04823daf1..7f2345a81fd0e708fc5c78dfbecc0170a4944f72 100644 (file)
--- a/ext/fts3/README.tokenizers
+++ b/ext/fts3/README.tokenizers
@@ -11,7 +11,7 @@
     );
 
   The built-in tokenizers (valid values to pass as <tokenizer name>) are
-  "simple" and "porter".
+  "simple", "porter" and "unicode".
 
   <tokenizer-args> should consist of zero or more white-space separated
   arguments to pass to the selected tokenizer implementation. The 

diff --git a/ext/fts3/fts3.c b/ext/fts3/fts3.c
index 408f192815d9c3668c482609d1ac2f1c767fd85c..cb1d175395255a597882b4eeaa1bb1cf2d7b1eee 100644 (file)
--- a/ext/fts3/fts3.c
+++ b/ext/fts3/fts3.c
@@ -3601,7 +3601,7 @@ int sqlite3Fts3Init(sqlite3 *db){
   if( rc==SQLITE_OK ){
     if( sqlite3Fts3HashInsert(pHash, "simple", 7, (void *)pSimple)
      || sqlite3Fts3HashInsert(pHash, "porter", 7, (void *)pPorter) 
-     || sqlite3Fts3HashInsert(pHash, "unicode", 8, (void *)pUnicode) 
+     || sqlite3Fts3HashInsert(pHash, "unicode61", 10, (void *)pUnicode) 
 #ifdef SQLITE_ENABLE_ICU
      || (pIcu && sqlite3Fts3HashInsert(pHash, "icu", 4, (void *)pIcu))
 #endif

diff --git a/manifest b/manifest
index 575fce7aa7a5dd340f8998bc1a99557ad70bcc0a..6e9d8051886b224e7455ff16dcbc8fedb71b301e 100644 (file)
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Add\sspecial\sfast\spaths\sto\ssqlite3FtsUnicodeTolower()\sand\sIsalnum()\sfor\scodepoints\sin\sthe\sASCII\srange.
-D 2012-05-25T19:50:12.903
+C Change\sthe\sname\sof\sthe\s"unicode"\stokenizer\sto\s"unicode61"\sto\semphasize\sthat\sthe\scase\sfolding\sand\sseparator-character\sidentification\sroutines\sare\sbased\son\sunicode\sversion\s6.1.
+D 2012-05-26T14:54:50.307
 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
 F Makefile.in 2f37e468503dbe79d35c9f6dffcf3fae1ae9ec20
 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@@ -53,9 +53,9 @@ F ext/fts2/fts2_tokenizer1.c 0123d21078e053bd98fd6186c5c6dc6d67969f2e
 F ext/fts2/mkfts2amal.tcl 974d5d438cb3f7c4a652639262f82418c1e4cff0
 F ext/fts3/README.content fdc666a70d5257a64fee209f97cf89e0e6e32b51
 F ext/fts3/README.syntax a19711dc5458c20734b8e485e75fb1981ec2427a
-F ext/fts3/README.tokenizers 998756696647400de63d5ba60e9655036cb966e9
+F ext/fts3/README.tokenizers e0a8b81383ea60d0334d274fadf305ea14a8c314
 F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d
-F ext/fts3/fts3.c 81c77264290b88ed80b7ad23ced152193eefae8f
+F ext/fts3/fts3.c aff512cbc4aea2c5a8d6b7614e81a7b5a4bb822c
 F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe
 F ext/fts3/fts3Int.h 7b163fa22e7a625c404c424f2779a4d7b14c14ad
 F ext/fts3/fts3_aux.c 5205182bd8f372782597888156404766edf5781e
@@ -501,7 +501,7 @@ F test/fts4langid.test 24a6e41063b416bbdf371ff6b4476fa41c194aa7
 F test/fts4merge.test c424309743fdd203f8e56a1f1cd7872cd66cc0ee
 F test/fts4merge2.test 5faa558d1b672f82b847d2a337465fa745e46891
 F test/fts4merge3.test aab02a09f50fe6baaddc2e159c3eabc116d45fc7
-F test/fts4unicode.test 0627683f8ca06035d677d4becc2cd0dc57149ef7
+F test/fts4unicode.test 073546a151a40c591062d4e66934964a2c3731f8
 F test/func.test 9809b7622d721904a8cc33c1ffb87f46d506ed01
 F test/func2.test 772d66227e4e6684b86053302e2d74a2500e1e0f
 F test/func3.test 001021e5b88bd02a3b365a5c5fd8f6f49d39744a
@@ -1004,7 +1004,7 @@ F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06
 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f
 F tool/warnings-clang.sh a8a0a3babda96dfb1ff51adda3cbbf3dfb7266c2
 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381
-P 3dc567ef4702d9a63d78d11ff705cb7f7359f7a6
-R a4a319ece993f7d02e2811bac8d8bd2b
+P cf7b25d47687635a04f4347d45f135c686b9d758
+R 5906d3a9fcf2b2ea0b4312db10ed028e
 U dan
-Z e7946b1fde61e6ce4870009a94d8c133
+Z 9d1b06ce70a747334f00416ba78508e5

diff --git a/manifest.uuid b/manifest.uuid
index 35c2e0876820aa9404df74014b289f36dbb3c52b..87cd2ff7b632398d1b55abd9d158dd9379fa7bec 100644 (file)
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-cf7b25d47687635a04f4347d45f135c686b9d758
\ No newline at end of file
+8f3e60aa2253f21bcee5d03982cfdd7f16c00060
\ No newline at end of file

diff --git a/test/fts4unicode.test b/test/fts4unicode.test
index f08f12bec36dce43bb869d8e229568a27261f062..502df97e81feb8016b84d02179ca17c65d46a8b2 100644 (file)
--- a/test/fts4unicode.test
+++ b/test/fts4unicode.test
@@ -20,7 +20,7 @@ set ::testprefix fts4unicode
 proc do_unicode_token_test {tn input res} {
   set input [string map {' ''} $input]
   uplevel [list do_execsql_test $tn "
-    SELECT fts3_tokenizer_test('unicode', '$input');
+    SELECT fts3_tokenizer_test('unicode61', '$input');
   " [list [list {*}$res]]]
 }
 
@@ -40,5 +40,100 @@ do_unicode_token_test 1.7 "The\u00bfquick\u224ebrown\u2263fox" {
   0 the The 1 quick quick 2 brown brown 3 fox fox
 }
 
+#-------------------------------------------------------------------------
+#
+set docs [list {
+  Enhance the INSERT syntax to allow multiple rows to be inserted via the
+  VALUES clause.
+} {
+  Enhance the CREATE VIRTUAL TABLE command to support the IF NOT EXISTS clause.
+} {
+  Added the sqlite3_stricmp() interface as a counterpart to sqlite3_strnicmp().
+} {
+  Added the sqlite3_db_readonly() interface.
+} {
+  Added the SQLITE_FCNTL_PRAGMA file control, giving VFS implementations the
+  ability to add new PRAGMA statements or to override built-in PRAGMAs.  
+} {
+  Queries of the form: "SELECT max(x), y FROM table" returns the value of y on
+  the same row that contains the maximum x value.
+} {
+  Added support for the FTS4 languageid option.
+} {
+  Documented support for the FTS4 content option. This feature has actually
+  been in the code since version 3.7.9 but is only now considered to be
+  officially supported.  
+} {
+  Pending statements no longer block ROLLBACK. Instead, the pending statement
+  will return SQLITE_ABORT upon next access after the ROLLBACK.  
+} {
+  Improvements to the handling of CSV inputs in the command-line shell
+} {
+  Fix a bug introduced in version 3.7.10 that might cause a LEFT JOIN to be
+  incorrectly converted into an INNER JOIN if the WHERE clause indexable terms
+  connected by OR.  
+}]
+
+set map(a) [list "\u00C4" "\u00E4"]  ; # LATIN LETTER A WITH DIAERESIS
+set map(e) [list "\u00CB" "\u00EB"]  ; # LATIN LETTER E WITH DIAERESIS
+set map(i) [list "\u00CF" "\u00EF"]  ; # LATIN LETTER I WITH DIAERESIS
+set map(o) [list "\u00D6" "\u00F6"]  ; # LATIN LETTER O WITH DIAERESIS
+set map(u) [list "\u00DC" "\u00FC"]  ; # LATIN LETTER U WITH DIAERESIS
+set map(y) [list "\u0178" "\u00FF"]  ; # LATIN LETTER Y WITH DIAERESIS
+set map(h) [list "\u1E26" "\u1E27"]  ; # LATIN LETTER H WITH DIAERESIS
+set map(w) [list "\u1E84" "\u1E85"]  ; # LATIN LETTER W WITH DIAERESIS
+set map(x) [list "\u1E8C" "\u1E8D"]  ; # LATIN LETTER X WITH DIAERESIS
+foreach k [array names map] {
+  lappend mappings [string toupper $k] [lindex $map($k) 0] 
+  lappend mappings $k [lindex $map($k) 1]
+}
+proc mapdoc {doc} { 
+  set doc [regsub -all {[[:space:]]+} $doc " "]
+  string map $::mappings [string trim $doc] 
+}
+
+do_test 2.0 {
+  execsql { CREATE VIRTUAL TABLE t2 USING fts4(tokenize=unicode61, x); }
+  foreach doc $docs {
+    set d [mapdoc $doc]
+    execsql { INSERT INTO t2 VALUES($d) }
+  }
+} {}
+
+do_test 2.1 {
+  set q [mapdoc "row"]
+  execsql { SELECT * FROM t2 WHERE t2 MATCH $q }
+} [list [mapdoc {
+  Queries of the form: "SELECT max(x), y FROM table" returns the value of y on
+  the same row that contains the maximum x value.
+}]]
+
+foreach {tn query snippet} {
+  2 "row" {
+     ...returns the value of y on the same [row] that contains 
+     the maximum x value.
+  }
+  3 "ROW" {
+     ...returns the value of y on the same [row] that contains 
+     the maximum x value.
+  }
+  4 "rollback" {
+     ...[ROLLBACK]. Instead, the pending statement
+     will return SQLITE_ABORT upon next access after the [ROLLBACK].
+  }
+  5 "rOllback" {
+     ...[ROLLBACK]. Instead, the pending statement
+     will return SQLITE_ABORT upon next access after the [ROLLBACK].
+  }
+  6 "lang*" {
+     Added support for the FTS4 [languageid] option.
+  }
+} {
+  do_test 2.$tn {
+    set q [mapdoc $query]
+    execsql { SELECT snippet(t2, '[', ']', '...') FROM t2 WHERE t2 MATCH $q }
+  } [list [mapdoc $snippet]]
+}
+
 finish_test