From: dan <dan@noemail.net>
Date: Sat, 26 May 2012 18:28:14 +0000 (+0000)
Subject: If SQLITE_DISABLE_FTS3_UNICODE is defined, do not build the "unicode61" tokenizer.
X-Git-Tag: version-3.7.13~27^2
X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=refs%2Fheads%2Ffts4-unicode;p=thirdparty%2Fsqlite.git

If SQLITE_DISABLE_FTS3_UNICODE is defined, do not build the "unicode61" tokenizer.

FossilOrigin-Name: e71495a817b479bc23c5403d99255e3f098eb054
---

diff --git a/ext/fts3/fts3.c b/ext/fts3/fts3.c
index cb1d175395..3531cb4df7 100644
--- a/ext/fts3/fts3.c
+++ b/ext/fts3/fts3.c
@@ -3554,7 +3554,9 @@ static void hashDestroy(void *p){
 */
 void sqlite3Fts3SimpleTokenizerModule(sqlite3_tokenizer_module const**ppModule);
 void sqlite3Fts3PorterTokenizerModule(sqlite3_tokenizer_module const**ppModule);
+#ifndef SQLITE_DISABLE_FTS3_UNICODE
 void sqlite3Fts3UnicodeTokenizer(sqlite3_tokenizer_module const**ppModule);
+#endif
 #ifdef SQLITE_ENABLE_ICU
 void sqlite3Fts3IcuTokenizerModule(sqlite3_tokenizer_module const**ppModule);
 #endif
@@ -3570,13 +3572,19 @@ int sqlite3Fts3Init(sqlite3 *db){
   Fts3Hash *pHash = 0;
   const sqlite3_tokenizer_module *pSimple = 0;
   const sqlite3_tokenizer_module *pPorter = 0;
+#ifndef SQLITE_DISABLE_FTS3_UNICODE
   const sqlite3_tokenizer_module *pUnicode = 0;
+#endif
 
 #ifdef SQLITE_ENABLE_ICU
   const sqlite3_tokenizer_module *pIcu = 0;
   sqlite3Fts3IcuTokenizerModule(&pIcu);
 #endif
 
+#ifndef SQLITE_DISABLE_FTS3_UNICODE
+  sqlite3Fts3UnicodeTokenizer(&pUnicode);
+#endif
+
 #ifdef SQLITE_TEST
   rc = sqlite3Fts3InitTerm(db);
   if( rc!=SQLITE_OK ) return rc;
@@ -3587,7 +3595,6 @@ int sqlite3Fts3Init(sqlite3 *db){
 
   sqlite3Fts3SimpleTokenizerModule(&pSimple);
   sqlite3Fts3PorterTokenizerModule(&pPorter);
-  sqlite3Fts3UnicodeTokenizer(&pUnicode);
 
   /* Allocate and initialise the hash-table used to store tokenizers. */
   pHash = sqlite3_malloc(sizeof(Fts3Hash));
@@ -3601,7 +3608,10 @@ int sqlite3Fts3Init(sqlite3 *db){
   if( rc==SQLITE_OK ){
     if( sqlite3Fts3HashInsert(pHash, "simple", 7, (void *)pSimple)
      || sqlite3Fts3HashInsert(pHash, "porter", 7, (void *)pPorter) 
+
+#ifndef SQLITE_DISABLE_FTS3_UNICODE
      || sqlite3Fts3HashInsert(pHash, "unicode61", 10, (void *)pUnicode) 
+#endif
 #ifdef SQLITE_ENABLE_ICU
      || (pIcu && sqlite3Fts3HashInsert(pHash, "icu", 4, (void *)pIcu))
 #endif
diff --git a/ext/fts3/fts3_unicode.c b/ext/fts3/fts3_unicode.c
index cbdc1099b3..83b1c322b2 100644
--- a/ext/fts3/fts3_unicode.c
+++ b/ext/fts3/fts3_unicode.c
@@ -13,6 +13,8 @@
 ** Implementation of the "unicode" full-text-search tokenizer.
 */
 
+#ifndef SQLITE_DISABLE_FTS3_UNICODE
+
 #include "fts3Int.h"
 #if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3)
 
@@ -241,3 +243,4 @@ void sqlite3Fts3UnicodeTokenizer(sqlite3_tokenizer_module const **ppModule){
 }
 
 #endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */
+#endif /* ifndef SQLITE_DISABLE_FTS3_UNICODE */
diff --git a/ext/fts3/fts3_unicode2.c b/ext/fts3/fts3_unicode2.c
index 64695e3440..311518dbc2 100644
--- a/ext/fts3/fts3_unicode2.c
+++ b/ext/fts3/fts3_unicode2.c
@@ -15,6 +15,8 @@
 ** DO NOT EDIT THIS MACHINE GENERATED FILE.
 */
 
+#ifndef SQLITE_DISABLE_FTS3_UNICODE
+
 #include <assert.h>
 
 /*
@@ -289,3 +291,4 @@ int sqlite3FtsUnicodeTolower(int c){
 
   return ret;
 }
+#endif /* ifndef SQLITE_DISABLE_FTS3_UNICODE */
diff --git a/ext/fts3/unicode/mkunicode.tcl b/ext/fts3/unicode/mkunicode.tcl
index 9a7854159b..0f78c147e2 100644
--- a/ext/fts3/unicode/mkunicode.tcl
+++ b/ext/fts3/unicode/mkunicode.tcl
@@ -507,6 +507,8 @@ proc print_fileheader {} {
 */
   }]
   puts ""
+  puts "#ifndef SQLITE_DISABLE_FTS3_UNICODE"
+  puts ""
   puts "#include <assert.h>"
   puts ""
 }
@@ -567,3 +569,5 @@ if {$::generate_test_code} {
   print_test_main 
 }
 
+puts "#endif /* ifndef SQLITE_DISABLE_FTS3_UNICODE */"
+
diff --git a/manifest b/manifest
index 2ab322934e..f3ca8b71f6 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Change\sthe\sformat\sof\sthe\stables\sused\sby\ssqlite3FtsUnicodeTolower()\sto\smake\sthem\sa\slittle\ssmaller.
-D 2012-05-26T17:57:02.187
+C If\sSQLITE_DISABLE_FTS3_UNICODE\sis\sdefined,\sdo\snot\sbuild\sthe\s"unicode61"\stokenizer.
+D 2012-05-26T18:28:14.350
 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
 F Makefile.in 2f37e468503dbe79d35c9f6dffcf3fae1ae9ec20
 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@@ -55,7 +55,7 @@ F ext/fts3/README.content fdc666a70d5257a64fee209f97cf89e0e6e32b51
 F ext/fts3/README.syntax a19711dc5458c20734b8e485e75fb1981ec2427a
 F ext/fts3/README.tokenizers e0a8b81383ea60d0334d274fadf305ea14a8c314
 F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d
-F ext/fts3/fts3.c aff512cbc4aea2c5a8d6b7614e81a7b5a4bb822c
+F ext/fts3/fts3.c 41824d0db7d244ca335ce98162df1244863a05c4
 F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe
 F ext/fts3/fts3Int.h 7b163fa22e7a625c404c424f2779a4d7b14c14ad
 F ext/fts3/fts3_aux.c 5205182bd8f372782597888156404766edf5781e
@@ -70,15 +70,15 @@ F ext/fts3/fts3_test.c 348f7d08cae05285794e23dc4fe8b8fdf66e264a
 F ext/fts3/fts3_tokenizer.c 3da7254a9881f7e270ab28e2004e0d22b3212bce
 F ext/fts3/fts3_tokenizer.h 66dec98e365854b6cd2d54f1a96bb6d428fc5a68
 F ext/fts3/fts3_tokenizer1.c 5c98225a53705e5ee34824087478cf477bdb7004
-F ext/fts3/fts3_unicode.c f487c6ef0ac2cc35aec3c3636c0a35483da6453c
-F ext/fts3/fts3_unicode2.c e43024fe05db9316b7825bfce809479fad6ce44e
+F ext/fts3/fts3_unicode.c 76b6f6fe6e86acd75b08272502fae74a13cef310
+F ext/fts3/fts3_unicode2.c 46ff2289f52e661ef030b4aed4c9e5b6132b6ea9
 F ext/fts3/fts3_write.c 6a6391d6b01114f885e24e1f66bbc11ffba0e9e2
 F ext/fts3/fts3speed.tcl b54caf6a18d38174f1a6e84219950d85e98bb1e9
 F ext/fts3/mkfts3amal.tcl 252ecb7fe6467854f2aa237bf2c390b74e71f100
 F ext/fts3/tool/fts3view.c 6cfc5b67a5f0e09c0d698f9fd012c784bfaa9197
 F ext/fts3/unicode/CaseFolding.txt 8c678ca52ecc95e16bc7afc2dbf6fc9ffa05db8c
 F ext/fts3/unicode/UnicodeData.txt cd07314edb62d49fde34debdaf92fa2aa69011e7
-F ext/fts3/unicode/mkunicode.tcl 2775280062b6595dd18e44ddef6327239ab290ab
+F ext/fts3/unicode/mkunicode.tcl de64862abd50307c6e40622970eb9c8f2f14ed0e
 F ext/icu/README.txt bf8461d8cdc6b8f514c080e4e10dc3b2bbdfefa9
 F ext/icu/icu.c eb9ae1d79046bd7871aa97ee6da51eb770134b5a
 F ext/icu/sqliteicu.h 728867a802baa5a96de7495e9689a8e01715ef37
@@ -200,7 +200,7 @@ F src/test_async.c 0612a752896fad42d55c3999a5122af10dcf22ad
 F src/test_autoext.c 30e7bd98ab6d70a62bb9ba572e4c7df347fe645e
 F src/test_backup.c c129c91127e9b46e335715ae2e75756e25ba27de
 F src/test_btree.c 47cd771250f09cdc6e12dda5bc71bc0b3abc96e2
-F src/test_config.c 0de329e736eb4aa5845069bed630e5c72f012264
+F src/test_config.c d2da9f1490c38d9b9cb52f399b6adb81d538273a
 F src/test_demovfs.c 20a4975127993f4959890016ae9ce5535a880094
 F src/test_devsym.c e7498904e72ba7491d142d5c83b476c4e76993bc
 F src/test_func.c 090f2c3339e85c2c964435f99aed6f3da9d59525
@@ -482,7 +482,7 @@ F test/fts3e.test 1f6c6ac9cc8b772ca256e6b22aaeed50c9350851
 F test/fts3expr.test 5e745b2b6348499d9ef8d59015de3182072c564c
 F test/fts3expr2.test 18da930352e5693eaa163a3eacf96233b7290d1a
 F test/fts3fault.test cb72dccb0a3b9f730f16c5240f3fcb9303eb1660
-F test/fts3fault2.test 9e3a0a0a9b0e29623e3fa9f96df346324d18fd48
+F test/fts3fault2.test 3198eef2804deea7cac8403e771d9cbcb752d887
 F test/fts3first.test dbdedd20914c8d539aa3206c9b34a23775644641
 F test/fts3malloc.test b86ea33db9e8c58c0c2f8027a9fcadaf6a1568be
 F test/fts3matchinfo.test 6507fe1c342e542300d65ea637d4110eccf894e6
@@ -501,7 +501,7 @@ F test/fts4langid.test 24a6e41063b416bbdf371ff6b4476fa41c194aa7
 F test/fts4merge.test c424309743fdd203f8e56a1f1cd7872cd66cc0ee
 F test/fts4merge2.test 5faa558d1b672f82b847d2a337465fa745e46891
 F test/fts4merge3.test aab02a09f50fe6baaddc2e159c3eabc116d45fc7
-F test/fts4unicode.test dd0b67a2edde62eb79c34345968f378aaf4dec10
+F test/fts4unicode.test c812e9cf843e26ba633f58b36a2629f878af20fd
 F test/func.test 9809b7622d721904a8cc33c1ffb87f46d506ed01
 F test/func2.test 772d66227e4e6684b86053302e2d74a2500e1e0f
 F test/func3.test 001021e5b88bd02a3b365a5c5fd8f6f49d39744a
@@ -1004,7 +1004,7 @@ F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06
 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f
 F tool/warnings-clang.sh a8a0a3babda96dfb1ff51adda3cbbf3dfb7266c2
 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381
-P 07d3ea8a3cb179fab6c48934fc6751f53b507d36
-R fbb3511eb94203c7d1dd12033a92a0a8
+P b89d3834f6690073fca0fc22c18afa1fb280ea7d
+R 721b18aaca6e897933f70f12a0639abb
 U dan
-Z d24d4148848cb3328b6f3577a1e7fca3
+Z f1ca3c8a69800dd8f2afaacbafb654ad
diff --git a/manifest.uuid b/manifest.uuid
index b13186f9c7..ef8279ad4f 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-b89d3834f6690073fca0fc22c18afa1fb280ea7d
\ No newline at end of file
+e71495a817b479bc23c5403d99255e3f098eb054
\ No newline at end of file
diff --git a/src/test_config.c b/src/test_config.c
index f096ebf236..6b56460602 100644
--- a/src/test_config.c
+++ b/src/test_config.c
@@ -307,6 +307,12 @@ static void set_options(Tcl_Interp *interp){
   Tcl_SetVar2(interp, "sqlite_options", "fts3", "0", TCL_GLOBAL_ONLY);
 #endif
 
+#if !defined(SQLITE_ENABLE_FTS3) || defined(SQLITE_DISABLE_FTS3_UNICODE)
+  Tcl_SetVar2(interp, "sqlite_options", "fts3_unicode", "0", TCL_GLOBAL_ONLY);
+#else
+  Tcl_SetVar2(interp, "sqlite_options", "fts3_unicode", "1", TCL_GLOBAL_ONLY);
+#endif
+
 #ifdef SQLITE_OMIT_GET_TABLE
   Tcl_SetVar2(interp, "sqlite_options", "gettable", "0", TCL_GLOBAL_ONLY);
 #else
diff --git a/test/fts3fault2.test b/test/fts3fault2.test
index 3387c25a1f..f2d10bc3f8 100644
--- a/test/fts3fault2.test
+++ b/test/fts3fault2.test
@@ -131,26 +131,28 @@ do_faultsim_test 4.1 -prep {
   faultsim_test_result {0 {}}
 }
 
-do_test 5.0 {
-  faultsim_delete_and_reopen
-  execsql {
-    CREATE VIRTUAL TABLE ft USING fts4(a, tokenize=unicode61);
-  }
-  faultsim_save_and_close
-} {}
-
-do_faultsim_test 5.1 -faults oom* -prep {
-  faultsim_restore_and_reopen
-  db eval {SELECT * FROM sqlite_master}
-} -body {
-  execsql { INSERT INTO ft VALUES('the quick brown fox'); }
-  execsql { INSERT INTO ft VALUES(
-      'theunusuallylongtokenthatjustdragsonandonandonandthendragsonsomemoreeof'
-    );
+ifcapable fts3_unicode {
+  do_test 5.0 {
+    faultsim_delete_and_reopen
+    execsql {
+      CREATE VIRTUAL TABLE ft USING fts4(a, tokenize=unicode61);
+    }
+    faultsim_save_and_close
+  } {}
+  
+  do_faultsim_test 5.1 -faults oom* -prep {
+    faultsim_restore_and_reopen
+    db eval {SELECT * FROM sqlite_master}
+  } -body {
+    execsql { INSERT INTO ft VALUES('the quick brown fox'); }
+    execsql { INSERT INTO ft VALUES(
+       'theunusuallylongtokenthatjustdragsonandonandonandthendragsonsomemoreeof'
+      );
+    }
+    execsql { SELECT docid FROM ft WHERE ft MATCH 'th*' }
+  } -test {
+    faultsim_test_result {0 {1 2}}
   }
-  execsql { SELECT docid FROM ft WHERE ft MATCH 'th*' }
-} -test {
-  faultsim_test_result {0 {1 2}}
 }
 
 finish_test
diff --git a/test/fts4unicode.test b/test/fts4unicode.test
index 9f9438b2a4..72afac7d30 100644
--- a/test/fts4unicode.test
+++ b/test/fts4unicode.test
@@ -14,7 +14,7 @@
 
 set testdir [file dirname $argv0]
 source $testdir/tester.tcl
-ifcapable !fts3 { finish_test ; return }
+ifcapable !fts3_unicode { finish_test ; return }
 set ::testprefix fts4unicode
 
 proc do_unicode_token_test {tn input res} {