nName = sqlite3_value_bytes(argv[0])+1;
if( argc==2 ){
+#ifdef SQLITE_ENABLE_FTS3_TOKENIZER
void *pOld;
int n = sqlite3_value_bytes(argv[1]);
if( zName==0 || n!=sizeof(pPtr) ){
sqlite3_result_error(context, "out of memory", -1);
return;
}
- }else{
+#else
+ sqlite3_result_error(context, "fts3tokenize: "
+ "disabled - rebuild with -DSQLITE_ENABLE_FTS3_TOKENIZER", -1
+ );
+ return;
+#endif /* SQLITE_ENABLE_FTS3_TOKENIZER */
+ }else
+ {
if( zName ){
pPtr = sqlite3Fts3HashFind(pHash, zName, nName);
}
assert( 0==strcmp(sqlite3_errmsg(db), "unknown tokenizer: nosuchtokenizer") );
/* Test the storage function */
+#ifdef SQLITE_ENABLE_FTS3_TOKENIZER
rc = registerTokenizer(db, "nosuchtokenizer", p1);
assert( rc==SQLITE_OK );
rc = queryTokenizer(db, "nosuchtokenizer", &p2);
assert( rc==SQLITE_OK );
assert( p2==p1 );
+#endif
sqlite3_result_text(context, "ok", -1, SQLITE_STATIC);
}
-C Changes\sto\shelp\sthe\stokenizer\srun\sabout\s33%\sfaster.
-D 2016-02-08T19:36:46.659
+C Disable\sthe\stwo-argument\sform\sof\sthe\sfts3_tokenizer()\sSQL\sfunction\sunless\sthe\slibrary\sis\sbuilt\swith\s-DSQLITE_ENABLE_FTS3_TOKENIZER.
+D 2016-02-08T19:40:17.991
F Makefile.in dac2776c84e0d533b158a9af6e57e05c4a6b19f3
F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
F Makefile.msc fcf377286d910b47e072da1ac7945976337c0925
F ext/fts3/fts3_term.c 88c55a6fa1a51ab494e33dced0401a6c28791fd7
F ext/fts3/fts3_test.c 8a3a78c4458b2d7c631fcf4b152a5cd656fa7038
F ext/fts3/fts3_tokenize_vtab.c a27593ab19657166f6fa5ec073b678cc29a75860
-F ext/fts3/fts3_tokenizer.c 50e7a69a549ac5882cc1971ee43f66aaabc11395
+F ext/fts3/fts3_tokenizer.c 0d3a77bbdc6f394d302f24980fa80667371c6e68
F ext/fts3/fts3_tokenizer.h 64c6ef6c5272c51ebe60fc607a896e84288fcbc3
F ext/fts3/fts3_tokenizer1.c 5c98225a53705e5ee34824087478cf477bdb7004
F ext/fts3/fts3_unicode.c a93f5edc0aff44ef8b06d7cb55b52026541ca145
F src/test_backup.c 2e6e6a081870150f20c526a2e9d0d29cda47d803
F src/test_blob.c e5a7a81d61a780da79101aeb1e60d300af169e07
F src/test_btree.c 2e9978eca99a9a4bfa8cae949efb00886860a64f
-F src/test_config.c 0dee90328e3dedf8ba002ee94b6a7e7ea7726fe4
+F src/test_config.c 7985332c806d1cece793475c75a6abcccde9d331
F src/test_demovfs.c 0de72c2c89551629f58486fde5734b7d90758852
F src/test_devsym.c e7498904e72ba7491d142d5c83b476c4e76993bc
F src/test_fs.c a61f54247fdb843761d709879c3bcd1989b2050c
F test/fts3am.test 218aa6ba0dfc50c7c16b2022aac5c6be593d08d8
F test/fts3an.test a49ccadc07a2f7d646ec1b81bc09da2d85a85b18
F test/fts3ao.test 3e4e3d5e75c076520341d0bdf4eb17c00e8cbde2
-F test/fts3atoken.test e3a126365131a6db52efc20a9a6053cd44e5f289
+F test/fts3atoken.test 76262be798f23a390717d14266f0df551e52a7ee
F test/fts3auto.test b981fea19b132b4e6878f50d7c1f369b28f68eb9
F test/fts3aux1.test f8f287a4a73f381f8fa15b6a70f36245f903d221
F test/fts3aux2.test 7ae2b2c13aefdf4169279a27a5f51780ce57f6ba
F test/fts4growth.test 60d6bb3f78e25b34f533797dd9f2f9402310a13a
F test/fts4growth2.test 13ad4e76451af6e6906c95cdc725d01b00044269
F test/fts4incr.test 4e353a0bd886ea984e56fce9e77724fc923b8d0d
-F test/fts4langid.test 24a6e41063b416bbdf371ff6b4476fa41c194aa7
+F test/fts4langid.test 8bd8759e0d4b04d71771544b861193a6841fee84
F test/fts4merge.test c424309743fdd203f8e56a1f1cd7872cd66cc0ee
F test/fts4merge2.test 5faa558d1b672f82b847d2a337465fa745e46891
F test/fts4merge3.test aab02a09f50fe6baaddc2e159c3eabc116d45fc7
F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b
F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P 852a529a8b112049f67a3126f677c06ae4a22d73 04f7da77c13925c1f1e287f4579bb85518297d81
-R 64edf86f87d772039cf70265114474cb
-T +closed 04f7da77c13925c1f1e287f4579bb85518297d81
-U drh
-Z feb6daf33d3b967be60d8c209f5e5987
+P a050e6f096d40aa5b6275797b96e62c228044f5a
+R b09b6ba902e95153c88d724675282802
+U dan
+Z a5b8919de2cf0c2997cac75a4e3c3a2c
-a050e6f096d40aa5b6275797b96e62c228044f5a
\ No newline at end of file
+e0eb217aca7e4aadf9c44ed20822b78139f7c83c
\ No newline at end of file
Tcl_SetVar2(interp, "sqlite_options", "fts3", "0", TCL_GLOBAL_ONLY);
#endif
+#ifdef SQLITE_ENABLE_FTS3_TOKENIZER
+ Tcl_SetVar2(interp, "sqlite_options", "fts3_tokenizer", "1", TCL_GLOBAL_ONLY);
+#else
+ Tcl_SetVar2(interp, "sqlite_options", "fts3_tokenizer", "0", TCL_GLOBAL_ONLY);
+#endif
+
#ifdef SQLITE_ENABLE_FTS5
Tcl_SetVar2(interp, "sqlite_options", "fts5", "1", TCL_GLOBAL_ONLY);
#else
return
}
-set ::testprefix fts3token
+set ::testprefix fts3atoken
proc escape_string {str} {
set out ""
}
#--------------------------------------------------------------------------
-# Test cases fts3token-1.* are the warm-body test for the SQL scalar
+# Test cases fts3atoken-1.* are the warm-body test for the SQL scalar
# function fts3_tokenizer(). The procedure is as follows:
#
# 1: Verify that there is no such fts3 tokenizer as 'blah'.
#
# 5: Test that the table created to use tokenizer 'blah' is usable.
#
-do_test fts3token-1.1 {
- catchsql {
- CREATE VIRTUAL TABLE t1 USING fts3(content, tokenize blah);
- }
-} {1 {unknown tokenizer: blah}}
-do_test fts3token-1.2 {
- execsql {
+ifcapable fts3_tokenizer {
+ do_test fts3atoken-1.1 {
+ catchsql {
+ CREATE VIRTUAL TABLE t1 USING fts3(content, tokenize blah);
+ }
+ } {1 {unknown tokenizer: blah}}
+ do_test fts3atoken-1.2 {
+ execsql {
+ SELECT fts3_tokenizer('blah', fts3_tokenizer('simple')) IS NULL;
+ }
+ } {0}
+ do_test fts3atoken-1.3 {
+ execsql {
+ SELECT fts3_tokenizer('blah') == fts3_tokenizer('simple');
+ }
+ } {1}
+ do_test fts3atoken-1.4 {
+ catchsql {
+ CREATE VIRTUAL TABLE t1 USING fts3(content, tokenize blah);
+ }
+ } {0 {}}
+ do_test fts3atoken-1.5 {
+ execsql {
+ INSERT INTO t1(content) VALUES('There was movement at the station');
+ INSERT INTO t1(content) VALUES('For the word has passed around');
+ INSERT INTO t1(content) VALUES('That the colt from ol regret had got');
+ SELECT content FROM t1 WHERE content MATCH 'movement'
+ }
+ } {{There was movement at the station}}
+} else {
+ do_catchsql_test 1.6 {
SELECT fts3_tokenizer('blah', fts3_tokenizer('simple')) IS NULL;
- }
-} {0}
-do_test fts3token-1.3 {
- execsql {
- SELECT fts3_tokenizer('blah') == fts3_tokenizer('simple');
- }
-} {1}
-do_test fts3token-1.4 {
- catchsql {
- CREATE VIRTUAL TABLE t1 USING fts3(content, tokenize blah);
- }
-} {0 {}}
-do_test fts3token-1.5 {
- execsql {
- INSERT INTO t1(content) VALUES('There was movement at the station');
- INSERT INTO t1(content) VALUES('For the word has passed around');
- INSERT INTO t1(content) VALUES('That the colt from ol regret had got away');
- SELECT content FROM t1 WHERE content MATCH 'movement'
- }
-} {{There was movement at the station}}
+ } {1 {fts3tokenize: disabled - rebuild with -DSQLITE_ENABLE_FTS3_TOKENIZER}}
+}
#--------------------------------------------------------------------------
-# Test cases fts3token-2.* test error cases in the scalar function based
+# Test cases fts3atoken-2.* test error cases in the scalar function based
# API for getting and setting tokenizers.
#
-do_test fts3token-2.1 {
+do_test fts3atoken-2.1 {
catchsql {
SELECT fts3_tokenizer('nosuchtokenizer');
}
} {1 {unknown tokenizer: nosuchtokenizer}}
#--------------------------------------------------------------------------
-# Test cases fts3token-3.* test the three built-in tokenizers with a
+# Test cases fts3atoken-3.* test the three built-in tokenizers with a
# simple input string via the built-in test function. This is as much
# to test the test function as the tokenizer implementations.
#
-do_test fts3token-3.1 {
+do_test fts3atoken-3.1 {
execsql {
SELECT fts3_tokenizer_test('simple', 'I don''t see how');
}
} {{0 i I 1 don don 2 t t 3 see see 4 how how}}
-do_test fts3token-3.2 {
+do_test fts3atoken-3.2 {
execsql {
SELECT fts3_tokenizer_test('porter', 'I don''t see how');
}
} {{0 i I 1 don don 2 t t 3 see see 4 how how}}
ifcapable icu {
- do_test fts3token-3.3 {
+ do_test fts3atoken-3.3 {
execsql {
SELECT fts3_tokenizer_test('icu', 'I don''t see how');
}
}
#--------------------------------------------------------------------------
-# Test cases fts3token-4.* test the ICU tokenizer. In practice, this
+# Test cases fts3atoken-4.* test the ICU tokenizer. In practice, this
# tokenizer only has two modes - "thai" and "everybody else". Some other
# Asian languages (Lao, Khmer etc.) require the same special treatment as
# Thai, but ICU doesn't support them yet.
} $output
}
- do_icu_test fts3token-4.1 en_US {} {}
- do_icu_test fts3token-4.2 en_US {Test cases fts3} [list \
+ do_icu_test fts3atoken-4.1 en_US {} {}
+ do_icu_test fts3atoken-4.2 en_US {Test cases fts3} [list \
0 test Test 1 cases cases 2 fts3 fts3
]
append output "1 \u0e19\u0e30 \u0e19\u0e30 "
append output "2 \u0e04\u0e23\u0e31\u0e1a \u0e04\u0e23\u0e31\u0e1a"
- do_icu_test fts3token-4.3 th_TH $input $output
- do_icu_test fts3token-4.4 en_US $input $output
+ do_icu_test fts3atoken-4.3 th_TH $input $output
+ do_icu_test fts3atoken-4.4 en_US $input $output
# ICU handles an unknown locale by falling back to the default.
# So this is not an error.
- do_icu_test fts3token-4.5 MiddleOfTheOcean $input $output
+ do_icu_test fts3atoken-4.5 MiddleOfTheOcean $input $output
set longtoken "AReallyReallyLongTokenOneThatWillSurelyRequire"
append longtoken "AReallocInTheIcuTokenizerCode"
append output "2 then then "
append output "3 [string tolower $longtoken] $longtoken"
- do_icu_test fts3token-4.6 MiddleOfTheOcean $input $output
- do_icu_test fts3token-4.7 th_TH $input $output
- do_icu_test fts3token-4.8 en_US $input $output
+ do_icu_test fts3atoken-4.6 MiddleOfTheOcean $input $output
+ do_icu_test fts3atoken-4.7 th_TH $input $output
+ do_icu_test fts3atoken-4.8 en_US $input $output
do_execsql_test 5.1 {
CREATE VIRTUAL TABLE x1 USING fts3(name,TOKENIZE icu en_US);
} {}
}
-do_test fts3token-internal {
+do_test fts3atoken-internal {
execsql { SELECT fts3_tokenizer_internal_test() }
} {ok}
do_catchsql_test 6.2.1 {
SELECT fts3_tokenizer(NULL);
} {1 {unknown tokenizer: }}
-do_catchsql_test 6.2.2 {
- SELECT fts3_tokenizer(NULL, X'1234567812345678');
-} {1 {argument type mismatch}}
-do_catchsql_test 6.2.3 {
- SELECT fts3_tokenizer(NULL, X'12345678');
-} {1 {argument type mismatch}}
+ifcapable fts3_tokenizer {
+ do_catchsql_test 6.2.2 {
+ SELECT fts3_tokenizer(NULL, X'1234567812345678');
+ } {1 {argument type mismatch}}
+ do_catchsql_test 6.2.3 {
+ SELECT fts3_tokenizer(NULL, X'12345678');
+ } {1 {argument type mismatch}}
+}
finish_test
}
}
-do_test 4.1.0 {
- reset_db
- set ptr [fts3_test_tokenizer]
- execsql { SELECT fts3_tokenizer('testtokenizer', $ptr) }
- build_multilingual_db_2 db
-} {}
-do_execsql_test 4.1.1 {
- SELECT docid FROM t4 WHERE t4 MATCH 'quick';
-} {0}
-do_execsql_test 4.1.2 {
- SELECT docid FROM t4 WHERE t4 MATCH 'quick' AND lid=1;
-} {}
-do_execsql_test 4.1.3 {
- SELECT docid FROM t4 WHERE t4 MATCH 'Quick' AND lid=1;
-} {1}
-for {set i 0} {$i < 50} {incr i} {
- do_execsql_test 4.1.4.$i {
- SELECT count(*) FROM t4 WHERE t4 MATCH 'fox' AND lid=$i;
- } [expr 0==($i%2)]
-}
-do_catchsql_test 4.1.5 {
- INSERT INTO t4(content, lid) VALUES('hello world', 101)
-} {1 {SQL logic error or missing database}}
+ifcapable fts3_tokenizer {
+ do_test 4.1.0 {
+ reset_db
+ set ptr [fts3_test_tokenizer]
+ execsql { SELECT fts3_tokenizer('testtokenizer', $ptr) }
+ build_multilingual_db_2 db
+ } {}
+ do_execsql_test 4.1.1 {
+ SELECT docid FROM t4 WHERE t4 MATCH 'quick';
+ } {0}
+ do_execsql_test 4.1.2 {
+ SELECT docid FROM t4 WHERE t4 MATCH 'quick' AND lid=1;
+ } {}
+ do_execsql_test 4.1.3 {
+ SELECT docid FROM t4 WHERE t4 MATCH 'Quick' AND lid=1;
+ } {1}
+ for {set i 0} {$i < 50} {incr i} {
+ do_execsql_test 4.1.4.$i {
+ SELECT count(*) FROM t4 WHERE t4 MATCH 'fox' AND lid=$i;
+ } [expr 0==($i%2)]
+ }
+ do_catchsql_test 4.1.5 {
+ INSERT INTO t4(content, lid) VALUES('hello world', 101)
+ } {1 {SQL logic error or missing database}}
+}
#-------------------------------------------------------------------------
# Test cases 5.*
projects / thirdparty / sqlite.git / commitdiff
