From: dan <Dan Kennedy>
Date: Fri, 5 Sep 2025 11:20:11 +0000 (+0000)
Subject: Add tests to check that SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER is working.
X-Git-Tag: major-release~239
X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=b9bf2d93821938caf16989e11950fc5cc768f368;p=thirdparty%2Fsqlite.git

Add tests to check that SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER is working.

FossilOrigin-Name: beb092d8105e5a1ce95339527bf96799e2ce073591b055f71a5f1dc1d0a0c49c
---

diff --git a/manifest b/manifest
index 561e51ce3c..7723baeced 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Generic\sdoc\supdates\sin\sext/wasm's\smakefiles.\sNo\sfunctional\schanges.
-D 2025-09-04T18:00:53.260
+C Add\stests\sto\scheck\sthat\sSQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER\sis\sworking.
+D 2025-09-05T11:20:11.669
 F .fossil-settings/binary-glob 61195414528fb3ea9693577e1980230d78a1f8b0a54c78cf1b9b24d0a409ed6a x
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
@@ -1147,6 +1147,7 @@ F test/fts3am.test 218aa6ba0dfc50c7c16b2022aac5c6be593d08d8
 F test/fts3an.test a49ccadc07a2f7d646ec1b81bc09da2d85a85b18
 F test/fts3ao.test 266989148fec6d9f1bb6c5382f7aa3dcea0e9cd444576e28dd2b9287ac7dd220
 F test/fts3atoken.test dc2078ce464914efe3a8dfc545dd034a0fc14f2ab425c240471d5a5f1c721400
+F test/fts3atoken2.test aa6664d26277064844ead962f5988c28b091c51b112f36d30fa33bd83fecf906
 F test/fts3auto.test 649aa4c198d7acc5cd6355e19ee073d051c40d9e88a43fc3d88af46bdf3e99d5
 F test/fts3aux1.test 1880eaa75c586cd10f53080479a2b819b3915ae7ce55c4e0ba8f1fe05ac0a6a7
 F test/fts3aux2.test 2459e7fa3e22734aed237d1e2ae192f5541c4d8b218956ad2d90754977bf907f
@@ -2172,8 +2173,8 @@ F tool/version-info.c 3b36468a90faf1bbd59c65fd0eb66522d9f941eedd364fabccd7227350
 F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee87c1b31a7
 F tool/warnings.sh 1ad0169b022b280bcaaf94a7fa231591be96b514230ab5c98fbf15cd7df842dd
 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P 26ecdde06cce063143144a79c0b4979ed5ec27548da6f127176a420c1c3f17ce
-R c906f44fce625506b02efa9aa8f0e11e
-U stephan
-Z a1c51bbc3308c27ff98d60395e238267
+P 02721457cea255a5117a46b77cc87d2e09acb64340ce94089c5b5e6edc5b5033
+R 26f54bfe6c4276aca1cf86f77dc2e1d0
+U dan
+Z 6d55ec4bbf370fd1d9d59964043fa3e8
 # Remove this line to create a well-formed Fossil manifest.
diff --git a/manifest.uuid b/manifest.uuid
index eedd2d7a66..d536f22c26 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-02721457cea255a5117a46b77cc87d2e09acb64340ce94089c5b5e6edc5b5033
+beb092d8105e5a1ce95339527bf96799e2ce073591b055f71a5f1dc1d0a0c49c
diff --git a/test/fts3atoken2.test b/test/fts3atoken2.test
new file mode 100644
index 0000000000..f6a2a29ab1
--- /dev/null
+++ b/test/fts3atoken2.test
@@ -0,0 +1,106 @@
+# 2025 September 5
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#*************************************************************************
+# This file implements regression tests for SQLite library. The focus 
+# of this script is testing the pluggable tokeniser feature of the 
+# FTS3 module.
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+
+# If SQLITE_ENABLE_FTS3 is defined, omit this file.
+ifcapable !fts3 {
+  finish_test
+  return
+}
+
+set ::testprefix fts3atoken2
+
+
+reset_db
+sqlite3_db_config db SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER 0
+
+# With ENABLE_FTS3_TOKENIZER set to 0:
+#
+#   * It is not possible to get a pointer to a token implementation
+#     using single arg fts3_tokenize() unless the name of the tokenizer
+#     is a bound paramter - function should return NULL.
+#
+#   * But it is possible with a bound parameter.
+#
+do_execsql_test 1.1.1 {
+  SELECT typeof( fts3_tokenizer('simple') );
+} {null}
+set bound "simple"
+do_execsql_test 1.1.2 {
+  SELECT typeof( fts3_tokenizer($bound) );
+} {blob}
+
+# With ENABLE_FTS3_TOKENIZER set to 0:
+#
+#   * It is not possible to create a token implementation using anything
+#     other than a bound parameter.
+#
+#   * But it is possible with a bound parameter.
+#
+set literal [db one {SELECT quote( fts3_tokenizer($bound) )}]
+set blob [db one {SELECT fts3_tokenizer($bound) }]
+
+do_catchsql_test 1.2.1 "
+  SELECT fts3_tokenizer('mytok', $literal)
+" {1 {fts3tokenize disabled}}
+do_catchsql_test 1.2.2 {
+  CREATE VIRTUAL TABLE x1 USING fts3(col, tokenize=mytok);
+} {1 {unknown tokenizer: mytok}}
+do_catchsql_test 1.2.3 {
+  SELECT fts3_tokenizer('mytok', $blob)
+} {0 {{}}}
+do_execsql_test 1.2.4 {
+  CREATE VIRTUAL TABLE x1 USING fts3(col, tokenize=mytok);
+}
+
+# With ENABLE_FTS3_TOKENIZER set to 1:
+#
+#   * It is possible to get a pointer to a token implementation with either
+#     a bound parameter or a literal.
+#
+sqlite3_db_config db SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER 1
+set bound "simple"
+do_execsql_test 1.3.1 {
+  SELECT typeof( fts3_tokenizer('simple') );
+} {blob}
+do_execsql_test 1.3.2 {
+  SELECT typeof( fts3_tokenizer($bound) );
+} {blob}
+
+# With ENABLE_FTS3_TOKENIZER set to 1:
+#
+#   * It is not possible to create a token implementation using either
+#     a bound parameter or a literal.
+#
+set literal [db one {SELECT quote( fts3_tokenizer($bound) )}]
+set blob    [db one {SELECT fts3_tokenizer($bound) }]
+
+do_execsql_test 1.4.1 "
+  SELECT typeof( fts3_tokenizer('mytok2', $literal) );
+" {blob}
+do_execsql_test 1.4.2 {
+  CREATE VIRTUAL TABLE x2 USING fts3(col, tokenize=mytok2);
+}
+do_execsql_test 1.4.3 {
+  SELECT typeof( fts3_tokenizer('mytok3', $blob) );
+} {blob}
+do_execsql_test 1.4.4 {
+  CREATE VIRTUAL TABLE x3 USING fts3(col, tokenize=mytok3);
+}
+
+finish_test
+