Add test case for fts5 trigram tokenizer.

FossilOrigin-Name: ba358d265b7ee360d62b5219faaa1010ea90dac4e20cc7adc3ebd46161a65f94

diff --git a/ext/fts5/fts5_tcl.c b/ext/fts5/fts5_tcl.c
index a8ab44096bc44b835aa8298d2133fc5ff963ddc7..df4b2daa4198a8523fd8ca5c82299ef435006b04 100644 (file)
--- a/ext/fts5/fts5_tcl.c
+++ b/ext/fts5/fts5_tcl.c
@@ -1626,6 +1626,64 @@ static int SQLITE_TCLAPI f5tDropCorruptTable(
   return TCL_OK;
 }
 
+/*
+** Free a buffer returned to SQLite by the str() function.
+*/
+void f5tFree(void *p){
+  char *x = (char *)p;
+  free(&x[-8]);
+}
+
+/*
+** Implementation of str().
+*/
+void f5tStrFunc(sqlite3_context *pCtx, int nArg, sqlite3_value **apArg){
+  const char *zText = 0;
+  assert( nArg==1 );
+
+  zText = sqlite3_value_text(apArg[0]);
+  if( zText ){
+    int nText = sqlite3Strlen30(zText);
+    char *zCopy = (char*)malloc(nText+8);
+    if( zCopy==0 ){
+      sqlite3_result_error_nomem(pCtx);
+    }else{
+      zCopy += 8;
+      memcpy(zCopy, zText, nText);
+      sqlite3_result_text(pCtx, zCopy, nText, f5tFree);
+    }
+  }
+}
+
+/*
+**      sqlite3_fts5_register_str DB
+**
+** Register the str() function with database handle DB. str() interprets
+** its only argument as text and returns a copy of the value in a
+** non-nul-terminated buffer.
+*/
+static int SQLITE_TCLAPI f5tRegisterStr(
+  void * clientData,
+  Tcl_Interp *interp,
+  int objc,
+  Tcl_Obj *CONST objv[]
+){
+  sqlite3 *db = 0;
+  int rc;
+
+  if( objc!=2 ){
+    Tcl_WrongNumArgs(interp, 1, objv, "DB");
+    return TCL_ERROR;
+  }
+  if( f5tDbPointer(interp, objv[1], &db) ){
+    return TCL_ERROR;
+  }
+
+  sqlite3_create_function(db, "str", 1, SQLITE_UTF8, 0, f5tStrFunc, 0, 0);
+
+  return TCL_OK;
+}
+
 /*
 ** Entry point.
 */
@@ -1645,7 +1703,8 @@ int Fts5tcl_Init(Tcl_Interp *interp){
     { "sqlite3_fts5_register_matchinfo", f5tRegisterMatchinfo, 0 },
     { "sqlite3_fts5_register_fts5tokenize", f5tRegisterTok, 0 },
     { "sqlite3_fts5_register_origintext",f5tRegisterOriginText, 0 },
-    { "sqlite3_fts5_drop_corrupt_table", f5tDropCorruptTable, 0 }
+    { "sqlite3_fts5_drop_corrupt_table", f5tDropCorruptTable, 0 },
+    { "sqlite3_fts5_register_str",       f5tRegisterStr, 0 }
   };
   int i;
   F5tTokenizerContext *pContext;

diff --git a/ext/fts5/test/fts5trigram.test b/ext/fts5/test/fts5trigram.test
index 3742c647f063d54e29ba7bf2682d68063a55ed4c..5048f8beea76fbe0a2d6ad874745d36b9b68d0ed 100644 (file)
--- a/ext/fts5/test/fts5trigram.test
+++ b/ext/fts5/test/fts5trigram.test
@@ -342,6 +342,13 @@ do_test 10.3 {
   }
 } {}
 
-
+do_execsql_test 11.0 {
+  CREATE VIRTUAL TABLE t4 USING fts5(y, tokenize=trigram);
+}
+sqlite3_fts5_register_str db
+do_execsql_test 11.1 {
+  INSERT INTO t4 VALUES( str('') );
+}
 
 finish_test
+

diff --git a/manifest b/manifest
index 2cb0cec7d6b1cd53a5a8e8fc3593fb28d715f9b6..169e0e7d1cbb8393cadd4e027a3c0b5ba7493297 100644 (file)
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C In\sMakefile.msc,\sbuild\sjimsh0.exe\slocally,\snot\sin\sthe\ssource\sdirectory.
-D 2024-10-26T17:47:19.419
+C Add\stest\scase\sfor\sfts5\strigram\stokenizer.
+D 2024-10-26T18:09:13.006
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md b6e6c1baf38e4339bd3f1e0e5e5bfd0a9a93d133360691b2785c2d4b2f2dcec2
@@ -120,7 +120,7 @@ F ext/fts5/fts5_hash.c adda4272be401566a6e0ba1acbe70ee5cb97fce944bc2e04dc707152a
 F ext/fts5/fts5_index.c 368a968570ce12ba40223e284a588d9f93ee23a0133727f0df1fcd64086b1fb6
 F ext/fts5/fts5_main.c 50eb059e51d730e8e0c77df4e568b018079e112a755c094488b0d5b1aa06afbb
 F ext/fts5/fts5_storage.c 337b05e4c66fc822d031e264d65bde807ec2fab08665ca2cc8aaf9c5fa06792c
-F ext/fts5/fts5_tcl.c 4db9258a7882c5eac0da4433042132aaf15b87dd1e1636c7a6ca203abd2c8bfe
+F ext/fts5/fts5_tcl.c 9b390c318e36c0dc53af14e20198f55aa3ed46a39350ed7f46bc034e422fe778
 F ext/fts5/fts5_test_mi.c 08c11ec968148d4cb4119d96d819f8c1f329812c568bac3684f5464be177d3ee
 F ext/fts5/fts5_test_tok.c 3cb0a9b508b30d17ef025ccddd26ae3dc8ddffbe76c057616e59a9aa85d36f3b
 F ext/fts5/fts5_tokenize.c 033e2e43b8e852c0ef6cecc611266d61e2346e52ec7dcfb76a428fe56a07efa9
@@ -255,7 +255,7 @@ F ext/fts5/test/fts5tok2.test dcacb32d4a2a3f0dd3215d4a3987f78ae4be21a2
 F ext/fts5/test/fts5tokenizer.test 7937cec672b148223fff8746d21d3e7ed0965fd7caf35ccdc888a005bb452f98
 F ext/fts5/test/fts5tokenizer2.test ddb8b10fbe4b84b2a75812671f127774c1d2e3e2bf82d2e0e4f0bb1cd8a2b2d6
 F ext/fts5/test/fts5tokenizer3.test eea778f7bb7024c3e904e28915f9d53286141671b138722148be22a9c758bdc3
-F ext/fts5/test/fts5trigram.test fb9ee982edd76280ce979905a2251081cd04ae4c470248bd5d391b2d096430ab
+F ext/fts5/test/fts5trigram.test 9927c9e9b35116ea00748c8e41d9cbc2b95a6c90845cd82a59c11fedfd16404a
 F ext/fts5/test/fts5trigram2.test 6fde9de7f63a6b4aa18dc731be56dbd6be4e755c9b13dcd55479e200d1df0e61
 F ext/fts5/test/fts5ubsan.test 9a2dcf399dc8d0e0de661f0d93884d1d27e5b7f0693cfceb97dd24d818df5dd2
 F ext/fts5/test/fts5umlaut.test a42fe2fe6387c40c49ab27ccbd070e1ae38e07f38d05926482cc0bccac9ad602
@@ -2237,8 +2237,8 @@ F vsixtest/vsixtest.tcl 6195aba1f12a5e10efc2b8c0009532167be5e301abe5b31385638080
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 47e50fa84dacf83c2aca62140413c7eeba934e57289a6f6e6fff3ce24448d90a
-R faad5bee19c3ac86eb55ce27078c47df
-U drh
-Z 4a825c1d043ad68945d1bd0013a800b9
+P 1b9eb4564bc38cbc6a51ed1c4508f1ba45459630cfda8765c243c9aa0fc7d763
+R 8ff21923791c350decdcbf04d2f23594
+U dan
+Z 3e4e525d7b1d9780dcf1603f839dea72
 # Remove this line to create a well-formed Fossil manifest.

diff --git a/manifest.uuid b/manifest.uuid
index dce88b5b27f2b0030d163ebb54ae3621ead4f27e..3927059f42247a156fc8911cefa9b9369e8b96a2 100644 (file)
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-1b9eb4564bc38cbc6a51ed1c4508f1ba45459630cfda8765c243c9aa0fc7d763
+ba358d265b7ee360d62b5219faaa1010ea90dac4e20cc7adc3ebd46161a65f94