Modify the implementation of "wrapper" tokenizers to make them more robust in the...

FossilOrigin-Name: 7c0001d6eb43f89144eb84d9e30f575a3feed401d685a0d1f260692e419b2df2

diff --git a/ext/fts5/fts5_main.c b/ext/fts5/fts5_main.c
index 4addc072941069e4ec4c6b617419ae639e69d0d7..5a97930ef7fd947911672086dbf90942465fee1d 100644 (file)
--- a/ext/fts5/fts5_main.c
+++ b/ext/fts5/fts5_main.c
@@ -3267,7 +3267,9 @@ static int fts5NewTokenizerModule(
 */
 typedef struct Fts5VtoVTokenizer Fts5VtoVTokenizer;
 struct Fts5VtoVTokenizer {
-  Fts5TokenizerModule *pMod;
+  int bV2Native;                  /* True if v2 native tokenizer */
+  fts5_tokenizer x1;              /* Tokenizer functions */
+  fts5_tokenizer_v2 x2;           /* V2 tokenizer functions */
   Fts5Tokenizer *pReal;
 };
 
@@ -3287,7 +3289,9 @@ static int fts5VtoVCreate(
 
   pNew = (Fts5VtoVTokenizer*)sqlite3Fts5MallocZero(&rc, sizeof(*pNew));
   if( rc==SQLITE_OK ){
-    pNew->pMod = pMod;
+    pNew->x1 = pMod->x1;
+    pNew->x2 = pMod->x2;
+    pNew->bV2Native = pMod->bV2Native;
     if( pMod->bV2Native ){
       rc = pMod->x2.xCreate(pMod->pUserData, azArg, nArg, &pNew->pReal);
     }else{
@@ -3309,11 +3313,10 @@ static int fts5VtoVCreate(
 static void fts5VtoVDelete(Fts5Tokenizer *pTok){
   Fts5VtoVTokenizer *p = (Fts5VtoVTokenizer*)pTok;
   if( p ){
-    Fts5TokenizerModule *pMod = p->pMod;
-    if( pMod->bV2Native ){
-      pMod->x2.xDelete(p->pReal);
+    if( p->bV2Native ){
+      p->x2.xDelete(p->pReal);
     }else{
-      pMod->x1.xDelete(p->pReal);
+      p->x1.xDelete(p->pReal);
     }
     sqlite3_free(p);
   }
@@ -3331,9 +3334,8 @@ static int fts5V1toV2Tokenize(
   int (*xToken)(void*, int, const char*, int, int, int)
 ){
   Fts5VtoVTokenizer *p = (Fts5VtoVTokenizer*)pTok;
-  Fts5TokenizerModule *pMod = p->pMod;
-  assert( pMod->bV2Native );
-  return pMod->x2.xTokenize(p->pReal, pCtx, flags, pText, nText, 0, 0, xToken);
+  assert( p->bV2Native );
+  return p->x2.xTokenize(p->pReal, pCtx, flags, pText, nText, 0, 0, xToken);
 }
 
 /*
@@ -3348,10 +3350,9 @@ static int fts5V2toV1Tokenize(
   int (*xToken)(void*, int, const char*, int, int, int)
 ){
   Fts5VtoVTokenizer *p = (Fts5VtoVTokenizer*)pTok;
-  Fts5TokenizerModule *pMod = p->pMod;
-  assert( pMod->bV2Native==0 );
+  assert( p->bV2Native==0 );
   UNUSED_PARAM2(pLocale,nLocale);
-  return pMod->x1.xTokenize(p->pReal, pCtx, flags, pText, nText, xToken);
+  return p->x1.xTokenize(p->pReal, pCtx, flags, pText, nText, xToken);
 }
 
 /*

diff --git a/manifest b/manifest
index e776cdf849e27d43213bda22d979b7eb556974b2..95f0b78d9ebb44783b6d56f50313f8df98e0b26d 100644 (file)
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Improvements\sto\stestrunner.tcl:\s\sShow\selapse\stime\sin\sMM:SS\sor\sHH:MM:SS.\nKeep\strack\sof\sthe\stotal\snumber\sof\stests\sand\sthe\stotal\snumber\sof\serrors\nand\sreport\sthose\svalues\son\sa\ssummary\sline\sat\sthe\send.
-D 2024-08-27T19:17:29.271
+C Modify\sthe\simplementation\sof\s"wrapper"\stokenizers\sto\smake\sthem\smore\srobust\sin\sthe\scase\swhere\sthe\sdatabase\sconnection\sis\sclosed\sbefore\sthe\stokenizers\sare\sdeleted.
+D 2024-08-27T19:27:40.626
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -100,7 +100,7 @@ F ext/fts5/fts5_config.c 353d2a0d12678cae6ab5b9ce54aed8dac0825667b69248b5a4ed81c
 F ext/fts5/fts5_expr.c 9a56f53700d1860f0ee2f373c2b9074eaf2a7aa0637d0e27a6476de26a3fee33
 F ext/fts5/fts5_hash.c adda4272be401566a6e0ba1acbe70ee5cb97fce944bc2e04dc707152a0ec91b1
 F ext/fts5/fts5_index.c eb9a0dda3bc6ef969a6be8d2746af56856e67251810ddba08622b45be8477abe
-F ext/fts5/fts5_main.c 3d8b778f65fe5be218f6a8e4019048f092c0c09621b035ce3e8804093036b578
+F ext/fts5/fts5_main.c 0dbe241c50b62e24a229a176d66d580e359b3c32cb0970c39bb6dd6b86e125d5
 F ext/fts5/fts5_storage.c 9a9b880be12901f1962ae2a5a7e1b74348b3099a1e728764e419f75d98e3e612
 F ext/fts5/fts5_tcl.c 4db9258a7882c5eac0da4433042132aaf15b87dd1e1636c7a6ca203abd2c8bfe
 F ext/fts5/fts5_test_mi.c 08c11ec968148d4cb4119d96d819f8c1f329812c568bac3684f5464be177d3ee
@@ -2211,8 +2211,8 @@ F vsixtest/vsixtest.tcl 6195aba1f12a5e10efc2b8c0009532167be5e301abe5b31385638080
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P ca21c942c30a3dbff0e7d118e105b847d80b5388c74d19c2eeea71581f8f40b8
-R e9d7eae7ed3949bc9ba3747c7b003f26
-U drh
-Z 1e89338cdc3d7df9980ff860413d15b9
+P 40b232924c973f8d94605946fff75acbe120d90634a2eaf38bd31649e9f1390b
+R 37747ff8c02c47392327f3909e12ba2c
+U dan
+Z 559c141ac65916995897dbbf28b9ea88
 # Remove this line to create a well-formed Fossil manifest.

diff --git a/manifest.uuid b/manifest.uuid
index 2d3f302a52ed6914d63ab3f9cb9f51704d1586d1..b2a520b2f89a082e80cc72513215e5858f2fec84 100644 (file)
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-40b232924c973f8d94605946fff75acbe120d90634a2eaf38bd31649e9f1390b
+7c0001d6eb43f89144eb84d9e30f575a3feed401d685a0d1f260692e419b2df2