Add documentation for new fts5 auxiliary function APIs.

FossilOrigin-Name: 9be8969edd49e3da96fb8ac2279aff6fe2e215d6ac55162b4734aca1b6316580

diff --git a/ext/fts5/fts5.h b/ext/fts5/fts5.h
index 5a2008882fe6279ea11cec57f0f6adfdf3bbb0cb..9feedbba19dd28267bf13894b4f0fc0f76d8893e 100644 (file)
--- a/ext/fts5/fts5.h
+++ b/ext/fts5/fts5.h
@@ -261,6 +261,27 @@ struct Fts5PhraseIter {
 **
 ** xPhraseNextColumn()
 **   See xPhraseFirstColumn above.
+**
+** xQueryToken(pFts5, iPhrase, iToken, ppToken, pnToken)
+**   This is used to access token iToken of phrase iPhrase of the current
+**   query. Before returning, output parameter *ppToken is set to point
+**   to a buffer containing the requested token, and *pnToken to the
+**   size of this buffer in bytes.
+**
+**   The output text is not a copy of the query text that specified the
+**   token. It is the output of the tokenizer module. For tokendata=1
+**   tables, this includes any embedded 0x00 and trailing data.
+**
+** xInstToken(pFts5, iIdx, iToken, ppToken, pnToken)
+**   This is used to access token iToken of phrase hit iIdx within the
+**   current row.
+**
+**   The output text is not a copy of the document text that was tokenized.
+**   It is the output of the tokenizer module. For tokendata=1 tables, this 
+**   includes any embedded 0x00 and trailing data.
+**
+**   This API can be quite slow if used with an FTS5 table created with the
+**   "detail=none" or "detail=column" option. 
 */
 struct Fts5ExtensionApi {
   int iVersion;                   /* Currently always set to 3 */
@@ -300,9 +321,11 @@ struct Fts5ExtensionApi {
   void (*xPhraseNextColumn)(Fts5Context*, Fts5PhraseIter*, int *piCol);
 
   /* Below this point are iVersion>=3 only */
-  int (*xQueryToken)(Fts5Context*, int iPhrase, int iToken, const char**, int*);
+  int (*xQueryToken)(Fts5Context*, 
+      int iPhrase, int iToken, 
+      const char **ppToken, int *pnToken
+  );
   int (*xInstToken)(Fts5Context*, int iIdx, int iToken, const char**, int*);
-  int (*xPhraseToken)(Fts5Context*, Fts5PhraseIter*, int, const char**, int*);
 };
 
 /* 

diff --git a/ext/fts5/fts5_main.c b/ext/fts5/fts5_main.c
index 5ef80719aed07b1bd41f5383f66efa9783293a5b..9f19b24b88fce32946da0a5de29e9363a59ea083 100644 (file)
--- a/ext/fts5/fts5_main.c
+++ b/ext/fts5/fts5_main.c
@@ -2360,19 +2360,6 @@ static int fts5ApiInstToken(
   return rc;
 }
 
-/*
-** xPhraseToken() API implemenetation.
-*/
-static int fts5ApiPhraseToken(
-  Fts5Context *pCtx, 
-  Fts5PhraseIter *pIter, 
-  int iToken,
-  const char **ppOut, 
-  int *pnOut
-){
-  return SQLITE_OK;
-}
-
 
 static int fts5ApiQueryPhrase(Fts5Context*, int, void*, 
     int(*)(const Fts5ExtensionApi*, Fts5Context*, void*)
@@ -2400,8 +2387,7 @@ static const Fts5ExtensionApi sFts5Api = {
   fts5ApiPhraseFirstColumn,
   fts5ApiPhraseNextColumn,
   fts5ApiQueryToken,
-  fts5ApiInstToken,
-  fts5ApiPhraseToken
+  fts5ApiInstToken
 };
 
 /*

diff --git a/manifest b/manifest
index dfaf7bb154df413ae43cdceda2eedd577cfd2224..8adb17e9817679523df39584c8966574f274f21c 100644 (file)
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Defer\sbuilding\sxInstToken()\shash-table\suntil\sit\sis\sto\sbe\sused.
-D 2023-11-22T20:02:55.862
+C Add\sdocumentation\sfor\snew\sfts5\sauxiliary\sfunction\sAPIs.
+D 2023-11-22T21:15:05.459
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -87,7 +87,7 @@ F ext/fts3/unicode/UnicodeData.txt cd07314edb62d49fde34debdaf92fa2aa69011e7
 F ext/fts3/unicode/mkunicode.tcl d5aebf022fa4577ee8cdf27468f0d847879993959101f6dbd6348ef0cfc324a7
 F ext/fts3/unicode/parseunicode.tcl a981bd6466d12dd17967515801c3ff23f74a281be1a03cf1e6f52a6959fc77eb
 F ext/fts5/extract_api_docs.tcl a36e54ec777172ddd3f9a88daf593b00848368e0
-F ext/fts5/fts5.h e27cdb10e38d87cb041dcb56cef97addf7d902aeab07e84e7102f5fc65d3357c
+F ext/fts5/fts5.h 5e5630fc81e212f658afaa5b2650dac939d2729d0723aef1eeaff908f1725648
 F ext/fts5/fts5Int.h 782151060d176be22861f57bf38e087a82cfb0dfc4b2fa6f9ccbc2641b6d01e3
 F ext/fts5/fts5_aux.c ee770eec0af8646db9e18fc01a0dad7345b5f5e8cbba236704cfae2d777022ad
 F ext/fts5/fts5_buffer.c 3001fbabb585d6de52947b44b455235072b741038391f830d6b729225eeaf6a5
@@ -95,7 +95,7 @@ F ext/fts5/fts5_config.c 8072a207034b51ae9b7694121d1b5715c794e94b275e088f70ae532
 F ext/fts5/fts5_expr.c 5d557c7ebefaeac5a5111cc47d4fee8a2fc6bc15245d5c99eebf53dd04bf794e
 F ext/fts5/fts5_hash.c adda4272be401566a6e0ba1acbe70ee5cb97fce944bc2e04dc707152a0ec91b1
 F ext/fts5/fts5_index.c 710b022dcdf152eb7bbbc3f83eb662e1e67c25e0643416096ed070b10d7829fb
-F ext/fts5/fts5_main.c f151eb2c6d27418d907c88cd623ad4508bdcf518a79d504e850270754c228b74
+F ext/fts5/fts5_main.c 55b53085dbd1693b5735463198a8d124dfbc27f08311c839637b44b8254ef7cb
 F ext/fts5/fts5_storage.c 5d10b9bdcce5b90656cad13c7d12ad4148677d4b9e3fca0481fca56d6601426d
 F ext/fts5/fts5_tcl.c cf0fd0dbe64ec272491b749e0d594f563cda03336aeb60900129e6d18b0aefb8
 F ext/fts5/fts5_test_mi.c 08c11ec968148d4cb4119d96d819f8c1f329812c568bac3684f5464be177d3ee
@@ -2145,8 +2145,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 37b271c19d772bd06524db816ded03377b426efed7a7783c8a96f6fb156ecd86
-R 0d85011f1dd994f228f63f6f67d53fdc
+P 9b005085ff4a53cda0a1dff0c836630d6d3b95b9c40658ffd2a886f3e1b37faa
+R 2f037553902ade27b74f0aa5f4f0c2dc
 U dan
-Z f3ab4e11005318d9a7fa0f164d5ead9f
+Z 56eb427eba85d076d9cd44ffbac445d7
 # Remove this line to create a well-formed Fossil manifest.

diff --git a/manifest.uuid b/manifest.uuid
index 81d2b689454029e07b8d7b13b2f72740d80e806e..eca2a6957f7ce02977f199c234b44542007879b2 100644 (file)
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-9b005085ff4a53cda0a1dff0c836630d6d3b95b9c40658ffd2a886f3e1b37faa
\ No newline at end of file
+9be8969edd49e3da96fb8ac2279aff6fe2e215d6ac55162b4734aca1b6316580
\ No newline at end of file