From: dan Date: Wed, 31 Jul 2024 15:46:41 +0000 (+0000) Subject: Add missing documentation for new features to fts5.h. X-Git-Tag: version-3.47.0~220^2~25 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=8e3042b5657fc41ab1c01881b80754c71b17aa95;p=thirdparty%2Fsqlite.git Add missing documentation for new features to fts5.h. FossilOrigin-Name: fa0da3b28e411affd45a918d6b7faba49f744ca8b4adf3cd5ce6609bb630499e --- diff --git a/ext/fts5/fts5.h b/ext/fts5/fts5.h index ecdb944f68..2f6c387760 100644 --- a/ext/fts5/fts5.h +++ b/ext/fts5/fts5.h @@ -317,10 +317,11 @@ struct Fts5PhraseIter { ** with column iCol of the current row. Usually, there is no associated ** locale, and output parameters (*pzLocale) and (*pnLocale) are set ** to NULL and 0, respectively. However, if the fts5_locale() function -** was used to associated a locale with the value when it was inserted -** into the fts5 table, then (*pzLocale) is set to point to a buffer -** containing the name of the locale in utf-8 encoding. (*pnLocale) is -** set to the size in bytes of the buffer. +** was used to associate a locale with the value when it was inserted +** into the fts5 table, then (*pzLocale) is set to point to a nul-terminated +** buffer containing the name of the locale in utf-8 encoding. (*pnLocale) +** is set to the size in bytes of the buffer, not including the +** nul-terminator. ** ** If successful, SQLITE_OK is returned. Or, if an error occurs, an ** SQLite error code is returned. The final value of the output parameters @@ -333,7 +334,7 @@ struct Fts5PhraseIter { ** ** Parameters pLocale and nLocale may both be 0, in which case the tokenizer ** is configured to use its default locale. Otherwise, pLocale should point -** to a buffer containing the name of the locale to use encoded as utf-8. +** to a buffer containing the utf-8 encoded name of the locale to use. ** It does not have to be nul-terminated. nLocale must be passed the size ** of the text in bytes. The buffer indicated by pLocale must remain valid ** for the duration of any calls made to xTokenize() by the auxiliary @@ -388,8 +389,7 @@ struct Fts5ExtensionApi { /* Below this point are iVersion>=4 only */ int (*xColumnLocale)(Fts5Context*, int iCol, const char **pz, int *pn); - - int (*xTokenizeSetLocale)(Fts5Context*, const char *z, int n); + int (*xTokenizeSetLocale)(Fts5Context*, const char *p, int n); }; /* @@ -401,16 +401,17 @@ struct Fts5ExtensionApi { ** ** Applications may also register custom tokenizer types. A tokenizer ** is registered by providing fts5 with a populated instance of the -** following structure. All structure methods must be defined, setting -** any member of the fts5_tokenizer struct to NULL leads to undefined -** behaviour. The structure methods are expected to function as follows: +** following structure. Of the three structure methods, xCreate, xDelete and +** xTokenize must be supplied, any fo these three members of the +** fts5_tokenizer_v2 struct to NULL leads to undefined behaviour. The +** structure methods are expected to function as follows: ** ** xCreate: ** This function is used to allocate and initialize a tokenizer instance. ** A tokenizer instance is required to actually tokenize text. ** ** The first argument passed to this function is a copy of the (void*) -** pointer provided by the application when the fts5_tokenizer object +** pointer provided by the application when the fts5_tokenizer_v2 object ** was registered with FTS5 (the third argument to xCreateTokenizer()). ** The second and third arguments are an array of nul-terminated strings ** containing the tokenizer arguments, if any, specified following the @@ -481,6 +482,33 @@ struct Fts5ExtensionApi { ** may abandon the tokenization and return any error code other than ** SQLITE_OK or SQLITE_DONE. ** +** xSetLocale: +** This function is invoked by FTS5 to configure the locale to use for +** subsequent calls to xTokenize. The second argument is a pointer to +** a nul-terminated buffer containing the utf-8 encoded name of the locale +** to use. The third argument is the size of the buffer in bytes, not +** including the nul-terminator character. This function may also be +** invoked with the second and third parameters set to 0 - instructing +** the tokenizer to use its default locale. +** +** FTS5 guarantees that any buffer passed to xSetLocale() will remain +** valid until either the next call to xSetLocale() or xDelete() on the +** same tokenizer object. +** +** This function should return SQLITE_OK if successful, or an SQLite +** error code if an error occurs. If an error does occur and an error +** code is returned, execution of the current statement is abandoned +** and FTS5 returns the error code to the caller. +** +** Often, this function is not required and is never invoked. It is only +** ever invoked when processing a value that has had a locale associated +** with it using SQL function fts5_locale(). +** +** It is not necessary to supply an implementation of this method when +** registering a tokenizer. If fts5_tokenizer_v2.xSetLocale is set to NULL, +** then no attempt is made to pass locale information through to the +** tokenizer. +** ** SYNONYM SUPPORT ** ** Custom tokenizers may also support synonyms. Consider a case in which a @@ -589,8 +617,10 @@ struct Fts5ExtensionApi { ** inefficient. */ typedef struct Fts5Tokenizer Fts5Tokenizer; -typedef struct fts5_tokenizer fts5_tokenizer; -struct fts5_tokenizer { +typedef struct fts5_tokenizer_v2 fts5_tokenizer_v2; +struct fts5_tokenizer_v2 { + int iVersion; /* Currently always 2 */ + int (*xCreate)(void*, const char **azArg, int nArg, Fts5Tokenizer **ppOut); void (*xDelete)(Fts5Tokenizer*); int (*xTokenize)(Fts5Tokenizer*, @@ -606,12 +636,17 @@ struct fts5_tokenizer { int iEnd /* Byte offset of end of token within input text */ ) ); -}; -typedef struct fts5_tokenizer_v2 fts5_tokenizer_v2; -struct fts5_tokenizer_v2 { - int iVersion; /* Currently always 2 */ + int (*xSetLocale)(Fts5Tokenizer*, const char *pLocale, int nLocale); +}; +/* +** New code should use the fts5_tokenizer_v2 type to define tokenizer +** implementations. The following type is included for legacy applications +** that still use it. +*/ +typedef struct fts5_tokenizer fts5_tokenizer; +struct fts5_tokenizer { int (*xCreate)(void*, const char **azArg, int nArg, Fts5Tokenizer **ppOut); void (*xDelete)(Fts5Tokenizer*); int (*xTokenize)(Fts5Tokenizer*, @@ -627,10 +662,9 @@ struct fts5_tokenizer_v2 { int iEnd /* Byte offset of end of token within input text */ ) ); - - int (*xSetLocale)(Fts5Tokenizer*, const char *pLocale, int nLocale); }; + /* Flags that may be passed as the third argument to xTokenize() */ #define FTS5_TOKENIZE_QUERY 0x0001 #define FTS5_TOKENIZE_PREFIX 0x0002 diff --git a/manifest b/manifest index a08c7ec6c7..a18841d7c4 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\slatest\schanges\sfrom\strunk\sinto\sthis\sbranch. -D 2024-07-31T10:52:38.878 +C Add\smissing\sdocumentation\sfor\snew\sfeatures\sto\sfts5.h. +D 2024-07-31T15:46:41.319 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -92,7 +92,7 @@ F ext/fts3/unicode/UnicodeData.txt cd07314edb62d49fde34debdaf92fa2aa69011e7 F ext/fts3/unicode/mkunicode.tcl d5aebf022fa4577ee8cdf27468f0d847879993959101f6dbd6348ef0cfc324a7 F ext/fts3/unicode/parseunicode.tcl a981bd6466d12dd17967515801c3ff23f74a281be1a03cf1e6f52a6959fc77eb F ext/fts5/extract_api_docs.tcl bc3a0ca78be7d3df08e7602c00ca48021ebae40682d75eb001bfdf6e54ffb44e -F ext/fts5/fts5.h 7cd47e5dfe68d798e667caa76722374f0c909f2db05bb4d42b1ec5300d18e658 +F ext/fts5/fts5.h f5451da088d0004c4be7314e2fdb41fda16ce682ce2aa3b54b9474ebe9a013d5 F ext/fts5/fts5Int.h 833a2fe729f926ebcde47e21e495d141b99ede9a188fc577873f24bea0f0bfa2 F ext/fts5/fts5_aux.c 652f839dc0c77431295f10b08f268631560bb5630e65fd701de7a58744428a82 F ext/fts5/fts5_buffer.c 0eec58bff585f1a44ea9147eae5da2447292080ea435957f7488c70673cb6f09 @@ -604,7 +604,7 @@ F ext/wasm/SQLTester/SQLTester.run.mjs c72b7fe2072d05992f7a3d8c6a1d34e95712513ce F ext/wasm/SQLTester/index.html 3f8a016df0776be76605abf20e815ecaafbe055abac0e1fe5ea080e7846b760d F ext/wasm/SQLTester/touint8array.c 2d5ece04ec1393a6a60c4bf96385bda5e1a10ad49f3038b96460fc5e5aa7e536 F ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-auth 7ac80cc3b6a6d52e041bb295e85555ce797be78c15ef2008a64ae58815014080 -F ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-core 400213eb52a7e5ad5f448053d375cacf4dac2cf45d134f3edfe485ae4a49a183 w ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api +F ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-core 400213eb52a7e5ad5f448053d375cacf4dac2cf45d134f3edfe485ae4a49a183 F ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-preupdate d1d62a2212099f2c0782d730beb8cb84a7a52d99c15ead2cb9b1411fff5fd6b1 F ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-see fb29e62082a658f0d81102488414d422c393c4b20cc2f685b216bc566237957b F ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-session 213b6c04267cb9bd760172db011eb1650732805fb3d01f9395478a8ceec18eb0 @@ -2201,8 +2201,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P ae435aff1785d5832821c19dc88ccf6c496a7ff55be80276b31e5c9abee723db 86de4e755e37dc1cbcbd59018927aa87ff49fc15f706a36187631d8f14075c12 -R 854c24bb1cc78c16c421b7985fead9f3 +P ff64939ec8399949393f1029fa6d514892fbf2bf1498545300cc5e120b9622a5 +R f7dbcf0c98ed562d42b896f7c143383b U dan -Z 7ee036ea576e2778dfe1b42f432458d4 +Z 344ab71a9372cfa91d638c7c0c606714 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 2864ea1b96..a1e4d24ea7 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ff64939ec8399949393f1029fa6d514892fbf2bf1498545300cc5e120b9622a5 +fa0da3b28e411affd45a918d6b7faba49f744ca8b4adf3cd5ce6609bb630499e