From: stephan Date: Thu, 3 Aug 2023 07:20:10 +0000 (+0000) Subject: Initial work on exposing the FTS5 APIs to wasm, per [forum:28402061cb|request in... X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=dd127837302293623398b87b6c7631ff815ed504;p=thirdparty%2Fsqlite.git Initial work on exposing the FTS5 APIs to wasm, per [forum:28402061cb|request in the forum]. This builds and loads the structs into JS but is completely untested. FossilOrigin-Name: 52c8b73ae3095b7e9f4d5440169c818677e484d85195036a0d82db0d04a428e5 --- diff --git a/ext/wasm/api/sqlite3-api-glue.js b/ext/wasm/api/sqlite3-api-glue.js index 572efeed5c..fcfccb553a 100644 --- a/ext/wasm/api/sqlite3-api-glue.js +++ b/ext/wasm/api/sqlite3-api-glue.js @@ -823,6 +823,9 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ 'version' ]; if(wasm.bigIntEnabled){ defineGroups.push('serialize', 'session', 'vtab'); + if(!!wasm.ctype.fts5){ + defineGroups.push('fts5'); + } } for(const t of defineGroups){ for(const e of Object.entries(wasm.ctype[t])){ diff --git a/ext/wasm/api/sqlite3-wasm.c b/ext/wasm/api/sqlite3-wasm.c index 94b16d7500..e7e7fedb3b 100644 --- a/ext/wasm/api/sqlite3-wasm.c +++ b/ext/wasm/api/sqlite3-wasm.c @@ -96,8 +96,8 @@ #ifndef SQLITE_ENABLE_EXPLAIN_COMMENTS # define SQLITE_ENABLE_EXPLAIN_COMMENTS 1 #endif -#ifndef SQLITE_ENABLE_FTS4 -# define SQLITE_ENABLE_FTS4 1 +#ifndef SQLITE_ENABLE_FTS5 +# define SQLITE_ENABLE_FTS5 1 #endif #ifndef SQLITE_ENABLE_MATH_FUNCTIONS # define SQLITE_ENABLE_MATH_FUNCTIONS 1 @@ -402,7 +402,10 @@ void sqlite3_wasm_test_struct(WasmTestStruct * s){ */ SQLITE_WASM_EXPORT const char * sqlite3_wasm_enum_json(void){ - static char aBuffer[1024 * 20] = {0} /* where the JSON goes */; + static char aBuffer[1024 * 22] = {0} + /* where the JSON goes. If this buffer is not large enough, this + function will assert (in debug builds) and return 0. When it does + so, this value needs to be increased. */; int n = 0, nChildren = 0, nStruct = 0 /* output counters for figuring out where commas go */; char * zPos = &aBuffer[1] /* skip first byte for now to help protect @@ -651,6 +654,16 @@ const char * sqlite3_wasm_enum_json(void){ DefInt(SQLITE_LOCK_EXCLUSIVE); } _DefGroup; +#ifdef SQLITE_ENABLE_FTS5 + DefGroup(fts5) { + DefInt(FTS5_TOKENIZE_QUERY); + DefInt(FTS5_TOKENIZE_PREFIX); + DefInt(FTS5_TOKENIZE_DOCUMENT); + DefInt(FTS5_TOKENIZE_AUX); + DefInt(FTS5_TOKEN_COLOCATED); + } _DefGroup; +#endif + DefGroup(ioCap) { DefInt(SQLITE_IOCAP_ATOMIC); DefInt(SQLITE_IOCAP_ATOMIC512); @@ -876,7 +889,7 @@ const char * sqlite3_wasm_enum_json(void){ DefInt(SQLITE_STMTSTATUS_FILTER_HIT); DefInt(SQLITE_STMTSTATUS_MEMUSED); } _DefGroup; - + DefGroup(syncFlags) { DefInt(SQLITE_SYNC_NORMAL); DefInt(SQLITE_SYNC_FULL); @@ -1094,7 +1107,7 @@ const char * sqlite3_wasm_enum_json(void){ M(xShadowName, "i(s)"); } _StructBinder; #undef CurrentStruct - + /** ** Workaround: in order to map the various inner structs from ** sqlite3_index_info, we have to uplift those into constructs we @@ -1171,6 +1184,87 @@ const char * sqlite3_wasm_enum_json(void){ } _StructBinder; #undef CurrentStruct +#ifdef SQLITE_ENABLE_FTS5 +#define CurrentStruct Fts5PhraseIter + StructBinder { + M(a, "p"); + M(b, "p"); + } _StructBinder; +#undef CurrentStruct + +#define CurrentStruct Fts5ExtensionApi + StructBinder { + M(iVersion, "i"); + M(xUserData, "p(p)");// void *(*)(Fts5Context*); + M(xColumnCount, "i(p)");// int (*)(Fts5Context*); + M(xRowCount, "i(pp)"); + //^^^ int (*)(Fts5Context*, sqlite3_int64 *pnRow); + M(xColumnTotalSize, "i(pip)"); + //^^^ int (*)(Fts5Context*, int iCol, sqlite3_int64 *pnToken); + M(xTokenize, "i(ppipp)"); + //^^^ int (*)(Fts5Context*, + // const char *pText, int nText, /* Text to tokenize */ + // void *pCtx, /* Context passed to xToken() */ + // int (*xToken)(void*, int, const char*, int, int, int) /* Callback */ + //); + M(xPhraseCount, "i(p)"); // int (*)(Fts5Context*); + M(xPhraseSize, "i(pi)"); // int (*)(Fts5Context*, int iPhrase); + M(xInstCount, "i(pp)"); // int (*)(Fts5Context*, int *pnInst); + M(xInst, "i(pippp)"); + //^^^ int (*)(Fts5Context*, int iIdx, int *piPhrase, int *piCol, int *piOff); + M(xRowid, "j(p)"); // sqlite3_int64 (*)(Fts5Context*); + M(xColumnText, "i(pipp)"); + //^^^ int (*)(Fts5Context*, int iCol, const char **pz, int *pn); + M(xColumnSize, "i(pip)"); + //^^^ int (*)(Fts5Context*, int iCol, int *pnToken); + M(xQueryPhrase, "i(pipp)"); + //^^^ int (*xQueryPhrase)(Fts5Context*, int iPhrase, void *pUserData, + // int(*)(const Fts5ExtensionApi*,Fts5Context*,void*) + // ); + M(xSetAuxdata, "i(ppp)"); + //^^^ int (*)(Fts5Context*, void *pAux, void(*xDelete)(void*)); + M(xGetAuxdata, "p(pi)"); // void *(*)(Fts5Context*, int bClear); + M(xPhraseFirst, "i(pippp)"); + //^^^ int (*)(Fts5Context*, int iPhrase, Fts5PhraseIter*, int*, int*); + M(xPhraseNext, "v(pppp)"); + //^^^ void (*)(Fts5Context*, Fts5PhraseIter*, int *piCol, int *piOff); + M(xPhraseFirstColumn, "i(pipp)"); + //^^^ int (*)(Fts5Context*, int iPhrase, Fts5PhraseIter*, int*); + M(xPhraseNextColumn, "v(ppp)"); + //^^^ void (*)(Fts5Context*, Fts5PhraseIter*, int *piCol); + } _StructBinder; +#undef CurrentStruct + +#define CurrentStruct fts5_api + StructBinder { + M(iVersion, "i");/* Currently always 2 */ + M(xCreateTokenizer, "i(ppppp)"); + //^^^ int (*)( + // fts5_api *pApi, + // const char *zName, + // void *pContext, + // fts5_tokenizer *pTokenizer, + // void (*xDestroy)(void*) + // ); + M(xFindTokenizer, "i(pppp)"); + //^^^ int (*)( + // fts5_api *pApi, + // const char *zName, + // void **ppContext, + // fts5_tokenizer *pTokenizer + // ); + M(xCreateFunction, "i(ppppp)"); + //^^^ int (*)( + // fts5_api *pApi, + // const char *zName, + // void *pContext, + // fts5_extension_function xFunction, + // void (*xDestroy)(void*) + // ); + } _StructBinder; +#undef CurrentStruct +#endif /* SQLITE_ENABLE_FTS5 */ + #if SQLITE_WASM_TESTS #define CurrentStruct WasmTestStruct StructBinder { diff --git a/manifest b/manifest index bd8750b7dc..0d4d289c90 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Performance\soptimization\sfor\sJSON\srendering\slogic. -D 2023-08-02T16:06:02.660 +C Initial\swork\son\sexposing\sthe\sFTS5\sAPIs\sto\swasm,\sper\s[forum:28402061cb|request\sin\sthe\sforum].\sThis\sbuilds\sand\sloads\sthe\sstructs\sinto\sJS\sbut\sis\scompletely\suntested. +D 2023-08-03T07:20:10.366 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -500,7 +500,7 @@ F ext/wasm/api/post-js-footer.js cd0a8ec768501d9bd45d325ab0442037fb0e33d1f3b4f08 F ext/wasm/api/post-js-header.js 47b6b281f39ad59fa6e8b658308cd98ea292c286a68407b35ff3ed9cfd281a62 F ext/wasm/api/pre-js.c-pp.js ad906703f7429590f2fbf5e6498513bf727a1a4f0ebfa057afb08161d7511219 F ext/wasm/api/sqlite3-api-cleanup.js d235ad237df6954145404305040991c72ef8b1881715d2a650dda7b3c2576d0e -F ext/wasm/api/sqlite3-api-glue.js cc6b0bb093bdb6279d4af259200b7b9e150e3796a8a3a4cd09a4928c43d25e56 +F ext/wasm/api/sqlite3-api-glue.js 8d087ebb00d1db2afa393b9ea7bfe1ec1fd7387c566d9bb160119e66dd8c2d04 F ext/wasm/api/sqlite3-api-oo1.js 9678dc4d9a5d39632b6ffe6ea94a023119260815bf32f265bf5f6c36c9516db8 F ext/wasm/api/sqlite3-api-prologue.js cbd7d6ba185f3a844a8b0020e954b49bbc2ca78b305d117bec2ceca21431795a F ext/wasm/api/sqlite3-api-worker1.js 9f32af64df1a031071912eea7a201557fe39b1738645c0134562bb84e88e2fec @@ -509,7 +509,7 @@ F ext/wasm/api/sqlite3-opfs-async-proxy.js 8cf8a897726f14071fae6be6648125162b256 F ext/wasm/api/sqlite3-v-helper.js 7daa0eab0a513a25b05e9abae7b5beaaa39209b3ed12f86aeae9ef8d2719ed25 F ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js 54cee22aacadb9dfaea438d72ac0882249d028c37903208d48c52871290ceff7 F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js e7a690e0e78ff4d563f2eca468f91db69f001ff4b79c6d2304cbb6f62dca437d -F ext/wasm/api/sqlite3-wasm.c 8867f1d41c112fb4a2cfe22ff224eccaf309fcdea266cee0ec554f85db72ef0f +F ext/wasm/api/sqlite3-wasm.c 2ef539a238a81d339ff0cec9dc1d8dc62402f3a9a1e7b72ebd130737a1590d17 F ext/wasm/api/sqlite3-worker1-promiser.c-pp.js bc06df0d599e625bde6a10a394e326dc68da9ff07fa5404354580f81566e591f F ext/wasm/api/sqlite3-worker1.c-pp.js da509469755035e919c015deea41b4514b5e84c12a1332e6cc8d42cb2cc1fb75 F ext/wasm/batch-runner.html 4deeed44fe41496dc6898d9fb17938ea3291f40f4bfb977e29d0cef96fbbe4c8 @@ -2049,8 +2049,11 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P c4347e4400e96f932ac12f8f22484a2ebce2a578d1b2181977954c432f117bfd -R a3168d15b44149a629121fdff1d59acd -U drh -Z 37bd268b83df81194e3e4e7be36a0655 +P ea0b9aecbaca9a8e784fd2bcb50f78cbdcf4c5cfb45a7700bb222e4cc104c644 +R 334f0a59a542a8edf0f33116c4ce928b +T *branch * wasm-fts5 +T *sym-wasm-fts5 * +T -sym-trunk * Cancelled\sby\sbranch. +U stephan +Z 7b32e66e5dffca6facb4b18e26761abd # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index ca5b942d1b..654bb314cb 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ea0b9aecbaca9a8e784fd2bcb50f78cbdcf4c5cfb45a7700bb222e4cc104c644 \ No newline at end of file +52c8b73ae3095b7e9f4d5440169c818677e484d85195036a0d82db0d04a428e5 \ No newline at end of file