From: stephan Date: Fri, 4 Aug 2023 08:39:21 +0000 (+0000) Subject: More work towards fts5 customzation in JS. X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=refs%2Fheads%2Fwasm-fts5;p=thirdparty%2Fsqlite.git More work towards fts5 customzation in JS. FossilOrigin-Name: ce2a65d80f7793ae8183e311fc7284b37caa78d747436f22496284dfa57e3533 --- diff --git a/ext/wasm/api/sqlite3-fts5-helper.js b/ext/wasm/api/sqlite3-fts5-helper.js index bbabdc5ba6..124aca5733 100644 --- a/ext/wasm/api/sqlite3-fts5-helper.js +++ b/ext/wasm/api/sqlite3-fts5-helper.js @@ -20,7 +20,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ return /*this build does not have FTS5*/; } const fts = sqlite3.fts5 = Object.create(null); - const xArgDb = wasm.xWrap.argAdapter('sqlite3*'); + const __xArgDb = wasm.xWrap.argAdapter('sqlite3*'); /** Move FTS-specific APIs (installed via automation) from @@ -28,13 +28,108 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ */ for(const c of [ 'Fts5ExtensionApi', 'Fts5PhraseIter', 'fts5_api', - 'fts5_api_from_db' + 'fts5_api_from_db', 'fts5_tokenizer' ]){ fts[c] = capi[c] || toss("Cannot find capi."+c); delete capi[c]; } - /* JS-to-WASM arg adapter for xCreateFunction()'s xFunction arg. */ + /** + Requires a JS Function intended to be used as an xFunction() + implementation. This function returns a proxy xFunction + wrapper which: + + - Converts all of its sqlite3_value arguments to an array + of JS values using sqlite3_values_to_js(). + + - Calls the given callback, passing it: + + (pFtsXApi, pFtsCx, pCtx, array-of-values) + + where the first 3 arguments are the first 3 pointers + in the xFunction interface. + + + The call is intended to set a result value into the db, and may + do so be either (A) explicitly returning non-undefined or (B) + using one of the sqlite3_result_XYZ() functions and returning + undefined. If the callback throws, its exception will be passed + to sqlite3_result_error_js(). + */ + fts.xFunctionProxy1 = function(callback){ + return (pFtsXApi, pFtsCx, pCtx, argc, pArgv)=>{ + try{ + capi.sqlite3_result_js(pCtx, callback( + pFtsXApi, pFtsCx, pCtx, + capi.sqlite3_values_to_js(argc, pArgv) + )); + }catch(e){ + capi.sqlite3_result_error_js(pCtx, e); + } + } + }; + + /** + Identical to xFunctionProxy1 except that the callback wrapper it + creates does _not_ perform sqlite3_value-to-JS conversion in + advance and calls the callback with: + + (pFtsXApi, pFtsCx, pCtx, array-of-ptr-to-sqlite3_value) + + It is up to the callback to use the sqlite3_value_XYZ() family of + functions to inspect or convert the values. + */ + fts.xFunctionProxy2 = function(callback){ + return (pFtsXApi, pFtsCx, pCtx, argc, pArgv)=>{ + try{ + const list = []; + let i; + for(i = 0; i < argc; ++i){ + list.push( wasm.peekPtr(pArgv + (wasm.ptrSizeof * i)) ); + } + capi.sqlite3_result_js(pCtx, callback( + pFtsXApi, pFtsCx, pCtx, list + )); + }catch(e){ + capi.sqlite3_result_error_js(pCtx, e); + } + } + }; + + /** + JS-to-WASM arg adapter for xCreateFunction()'s xFunction arg. + This binds JS impls of xFunction to WASM so that they can be + called from native code. Its context is limited to the + combination of ((fts5_api*) + functionNameCaseInsensitive), and + will replace any existing impl for subsequent invocations for the + same combination. + + The functions is creates are intended to set a result value into + the db, and may do so be either (A) explicitly returning + non-undefined or (B) using one of the sqlite3_result_XYZ() + functions and returning undefined. If the callback throws, its + exception will be passed to sqlite3_result_error_js(). + + PENDING DESIGN DECISION: this framework currently converts each + argument in its JS equivalent before passing them on to the + xFunction impl. We could, and possibly should, instead pass a JS + array of sqlite3_value pointers. The advantages would be: + + - No in-advance to-JS overhead which xFunction might not use. + + Disadvantages include: + + - xFunction would be required to call sqlite3_value_to_js(), + or one of the many sqlite3_value_XYZ() functions on their own. + This would be more cumbersome for most users. + + Regardless of which approach is chosen here, clients could + provide a function of their own which takes the _other_ approach, + install it with wasm.installFunction(), and then pass that + generated pointer to createFunction(), in which case this layer + does not proxying and passes all native-level arguments as-is to + the client-defined function. + */ const xFunctionArgAdapter = new wasm.xWrap.FuncPtrAdapter({ name: 'fts5_api::xCreateFunction(xFunction)', signature: 'v(pppip)', @@ -42,18 +137,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ return (argv[0]/*(fts5_api*)*/ + wasm.cstrToJs(argv[1]).toLowerCase()/*name*/) }, - callProxy: (callback)=>{ - return (pFtsXApi, pFtsCx, pCtx, argc, pArgv)=>{ - try{ - capi.sqlite3_result_js(pCtx, callback( - pFtsXApi, pFtsCx, pCtx, - capi.sqlite3_values_to_js(argc, pArgv) - )); - }catch(e){ - capi.sqlite3_result_error_js(pCtx, e); - } - } - } + callProxy: fts.xFunctionProxy1 }); /** Map of (sqlite3*) to fts.fts5_api. */ @@ -61,7 +145,9 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ const __fts5_api_from_db = function(pDb, createIfNeeded){ let rc = __ftsApiToStruct[pDb]; if(!rc && createIfNeeded){ - rc = new fts.fts5_api(fts.fts5_api_from_db(pDb)); + const fapi = fts.fts5_api_from_db(pDb) + || toss("Internal error - cannot get FTS5 API object for db."); + rc = new fts.fts5_api(fapi); __ftsApiToStruct[pDb] = rc; } return rc; @@ -79,7 +165,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ /** Callback to be invoked via the JS binding of sqlite3_close_v2(), - after the db has been closed, meaing that the argument to this + after the db has been closed, meaning that the argument to this function is not a valid object. We use its address only as a lookup key. */ @@ -109,7 +195,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ (shadowed). */ if(pDestroy) wasm.uninstallFunction(pDestroy); }catch(e){ - sqlite3.config.error("Error removing FTS func",name,e); + sqlite3.config.warn("Could not remove FTS func",name,e); } } }finally{ @@ -121,6 +207,12 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ } }); + const __affirmDbArg = (arg)=>{ + arg = __xArgDb(arg); + if(!arg || !wasm.isPtr(arg)) toss("Invalid db argument."); + return arg; + }; + /** Convenience wrapper to fts5_api::xCreateFunction. @@ -132,38 +224,37 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ - name: name (JS string) of the function - xFunction either a Function or a pointer to a WASM function. In - the former case a WASM-bound wrapper for the function gets - installed for the life of the given db handle. The function - must return void and accept args in the form - (ptrToFts5ExtensionApi, ptrToFts5Context, ptrToSqlite3Context, - array-of-JS-values). + the former case a WASM-bound wrapper, behaving as documented + for fts5.xFunctionProxy1(), gets installed for the life of the + given db handle. In the latter case the function is + passed-through as-is, with no argument conversion or lifetime + tracking. In the former case the function is called as + documented for xFunctionProxy1() and in the latter it must + return void and is called with args (ptrToFts5ExtensionApi, + ptrToFts5Context, ptrToSqlite3Context, int argc, + C-array-of-sqlite3_value-pointers). - xDestroy optional Function or pointer to WASM function to call when the binding is destroyed (when the db handle is closed). The function will, in this context, always be passed 0 as its only argument. A passed-in function must, however, - have one argument so that type signature checks will pass. + have one parameter so that type signature checks will pass. + It must return void and must not throw. The 2nd and subsequent aruguments may optionally be packed into a single Object with like-named properties. - The callback functions may optionally be provided in the form of - a single object with xFunction and xDestroy properties. - This function throws on error, of which there are many potential candidates. It returns `undefined`. */ fts.createFunction = function(db, name, xFunction, xDestroy = 0){ - db = xArgDb(db); - if(!wasm.isPtr(db)) toss("Invalid db argument."); + db = __affirmDbArg(db); if( 2 === arguments.length && 'string' !== typeof name){ xDestroy = name.xDestroy || null; xFunction = name.xFunction || null; name = name.name; } if( !name || 'string' !== typeof name ) toss("Invalid name argument."); - const fapi = fts.fts5_api_from_db(db) - || toss("Internal error - cannot get FTS5 API object for db."); const sfapi = __fts5_api_from_db(db, true); let pDestroy = 0; try{ @@ -181,7 +272,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ '*', 'string', '*', xFunctionArgAdapter, '*' ]) ); - const rc = xcf(fapi, name, 0, xFunction || 0, pDestroy || xDestroy || 0 ); + const rc = xcf(sfapi.pointer, name, 0, xFunction || 0, pDestroy || xDestroy || 0 ); if(rc) toss(rc,"FTS5::xCreateFunction() failed."); __addCleanupForFunc(sfapi, name, pDestroy); }catch(e){ @@ -191,4 +282,66 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ } }; + /** + ! UNTESTED + + Convenience wrapper for fts5_api::xCreateTokenizer(). + + - db = the db to install the tokenizer into. + + - name = the JS string name of the tokenizer. + + - pTokenizer = the tokenizer instance, which must be a + fts5.fts5_tokenizer instance or a valid WASM pointer to one. + + - xDestroy = as documented for createFunction(). + + The C layer makes a bitwise copy of the tokenizer, so any + changes made to it after installation will have no effect. + + Throws on error. + */ + const createTokenizer = function(db, name, pTokenizer, xDestroy = 0){ + db = __affirmDbArg(db); + if( 2 === arguments.length && 'string' !== typeof name){ + pTokenizer = name.pTokenizer; + xDestroy = name.xDestroy || null; + name = name.name; + } + if( !name || 'string' !== typeof name ) toss("Invalid name argument."); + if(pTokenizer instanceof fts.fts5_tokenizer){ + pTokenizer = pTokenizer.pointer; + } + if(!pTokenizer || !wasm.isPtr(pTokenizer)){ + toss("Invalid pTokenizer argument - must be a valid fts5.fts5_tokenizer", + "instance or a WASM pointer to one."); + } + const sfapi = __fts5_api_from_db(db, true); + let pDestroy = 0; + const stackPos = wasm.pstack.pointer; + try{ + if(xDestroy instanceof Function){ + pDestroy = wasm.installFunction(xDestroy, 'v(p)'); + } + const xct = sfapi.$$xCreateTokenizer || ( + sfapi.$$xCreateTokenizer = wasm.xWrap(sfapi.$xCreateTokenizer, 'int', [ + '*', 'string', '*', '*', '*' + /* fts5_api*, const char *zName, void *pContext, + fts5_tokenizer *pTokenizer, void(*xDestroy)(void*) */ + ]) + ); + const outPtr = wasm.pstack.allocPtr(); + const rc = xct(fapi.pointer, name, 0, pTokenizer, pDestroy || xDestroy || 0 ); + if(rc) toss(rc,"FTS5::xCreateFunction() failed."); + if(pDestroy) __addCleanupForFunc(sfapi, name, pDestroy); + }catch(e){ + if(pDestroy) wasm.uninstallFunction(pDestroy); + sfapi.dispose(); + throw e; + }finally{ + wasm.pstack.restore(stackPost); + } + }; + //fts.createTokenizer = createTokenizer; + }/*sqlite3ApiBootstrap.initializers.push()*/); diff --git a/ext/wasm/api/sqlite3-wasm.c b/ext/wasm/api/sqlite3-wasm.c index a2dda0ca17..f0b10b13f7 100644 --- a/ext/wasm/api/sqlite3-wasm.c +++ b/ext/wasm/api/sqlite3-wasm.c @@ -1284,6 +1284,31 @@ const char * sqlite3_wasm_enum_json(void){ // ); } _StructBinder; #undef CurrentStruct + +#define CurrentStruct fts5_tokenizer + StructBinder { + M(xCreate, "i(ppip)"); + //^^^ int (*)(void*, const char **azArg, int nArg, Fts5Tokenizer **ppOut); + M(xDelete, "v(p)"); + //^^^ void(Fts5Tokenizer*) + M(xTokenize, "i(ppipip)"); + /**^^^ int (*xTokenize)(Fts5Tokenizer*, + void *pCtx, + int flags, + const char *pText, + int nText, + int (*xToken)( + void *pCtx, + int tflags, + const char *pToken, + int nToken, + int iStart, + int iEnd + ) + ); */ + } _StructBinder; +#undef CurrentStruct + #endif /* SQLITE_ENABLE_FTS5 */ #if SQLITE_WASM_TESTS diff --git a/ext/wasm/tester1.c-pp.js b/ext/wasm/tester1.c-pp.js index ee6696ea3b..83e15ef75f 100644 --- a/ext/wasm/tester1.c-pp.js +++ b/ext/wasm/tester1.c-pp.js @@ -2942,6 +2942,10 @@ globalThis.sqlite3InitModule = sqlite3InitModule; ); T.assert( 1 === list.length ) .assert( 'MY+a2:b2' === list[0] ); + + //const fTok = new fts.fts5_tokenizer(); + //fTok.installMethods({}); + db.close(); T.assert( destroyCalled ); //toss("Testing"); diff --git a/manifest b/manifest index 4029666b20..091074effc 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Minor\sinternal\scleanups\sin\sthe\sJS-side\sfts5\scleanup\ssteps. -D 2023-08-03T22:43:39.189 +C More\swork\stowards\sfts5\scustomzation\sin\sJS. +D 2023-08-04T08:39:21.730 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -505,13 +505,13 @@ F ext/wasm/api/sqlite3-api-glue.js 861ff5ccfbaa50579c46618be53ac374202ad73cc27eb F ext/wasm/api/sqlite3-api-oo1.js 9678dc4d9a5d39632b6ffe6ea94a023119260815bf32f265bf5f6c36c9516db8 F ext/wasm/api/sqlite3-api-prologue.js 76258e160bf6a89cc75a7d3c05646a054c8cab7219cd1e10bc20cacaad022131 F ext/wasm/api/sqlite3-api-worker1.js 9f32af64df1a031071912eea7a201557fe39b1738645c0134562bb84e88e2fec -F ext/wasm/api/sqlite3-fts5-helper.js 9cdc409db558b9321aa4eedffb30d62b5fe59ccd62c568f83d432d5f1d8f4c68 +F ext/wasm/api/sqlite3-fts5-helper.js b7716c46acfc591421aa6f92c3962764666786a92b5bc116e85523c2d15f3f09 F ext/wasm/api/sqlite3-license-version-header.js 0c807a421f0187e778dc1078f10d2994b915123c1223fe752b60afdcd1263f89 F ext/wasm/api/sqlite3-opfs-async-proxy.js 8cf8a897726f14071fae6be6648125162b256dfb4f96555b865dbb7a6b65e379 F ext/wasm/api/sqlite3-v-helper.js 8dc3da6e69d51f455b64cc88fec7977f528d79ec267cfcdd97b606c8cc4cd156 F ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js 54cee22aacadb9dfaea438d72ac0882249d028c37903208d48c52871290ceff7 F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js e7a690e0e78ff4d563f2eca468f91db69f001ff4b79c6d2304cbb6f62dca437d -F ext/wasm/api/sqlite3-wasm.c 45d2ce0089f1a41123fb88998c6f55be8d6cc2fdde8f861a5181a93f4ba57a97 +F ext/wasm/api/sqlite3-wasm.c 4f0ff557fa67a4b8da8a68a6736e71265378b9d1a24ac33165a7d58fcad6cda4 F ext/wasm/api/sqlite3-worker1-promiser.c-pp.js bc06df0d599e625bde6a10a394e326dc68da9ff07fa5404354580f81566e591f F ext/wasm/api/sqlite3-worker1.c-pp.js da509469755035e919c015deea41b4514b5e84c12a1332e6cc8d42cb2cc1fb75 F ext/wasm/batch-runner.html 4deeed44fe41496dc6898d9fb17938ea3291f40f4bfb977e29d0cef96fbbe4c8 @@ -556,7 +556,7 @@ F ext/wasm/test-opfs-vfs.html 1f2d672f3f3fce810dfd48a8d56914aba22e45c6834e262555 F ext/wasm/test-opfs-vfs.js f09266873e1a34d9bdb6d3981ec8c9e382f31f215c9fd2f9016d2394b8ae9b7b F ext/wasm/tester1-worker.html ebc4b820a128963afce328ecf63ab200bd923309eb939f4110510ab449e9814c F ext/wasm/tester1.c-pp.html 1c1bc78b858af2019e663b1a31e76657b73dc24bede28ca92fbe917c3a972af2 -F ext/wasm/tester1.c-pp.js 4ef8b4d4f369a0ad0975dbc3709dd4fd9d2d099f8053595f83ef58515b9656d2 +F ext/wasm/tester1.c-pp.js 5275b89d3657e1ca1b868f227539136a4642d42474e0202191fff9d7f4f91fff F ext/wasm/tests/opfs/concurrency/index.html 0802373d57034d51835ff6041cda438c7a982deea6079efd98098d3e42fbcbc1 F ext/wasm/tests/opfs/concurrency/test.js a98016113eaf71e81ddbf71655aa29b0fed9a8b79a3cdd3620d1658eb1cc9a5d F ext/wasm/tests/opfs/concurrency/worker.js 0a8c1a3e6ebb38aabbee24f122693f1fb29d599948915c76906681bb7da1d3d2 @@ -2051,8 +2051,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P f4b9743abd4fe24f1604cbcc7f9f95cadd1cefdc053eeabb35dc6773c99d1277 -R f44eb85aef02129cd0dbe3caa8ffb3ae +P 2666f52e88691201a30dc0f27c294c498bba3f16d5d2dddcadcfac115a3cba20 +R 643c955846e19c824f4b95b5e770827f U stephan -Z 71921d9fea535a9531daebace1ceec5c +Z 00d6d6bd94d004d3b32085de27313db7 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 46179932b7..ae48624d7c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -2666f52e88691201a30dc0f27c294c498bba3f16d5d2dddcadcfac115a3cba20 \ No newline at end of file +ce2a65d80f7793ae8183e311fc7284b37caa78d747436f22496284dfa57e3533 \ No newline at end of file