From: stephan Date: Tue, 3 Mar 2026 23:43:40 +0000 (+0000) Subject: An initial attempt to plugging opfs-wl into the build but its initial handshake with... X-Git-Tag: major-release~100^2~28 X-Git-Url: http://git.ipfire.org/gitweb.cgi?a=commitdiff_plain;h=a2d3a54f80b890a25e82877bcf2b612a64919255;p=thirdparty%2Fsqlite.git An initial attempt to plugging opfs-wl into the build but its initial handshake with the async half collides with the opfs VFS's handshake, causing bootstrapping to fail miserably. We'll need to either devise a handshake which can differentiate between the two instances or we'll need to preprocess sqlite3-opfs-async-proxy into two copies. Move the (now) three copies of some common code shared by the opfs pieces into a preprocessor #include. FossilOrigin-Name: 1e0b72631aecb0bb72f4089116da221e6c4abf962db589de08132cd52c2be0e2 --- diff --git a/ext/wasm/GNUmakefile b/ext/wasm/GNUmakefile index dc3c2d2555..371bec6bdd 100644 --- a/ext/wasm/GNUmakefile +++ b/ext/wasm/GNUmakefile @@ -913,6 +913,7 @@ endif sqlite3-api.jses += $(dir.api)/sqlite3-vfs-kvvfs.c-pp.js sqlite3-api.jses += $(dir.api)/sqlite3-vfs-opfs.c-pp.js sqlite3-api.jses += $(dir.api)/sqlite3-vfs-opfs-sahpool.c-pp.js +#sqlite3-api.jses += $(dir.api)/sqlite3-vfs-opfs-wl.c-pp.js # Parallel builds can fail if $(sqlite3-license-version.js) is not # created early enough, so make all files in $(sqlite-api.jses) except @@ -1182,13 +1183,13 @@ all: demos ####################################################################### # -# "SOAP" is a static file which is not part of the amalgamation but -# gets copied into the build output folder and into each of the fiddle -# builds. +# "SOAP" is not part of the amalgamation but gets copied into the +# build output folder and into each of the fiddle builds. # sqlite3.ext.js += $(dir.dout)/sqlite3-opfs-async-proxy.js -$(dir.dout)/sqlite3-opfs-async-proxy.js: $(dir.api)/sqlite3-opfs-async-proxy.js - @$(call b.cp,@,$<,$@) +$(eval $(call b.c-pp.target,soap,\ + $(dir.api)/sqlite3-opfs-async-proxy.c-pp.js,\ + $(dir.dout)/sqlite3-opfs-async-proxy.js)) # # Add a dep of $(sqlite3.ext.js) on every individual build's JS file. diff --git a/ext/wasm/api/opfs-common.c-pp.js b/ext/wasm/api/opfs-common.c-pp.js new file mode 100644 index 0000000000..1a7818927a --- /dev/null +++ b/ext/wasm/api/opfs-common.c-pp.js @@ -0,0 +1,200 @@ +//#if nope +/** + This file is for preprocessor #include into the "opfs" and + "opfs-wl" impls, as well as their async-proxy part. +*/ +//#endif + +//#if not defined opfs-async-proxy +/** + TODO: move the sqlite3.opfs (private/internal) namespace object + init from sqlite3-vfs-opfs*.js into here. That namespace gets + removed from the sqlite3 namespace in the final stages of library + bootstrapping except in test runs, where it's retained so that + tests can clean up OPFS so their test cases work (the down-side of + them being persistent). +*/ +//#endif not defined opfs-async-proxy + +// This function won't work as-is if we #include it, but the +// missing elements have not yet been identified. +const initS11n = function(){ + /* This function is needed by the "opfs" and "opfs-wl" VFSes + and sqlite-opfs-async-proxy.js (used by those of those). */ + /** + This proxy de/serializes cross-thread function arguments and + output-pointer values via the state.sabIO SharedArrayBuffer, + using the region defined by (state.sabS11nOffset, + state.sabS11nOffset + state.sabS11nSize]. Only one dataset is + recorded at a time. + + This is not a general-purpose format. It only supports the + range of operations, and data sizes, needed by the + sqlite3_vfs and sqlite3_io_methods operations. Serialized + data are transient and this serialization algorithm may + change at any time. + + The data format can be succinctly summarized as: + + Nt...Td...D + + Where: + + - N = number of entries (1 byte) + + - t = type ID of first argument (1 byte) + + - ...T = type IDs of the 2nd and subsequent arguments (1 byte + each). + + - d = raw bytes of first argument (per-type size). + + - ...D = raw bytes of the 2nd and subsequent arguments (per-type + size). + + All types except strings have fixed sizes. Strings are stored + using their TextEncoder/TextDecoder representations. It would + arguably make more sense to store them as Int16Arrays of + their JS character values, but how best/fastest to get that + in and out of string form is an open point. Initial + experimentation with that approach did not gain us any speed. + + Historical note: this impl was initially about 1% this size by + using using JSON.stringify/parse(), but using fit-to-purpose + serialization saves considerable runtime. + */ + if(state.s11n) return state.s11n; + const textDecoder = new TextDecoder(), + textEncoder = new TextEncoder('utf-8'), + viewU8 = new Uint8Array(state.sabIO, state.sabS11nOffset, state.sabS11nSize), + viewDV = new DataView(state.sabIO, state.sabS11nOffset, state.sabS11nSize); + state.s11n = Object.create(null); + /* Only arguments and return values of these types may be + serialized. This covers the whole range of types needed by the + sqlite3_vfs API. */ + const TypeIds = Object.create(null); + TypeIds.number = { id: 1, size: 8, getter: 'getFloat64', setter: 'setFloat64' }; + TypeIds.bigint = { id: 2, size: 8, getter: 'getBigInt64', setter: 'setBigInt64' }; + TypeIds.boolean = { id: 3, size: 4, getter: 'getInt32', setter: 'setInt32' }; + TypeIds.string = { id: 4 }; + + const getTypeId = (v)=>( + TypeIds[typeof v] + || toss("Maintenance required: this value type cannot be serialized.",v) + ); + const getTypeIdById = (tid)=>{ + switch(tid){ + case TypeIds.number.id: return TypeIds.number; + case TypeIds.bigint.id: return TypeIds.bigint; + case TypeIds.boolean.id: return TypeIds.boolean; + case TypeIds.string.id: return TypeIds.string; + default: toss("Invalid type ID:",tid); + } + }; + + /** + Returns an array of the deserialized state stored by the most + recent serialize() operation (from this thread or the + counterpart thread), or null if the serialization buffer is + empty. If passed a truthy argument, the serialization buffer + is cleared after deserialization. + */ + state.s11n.deserialize = function(clear=false){ +//#if defined opfs-has-metrics + ++metrics.s11n.deserialize.count; +//#endif + const t = performance.now(); + const argc = viewU8[0]; + const rc = argc ? [] : null; + if(argc){ + const typeIds = []; + let offset = 1, i, n, v; + for(i = 0; i < argc; ++i, ++offset){ + typeIds.push(getTypeIdById(viewU8[offset])); + } + for(i = 0; i < argc; ++i){ + const t = typeIds[i]; + if(t.getter){ + v = viewDV[t.getter](offset, state.littleEndian); + offset += t.size; + }else{/*String*/ + n = viewDV.getInt32(offset, state.littleEndian); + offset += 4; + v = textDecoder.decode(viewU8.slice(offset, offset+n)); + offset += n; + } + rc.push(v); + } + } + if(clear) viewU8[0] = 0; + //log("deserialize:",argc, rc); +//#if defined opfs-has-metrics + metrics.s11n.deserialize.time += performance.now() - t; +//#endif + return rc; + }; + + /** + Serializes all arguments to the shared buffer for consumption + by the counterpart thread. + + This routine is only intended for serializing OPFS VFS + arguments and (in at least one special case) result values, + and the buffer is sized to be able to comfortably handle + those. + + If passed no arguments then it zeroes out the serialization + state. + */ + state.s11n.serialize = function(...args){ + const t = performance.now(); +//#if defined opfs-has-metrics + ++metrics.s11n.serialize.count; +//#endif + if(args.length){ + //log("serialize():",args); + const typeIds = []; + let i = 0, offset = 1; + viewU8[0] = args.length & 0xff /* header = # of args */; + for(; i < args.length; ++i, ++offset){ + /* Write the TypeIds.id value into the next args.length + bytes. */ + typeIds.push(getTypeId(args[i])); + viewU8[offset] = typeIds[i].id; + } + for(i = 0; i < args.length; ++i) { + /* Deserialize the following bytes based on their + corresponding TypeIds.id from the header. */ + const t = typeIds[i]; + if(t.setter){ + viewDV[t.setter](offset, args[i], state.littleEndian); + offset += t.size; + }else{/*String*/ + const s = textEncoder.encode(args[i]); + viewDV.setInt32(offset, s.byteLength, state.littleEndian); + offset += 4; + viewU8.set(s, offset); + offset += s.byteLength; + } + } + //log("serialize() result:",viewU8.slice(0,offset)); + }else{ + viewU8[0] = 0; + } +//#if defined opfs-has-metrics + metrics.s11n.serialize.time += performance.now() - t; +//#endif + }; + +//#if defined opfs-async-proxy + state.s11n.storeException = state.asyncS11nExceptions + ? ((priority,e)=>{ + if(priority<=state.asyncS11nExceptions){ + state.s11n.serialize([e.name,': ',e.message].join("")); + } + }) + : ()=>{}; +//#endif + + return state.s11n; +}/*initS11n()*/; diff --git a/ext/wasm/api/post-js-header.js b/ext/wasm/api/post-js-header.js index 670051bd86..348f80ea0b 100644 --- a/ext/wasm/api/post-js-header.js +++ b/ext/wasm/api/post-js-header.js @@ -34,6 +34,7 @@ Module.runSQLite3PostLoadInit = async function( - sqlite3-vtab-helper.c-pp.js => Utilities for virtual table impls - sqlite3-vfs-opfs.c-pp.js => OPFS VFS - sqlite3-vfs-opfs-sahpool.c-pp.js => OPFS SAHPool VFS + - sqlite3-vfs-opfs-wl.c-pp.js => WebLock-using OPFS VFS - post-js-footer.js => this file's epilogue And all of that gets sandwiched between extern-pre-js.js and diff --git a/ext/wasm/api/sqlite3-api-prologue.js b/ext/wasm/api/sqlite3-api-prologue.js index c53acee769..3ee44d33b0 100644 --- a/ext/wasm/api/sqlite3-api-prologue.js +++ b/ext/wasm/api/sqlite3-api-prologue.js @@ -2002,6 +2002,7 @@ globalThis.sqlite3ApiBootstrap = async function sqlite3ApiBootstrap( so that we can add tests for them. */ delete sqlite3.util; delete sqlite3.StructBinder; + delete sqlite3.opfs; } return sqlite3; }; @@ -2034,7 +2035,7 @@ globalThis.sqlite3ApiBootstrap = async function sqlite3ApiBootstrap( */ scriptInfo: undefined }; - if( ('undefined'!==typeof sqlite3IsUnderTest/* from post-js-header.js */) ){ + if( 'undefined'!==typeof sqlite3IsUnderTest/* from post-js-header.js */ ){ sqlite3.__isUnderTest = !!sqlite3IsUnderTest; } try{ diff --git a/ext/wasm/api/sqlite3-opfs-async-proxy.js b/ext/wasm/api/sqlite3-opfs-async-proxy.c-pp.js similarity index 88% rename from ext/wasm/api/sqlite3-opfs-async-proxy.js rename to ext/wasm/api/sqlite3-opfs-async-proxy.c-pp.js index ede2bc7cd4..294e7687be 100644 --- a/ext/wasm/api/sqlite3-opfs-async-proxy.js +++ b/ext/wasm/api/sqlite3-opfs-async-proxy.c-pp.js @@ -603,105 +603,8 @@ const installAsyncProxy = function(){ } }/*vfsAsyncImpls*/; - const initS11n = ()=>{ - /** - ACHTUNG: this code is 100% duplicated in the other half of this - proxy! The documentation is maintained in the "synchronous half". - */ - if(state.s11n) return state.s11n; - const textDecoder = new TextDecoder(), - textEncoder = new TextEncoder('utf-8'), - viewU8 = new Uint8Array(state.sabIO, state.sabS11nOffset, state.sabS11nSize), - viewDV = new DataView(state.sabIO, state.sabS11nOffset, state.sabS11nSize); - state.s11n = Object.create(null); - const TypeIds = Object.create(null); - TypeIds.number = { id: 1, size: 8, getter: 'getFloat64', setter: 'setFloat64' }; - TypeIds.bigint = { id: 2, size: 8, getter: 'getBigInt64', setter: 'setBigInt64' }; - TypeIds.boolean = { id: 3, size: 4, getter: 'getInt32', setter: 'setInt32' }; - TypeIds.string = { id: 4 }; - const getTypeId = (v)=>( - TypeIds[typeof v] - || toss("Maintenance required: this value type cannot be serialized.",v) - ); - const getTypeIdById = (tid)=>{ - switch(tid){ - case TypeIds.number.id: return TypeIds.number; - case TypeIds.bigint.id: return TypeIds.bigint; - case TypeIds.boolean.id: return TypeIds.boolean; - case TypeIds.string.id: return TypeIds.string; - default: toss("Invalid type ID:",tid); - } - }; - state.s11n.deserialize = function(clear=false){ - const argc = viewU8[0]; - const rc = argc ? [] : null; - if(argc){ - const typeIds = []; - let offset = 1, i, n, v; - for(i = 0; i < argc; ++i, ++offset){ - typeIds.push(getTypeIdById(viewU8[offset])); - } - for(i = 0; i < argc; ++i){ - const t = typeIds[i]; - if(t.getter){ - v = viewDV[t.getter](offset, state.littleEndian); - offset += t.size; - }else{/*String*/ - n = viewDV.getInt32(offset, state.littleEndian); - offset += 4; - v = textDecoder.decode(viewU8.slice(offset, offset+n)); - offset += n; - } - rc.push(v); - } - } - if(clear) viewU8[0] = 0; - //log("deserialize:",argc, rc); - return rc; - }; - state.s11n.serialize = function(...args){ - if(args.length){ - //log("serialize():",args); - const typeIds = []; - let i = 0, offset = 1; - viewU8[0] = args.length & 0xff /* header = # of args */; - for(; i < args.length; ++i, ++offset){ - /* Write the TypeIds.id value into the next args.length - bytes. */ - typeIds.push(getTypeId(args[i])); - viewU8[offset] = typeIds[i].id; - } - for(i = 0; i < args.length; ++i) { - /* Deserialize the following bytes based on their - corresponding TypeIds.id from the header. */ - const t = typeIds[i]; - if(t.setter){ - viewDV[t.setter](offset, args[i], state.littleEndian); - offset += t.size; - }else{/*String*/ - const s = textEncoder.encode(args[i]); - viewDV.setInt32(offset, s.byteLength, state.littleEndian); - offset += 4; - viewU8.set(s, offset); - offset += s.byteLength; - } - } - //log("serialize() result:",viewU8.slice(0,offset)); - }else{ - viewU8[0] = 0; - } - }; - - state.s11n.storeException = state.asyncS11nExceptions - ? ((priority,e)=>{ - if(priority<=state.asyncS11nExceptions){ - state.s11n.serialize([e.name,': ',e.message].join("")); - } - }) - : ()=>{}; - - return state.s11n; - }/*initS11n()*/; +//#define opfs-async-proxy +//#include api/opfs-common.c-pp.js /** Starts a new WebLock request. diff --git a/ext/wasm/api/sqlite3-vfs-opfs-wl.c-pp.js b/ext/wasm/api/sqlite3-vfs-opfs-wl.c-pp.js index 0184bd480e..dfa44af5e1 100644 --- a/ext/wasm/api/sqlite3-vfs-opfs-wl.c-pp.js +++ b/ext/wasm/api/sqlite3-vfs-opfs-wl.c-pp.js @@ -11,54 +11,51 @@ *********************************************************************** - This file is a placeholder for a reimplementation of the "opfs" VFS - (as distinct from "opfs-sahpool") which uses WebLocks instead - locking based on a bespoke custom Atomics.wait()/notify() - protocol. This file holds the "synchronous half" of the VFS, whereas - it shares the "asynchronous half" of the "opfs" VFS. + This file is a reimplementation of the "opfs" VFS (as distinct from + "opfs-sahpool") which uses WebLocks for locking instead of a bespoke + custom Atomics.wait()/notify() protocol. This file holds the + "synchronous half" of the VFS, whereas it shares the "asynchronous + half" of the "opfs" VFS. This file is intended to be appended to the main sqlite3 JS deliverable somewhere after sqlite3-api-oo1.js. + + TODOs (2026-0303): + + - Move pieces of this file which are common to it, + sqlite3-vfs-opfs.c-pp.js, and/or sqlite3-opfs-async-proxy.js into + separate files and #include them in each using the preprocessor. + e.g. the s11n namespace object is duplicated in all three files. + + - For purposes of tester1.js we need to figure out which of these + VFSes will install the (internal-use-only) sqlite3.opfs utility code + namespace. We need that in order to clean up OPFS files during test + runs. Alternately, move those into their own + sqlite3ApiBootstrap.initializers entry which precedes both of the + VFSes, so they'll have access to it during bootstrapping. The + sqlite3.opfs namespace is removed at the end of bootstrapping unless + the library is told to run in testing mode (which is not a + documented feature). */ 'use strict'; globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ /** installOpfsVfs() returns a Promise which, on success, installs an - sqlite3_vfs named "opfs", suitable for use with all sqlite3 APIs + sqlite3_vfs named "opfs-wl", suitable for use with all sqlite3 APIs which accept a VFS. It is intended to be called via sqlite3ApiBootstrap.initializers or an equivalent mechanism. - The installed VFS uses the Origin-Private FileSystem API for - all file storage. On error it is rejected with an exception - explaining the problem. Reasons for rejection include, but are - not limited to: - - - The counterpart Worker (see below) could not be loaded. - - - The environment does not support OPFS. That includes when - this function is called from the main window thread. + This VFS is essentially a copy of the "opfs" VFS but uses + WebLocks for its xLock() and xUnlock() implementations. - Significant notes and limitations: + Quirks specific to this VFS: - - The OPFS features used here are only available in dedicated Worker - threads. This file tries to detect that case, resulting in a - rejected Promise if those features do not seem to be available. + - Because WebLocks effectively block until they return, they will + effectively hang on locks rather than returning SQLITE_BUSY. - - It requires the SharedArrayBuffer and Atomics classes, and the - former is only available if the HTTP server emits the so-called - COOP and COEP response headers. These features are required for - proxying OPFS's synchronous API via the synchronous interface - required by the sqlite3_vfs API. - - This function may only be called a single time. When called, this - function removes itself from the sqlite3 object. - - All arguments to this function are for internal/development purposes - only. They do not constitute a public API and may change at any - time. - - The argument may optionally be a plain object with the following - configuration options: + The argument may optionally be a plain object with the following + configuration options: - proxyUri: name of the async proxy JS file. @@ -67,14 +64,8 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ additionally enables debugging info. Logging is performed via the sqlite3.config.{log|warn|error}() functions. - - sanityChecks (=false): if true, some basic sanity tests are run on - the OPFS VFS API after it's initialized, before the returned - Promise resolves. This is only intended for testing and - development of the VFS, not client-side use. - On success, the Promise resolves to the top-most sqlite3 namespace - object and that object gets a new object installed in its - `opfs` property, containing several OPFS-specific utilities. + object. */ const installOpfsVfs = function callee(options){ if(!globalThis.SharedArrayBuffer @@ -115,9 +106,6 @@ const installOpfsVfs = function callee(options){ if(undefined===options.proxyUri){ options.proxyUri = callee.defaultProxyUri; } - - //sqlite3.config.warn("OPFS options =",options,globalThis.location); - if('function' === typeof options.proxyUri){ options.proxyUri = options.proxyUri(); } @@ -524,171 +512,8 @@ const installOpfsVfs = function callee(options){ } }; - const initS11n = ()=>{ - /** - !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - ACHTUNG: this code is 100% duplicated in the other half of - this proxy! The documentation is maintained in the - "synchronous half". - !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - - This proxy de/serializes cross-thread function arguments and - output-pointer values via the state.sabIO SharedArrayBuffer, - using the region defined by (state.sabS11nOffset, - state.sabS11nOffset + state.sabS11nSize]. Only one dataset is - recorded at a time. - - This is not a general-purpose format. It only supports the - range of operations, and data sizes, needed by the - sqlite3_vfs and sqlite3_io_methods operations. Serialized - data are transient and this serialization algorithm may - change at any time. - - The data format can be succinctly summarized as: - - Nt...Td...D - - Where: - - - N = number of entries (1 byte) - - - t = type ID of first argument (1 byte) - - - ...T = type IDs of the 2nd and subsequent arguments (1 byte - each). - - - d = raw bytes of first argument (per-type size). - - - ...D = raw bytes of the 2nd and subsequent arguments (per-type - size). - - All types except strings have fixed sizes. Strings are stored - using their TextEncoder/TextDecoder representations. It would - arguably make more sense to store them as Int16Arrays of - their JS character values, but how best/fastest to get that - in and out of string form is an open point. Initial - experimentation with that approach did not gain us any speed. - - Historical note: this impl was initially about 1% this size by - using using JSON.stringify/parse(), but using fit-to-purpose - serialization saves considerable runtime. - */ - if(state.s11n) return state.s11n; - const textDecoder = new TextDecoder(), - textEncoder = new TextEncoder('utf-8'), - viewU8 = new Uint8Array(state.sabIO, state.sabS11nOffset, state.sabS11nSize), - viewDV = new DataView(state.sabIO, state.sabS11nOffset, state.sabS11nSize); - state.s11n = Object.create(null); - /* Only arguments and return values of these types may be - serialized. This covers the whole range of types needed by the - sqlite3_vfs API. */ - const TypeIds = Object.create(null); - TypeIds.number = { id: 1, size: 8, getter: 'getFloat64', setter: 'setFloat64' }; - TypeIds.bigint = { id: 2, size: 8, getter: 'getBigInt64', setter: 'setBigInt64' }; - TypeIds.boolean = { id: 3, size: 4, getter: 'getInt32', setter: 'setInt32' }; - TypeIds.string = { id: 4 }; - - const getTypeId = (v)=>( - TypeIds[typeof v] - || toss("Maintenance required: this value type cannot be serialized.",v) - ); - const getTypeIdById = (tid)=>{ - switch(tid){ - case TypeIds.number.id: return TypeIds.number; - case TypeIds.bigint.id: return TypeIds.bigint; - case TypeIds.boolean.id: return TypeIds.boolean; - case TypeIds.string.id: return TypeIds.string; - default: toss("Invalid type ID:",tid); - } - }; - - /** - Returns an array of the deserialized state stored by the most - recent serialize() operation (from this thread or the - counterpart thread), or null if the serialization buffer is - empty. If passed a truthy argument, the serialization buffer - is cleared after deserialization. - */ - state.s11n.deserialize = function(clear=false){ - ++metrics.s11n.deserialize.count; - const t = performance.now(); - const argc = viewU8[0]; - const rc = argc ? [] : null; - if(argc){ - const typeIds = []; - let offset = 1, i, n, v; - for(i = 0; i < argc; ++i, ++offset){ - typeIds.push(getTypeIdById(viewU8[offset])); - } - for(i = 0; i < argc; ++i){ - const t = typeIds[i]; - if(t.getter){ - v = viewDV[t.getter](offset, state.littleEndian); - offset += t.size; - }else{/*String*/ - n = viewDV.getInt32(offset, state.littleEndian); - offset += 4; - v = textDecoder.decode(viewU8.slice(offset, offset+n)); - offset += n; - } - rc.push(v); - } - } - if(clear) viewU8[0] = 0; - //log("deserialize:",argc, rc); - metrics.s11n.deserialize.time += performance.now() - t; - return rc; - }; - - /** - Serializes all arguments to the shared buffer for consumption - by the counterpart thread. - - This routine is only intended for serializing OPFS VFS - arguments and (in at least one special case) result values, - and the buffer is sized to be able to comfortably handle - those. - - If passed no arguments then it zeroes out the serialization - state. - */ - state.s11n.serialize = function(...args){ - const t = performance.now(); - ++metrics.s11n.serialize.count; - if(args.length){ - //log("serialize():",args); - const typeIds = []; - let i = 0, offset = 1; - viewU8[0] = args.length & 0xff /* header = # of args */; - for(; i < args.length; ++i, ++offset){ - /* Write the TypeIds.id value into the next args.length - bytes. */ - typeIds.push(getTypeId(args[i])); - viewU8[offset] = typeIds[i].id; - } - for(i = 0; i < args.length; ++i) { - /* Deserialize the following bytes based on their - corresponding TypeIds.id from the header. */ - const t = typeIds[i]; - if(t.setter){ - viewDV[t.setter](offset, args[i], state.littleEndian); - offset += t.size; - }else{/*String*/ - const s = textEncoder.encode(args[i]); - viewDV.setInt32(offset, s.byteLength, state.littleEndian); - offset += 4; - viewU8.set(s, offset); - offset += s.byteLength; - } - } - //log("serialize() result:",viewU8.slice(0,offset)); - }else{ - viewU8[0] = 0; - } - metrics.s11n.serialize.time += performance.now() - t; - }; - return state.s11n; - }/*initS11n()*/; +//#define opfs-has-metrics +//#include api/opfs-common.c-pp.js /** Generates a random ASCII string len characters long, intended for @@ -1323,14 +1148,14 @@ const installOpfsVfs = function callee(options){ }; if(sqlite3.oo1){ - const OpfsDb = function(...args){ + const OpfsWlDb = function(...args){ const opt = sqlite3.oo1.DB.dbCtorHelper.normalizeArgs(...args); opt.vfs = opfsVfs.$zName; sqlite3.oo1.DB.dbCtorHelper.call(this, opt); }; - OpfsDb.prototype = Object.create(sqlite3.oo1.DB.prototype); - sqlite3.oo1.OpfsDb = OpfsDb; - OpfsDb.importDb = opfsUtil.importDb; + OpfsWlDb.prototype = Object.create(sqlite3.oo1.DB.prototype); + sqlite3.oo1.OpfsWlDb = OpfsWlDb; + OpfsWlDb.importDb = opfsUtil.importDb; sqlite3.oo1.DB.dbCtorHelper.setVfsPostOpenCallback( opfsVfs.pointer, function(oo1Db, sqlite3){ @@ -1446,7 +1271,7 @@ const installOpfsVfs = function callee(options){ navigator.storage.getDirectory().then((d)=>{ W.onerror = W._originalOnError; delete W._originalOnError; - sqlite3.opfs = opfsUtil; + //sqlite3.opfs = opfsUtil; opfsUtil.rootDirectory = d; log("End of OPFS sqlite3_vfs setup.", opfsVfs); promiseResolve(); diff --git a/ext/wasm/api/sqlite3-vfs-opfs.c-pp.js b/ext/wasm/api/sqlite3-vfs-opfs.c-pp.js index ffa90ed068..dd1b4ba90b 100644 --- a/ext/wasm/api/sqlite3-vfs-opfs.c-pp.js +++ b/ext/wasm/api/sqlite3-vfs-opfs.c-pp.js @@ -71,8 +71,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ development of the VFS, not client-side use. On success, the Promise resolves to the top-most sqlite3 namespace - object and that object gets a new object installed in its - `opfs` property, containing several OPFS-specific utilities. + object. */ const installOpfsVfs = function callee(options){ if(!globalThis.SharedArrayBuffer @@ -509,171 +508,8 @@ const installOpfsVfs = function callee(options){ } }; - const initS11n = ()=>{ - /** - !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - ACHTUNG: this code is 100% duplicated in the other half of - this proxy! The documentation is maintained in the - "synchronous half". - !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - - This proxy de/serializes cross-thread function arguments and - output-pointer values via the state.sabIO SharedArrayBuffer, - using the region defined by (state.sabS11nOffset, - state.sabS11nOffset + state.sabS11nSize]. Only one dataset is - recorded at a time. - - This is not a general-purpose format. It only supports the - range of operations, and data sizes, needed by the - sqlite3_vfs and sqlite3_io_methods operations. Serialized - data are transient and this serialization algorithm may - change at any time. - - The data format can be succinctly summarized as: - - Nt...Td...D - - Where: - - - N = number of entries (1 byte) - - - t = type ID of first argument (1 byte) - - - ...T = type IDs of the 2nd and subsequent arguments (1 byte - each). - - - d = raw bytes of first argument (per-type size). - - - ...D = raw bytes of the 2nd and subsequent arguments (per-type - size). - - All types except strings have fixed sizes. Strings are stored - using their TextEncoder/TextDecoder representations. It would - arguably make more sense to store them as Int16Arrays of - their JS character values, but how best/fastest to get that - in and out of string form is an open point. Initial - experimentation with that approach did not gain us any speed. - - Historical note: this impl was initially about 1% this size by - using using JSON.stringify/parse(), but using fit-to-purpose - serialization saves considerable runtime. - */ - if(state.s11n) return state.s11n; - const textDecoder = new TextDecoder(), - textEncoder = new TextEncoder('utf-8'), - viewU8 = new Uint8Array(state.sabIO, state.sabS11nOffset, state.sabS11nSize), - viewDV = new DataView(state.sabIO, state.sabS11nOffset, state.sabS11nSize); - state.s11n = Object.create(null); - /* Only arguments and return values of these types may be - serialized. This covers the whole range of types needed by the - sqlite3_vfs API. */ - const TypeIds = Object.create(null); - TypeIds.number = { id: 1, size: 8, getter: 'getFloat64', setter: 'setFloat64' }; - TypeIds.bigint = { id: 2, size: 8, getter: 'getBigInt64', setter: 'setBigInt64' }; - TypeIds.boolean = { id: 3, size: 4, getter: 'getInt32', setter: 'setInt32' }; - TypeIds.string = { id: 4 }; - - const getTypeId = (v)=>( - TypeIds[typeof v] - || toss("Maintenance required: this value type cannot be serialized.",v) - ); - const getTypeIdById = (tid)=>{ - switch(tid){ - case TypeIds.number.id: return TypeIds.number; - case TypeIds.bigint.id: return TypeIds.bigint; - case TypeIds.boolean.id: return TypeIds.boolean; - case TypeIds.string.id: return TypeIds.string; - default: toss("Invalid type ID:",tid); - } - }; - - /** - Returns an array of the deserialized state stored by the most - recent serialize() operation (from this thread or the - counterpart thread), or null if the serialization buffer is - empty. If passed a truthy argument, the serialization buffer - is cleared after deserialization. - */ - state.s11n.deserialize = function(clear=false){ - ++metrics.s11n.deserialize.count; - const t = performance.now(); - const argc = viewU8[0]; - const rc = argc ? [] : null; - if(argc){ - const typeIds = []; - let offset = 1, i, n, v; - for(i = 0; i < argc; ++i, ++offset){ - typeIds.push(getTypeIdById(viewU8[offset])); - } - for(i = 0; i < argc; ++i){ - const t = typeIds[i]; - if(t.getter){ - v = viewDV[t.getter](offset, state.littleEndian); - offset += t.size; - }else{/*String*/ - n = viewDV.getInt32(offset, state.littleEndian); - offset += 4; - v = textDecoder.decode(viewU8.slice(offset, offset+n)); - offset += n; - } - rc.push(v); - } - } - if(clear) viewU8[0] = 0; - //log("deserialize:",argc, rc); - metrics.s11n.deserialize.time += performance.now() - t; - return rc; - }; - - /** - Serializes all arguments to the shared buffer for consumption - by the counterpart thread. - - This routine is only intended for serializing OPFS VFS - arguments and (in at least one special case) result values, - and the buffer is sized to be able to comfortably handle - those. - - If passed no arguments then it zeroes out the serialization - state. - */ - state.s11n.serialize = function(...args){ - const t = performance.now(); - ++metrics.s11n.serialize.count; - if(args.length){ - //log("serialize():",args); - const typeIds = []; - let i = 0, offset = 1; - viewU8[0] = args.length & 0xff /* header = # of args */; - for(; i < args.length; ++i, ++offset){ - /* Write the TypeIds.id value into the next args.length - bytes. */ - typeIds.push(getTypeId(args[i])); - viewU8[offset] = typeIds[i].id; - } - for(i = 0; i < args.length; ++i) { - /* Deserialize the following bytes based on their - corresponding TypeIds.id from the header. */ - const t = typeIds[i]; - if(t.setter){ - viewDV[t.setter](offset, args[i], state.littleEndian); - offset += t.size; - }else{/*String*/ - const s = textEncoder.encode(args[i]); - viewDV.setInt32(offset, s.byteLength, state.littleEndian); - offset += 4; - viewU8.set(s, offset); - offset += s.byteLength; - } - } - //log("serialize() result:",viewU8.slice(0,offset)); - }else{ - viewU8[0] = 0; - } - metrics.s11n.serialize.time += performance.now() - t; - }; - return state.s11n; - }/*initS11n()*/; +//#define opfs-has-metrics +//#include api/opfs-common.c-pp.js /** Generates a random ASCII string len characters long, intended for diff --git a/ext/wasm/mkwasmbuilds.c b/ext/wasm/mkwasmbuilds.c index 37f2967d88..748594f4ca 100644 --- a/ext/wasm/mkwasmbuilds.c +++ b/ext/wasm/mkwasmbuilds.c @@ -994,7 +994,8 @@ static void mk_fiddle(void){ pf("$(out.%s.js): $(MAKEFILE_LIST) " "$(EXPORTED_FUNCTIONS.fiddle) " "$(fiddle.c.in) " - "$(pre-post.%s.deps)", + "$(pre-post.%s.deps) " + "$(dir.dout)/sqlite3-opfs-async-proxy.js", zBuildName, zBuildName); if( isDebug ){ pf(" $(dir.fiddle)/fiddle-worker.js" @@ -1014,10 +1015,6 @@ static void mk_fiddle(void){ pf("\t@$(call b.call.wasm-strip,%s)\n", zBuildName); pf("\t@$(call b.strip-js-emcc-bindings,$(logtag.%s))\n", zBuildName); - pf("\t@$(call b.cp," - "%s," - "$(dir.api)/sqlite3-opfs-async-proxy.js," - "$(dir $@))\n", zBuildName); if( isDebug ){ pf("\t@$(call b.cp,%s," "$(dir.fiddle)/index.html " diff --git a/ext/wasm/tests/opfs/concurrency/worker.js b/ext/wasm/tests/opfs/concurrency/worker.js index 5d28bedee0..a28d80f202 100644 --- a/ext/wasm/tests/opfs/concurrency/worker.js +++ b/ext/wasm/tests/opfs/concurrency/worker.js @@ -1,8 +1,9 @@ importScripts( - (new URL(self.location.href).searchParams).get('sqlite3.dir') + '/sqlite3.js' + (new URL(globalThis.location.href).searchParams).get('sqlite3.dir') + '/sqlite3.js' ); -self.sqlite3InitModule().then(async function(sqlite3){ - const urlArgs = new URL(self.location.href).searchParams; +globalThis.sqlite3InitModule.__isUnderTest = true; +globalThis.sqlite3InitModule().then(async function(sqlite3){ + const urlArgs = new URL(globalThis.location.href).searchParams; const options = { workerName: urlArgs.get('workerId') || Math.round(Math.random()*10000), unlockAsap: urlArgs.get('opfs-unlock-asap') || 0 /*EXPERIMENTAL*/ @@ -98,7 +99,7 @@ self.sqlite3InitModule().then(async function(sqlite3){ } }/*run()*/; - self.onmessage = function({data}){ + globalThis.onmessage = function({data}){ switch(data.type){ case 'run': run().catch((e)=>{ if(!interval.error) interval.error = e; diff --git a/manifest b/manifest index b8efc4b0b2..a23506fd3a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sa\smissing\sreset\sof\sthe\slock\shandshake\sSharedArrayBuffer\sslot. -D 2026-03-03T21:46:44.640 +C An\sinitial\sattempt\sto\splugging\sopfs-wl\sinto\sthe\sbuild\sbut\sits\sinitial\shandshake\swith\sthe\sasync\shalf\scollides\swith\sthe\sopfs\sVFS's\shandshake,\scausing\sbootstrapping\sto\sfail\smiserably.\sWe'll\sneed\sto\seither\sdevise\sa\shandshake\swhich\scan\sdifferentiate\sbetween\sthe\stwo\sinstances\sor\swe'll\sneed\sto\spreprocess\ssqlite3-opfs-async-proxy\sinto\stwo\scopies.\sMove\sthe\s(now)\sthree\scopies\sof\ssome\scommon\scode\sshared\sby\sthe\sopfs\spieces\sinto\sa\spreprocessor\s#include. +D 2026-03-03T23:43:40.326 F .fossil-settings/binary-glob 61195414528fb3ea9693577e1980230d78a1f8b0a54c78cf1b9b24d0a409ed6a x F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea @@ -572,7 +572,7 @@ F ext/session/sessionwor.test 6fd9a2256442cebde5b2284936ae9e0d54bde692d0f5fd009e F ext/session/sqlite3session.c 6ebd02be470f36d41c4bd78927f39d507b62051ba025eacaed9936c769902a07 F ext/session/sqlite3session.h 7404723606074fcb2afdc6b72c206072cdb2b7d8ba097ca1559174a80bc26f7a F ext/session/test_session.c 190110e3bd9463717248dec1272b44fe9943e57b7646d0b4200dcf11e4dccee6 -F ext/wasm/GNUmakefile 79236447d750609aa6beda30feec1314180c5462a493ad94214122887232bfd4 +F ext/wasm/GNUmakefile a64605e68d4561f440bdc538e01364210f75052469d6ea21bf2843f1539144e9 F ext/wasm/README-dist.txt f01081a850ce38a56706af6b481e3a7878e24e42b314cfcd4b129f0f8427066a F ext/wasm/README.md 2e87804e12c98f1d194b7a06162a88441d33bb443efcfe00dc6565a780d2f259 F ext/wasm/SQLTester/GNUmakefile e0794f676d55819951bbfae45cc5e8d7818dc460492dc317ce7f0d2eca15caff @@ -584,20 +584,21 @@ F ext/wasm/api/EXPORTED_FUNCTIONS.c-pp 7ba933e8f1290cc65459dd371c0c9a031d96bdf14 F ext/wasm/api/README.md a905d5c6bfc3e2df875bd391d6d6b7b48d41b43bdee02ad115b47244781a7e81 F ext/wasm/api/extern-post-js.c-pp.js d9f42ecbedc784c0d086bc37800e52946a14f7a21600b291daa3f963c314f930 F ext/wasm/api/extern-pre-js.js cc61c09c7a24a07dbecb4c352453c3985170cec12b4e7e7e7a4d11d43c5c8f41 +F ext/wasm/api/opfs-common.c-pp.js 5540c58aee8fbbf2c67984e0a35e2b680a7603052f2cc442d407da3cc81c4819 F ext/wasm/api/post-js-footer.js a50c1a2c4d008aede7b2aa1f18891a7ee71437c2f415b8aeb3db237ddce2935b -F ext/wasm/api/post-js-header.js d24bd0d065f3489c8b78ddf3ead6321e5d047187a162cd503c41700e03dd1f06 +F ext/wasm/api/post-js-header.js f35d2dcf1ab7f22a93d565f8e0b622a2934fc4e743edf3b708e4dd8140eeff55 F ext/wasm/api/pre-js.c-pp.js 9234ea680a2f6a2a177e8dcd934bdc5811a9f8409165433a252b87f4c07bba6f F ext/wasm/api/sqlite3-api-glue.c-pp.js 9b33e3ee467791dec4fd1b444b12a8545dfbb6c8b28ac651c7bdc7661a3b5a5c F ext/wasm/api/sqlite3-api-oo1.c-pp.js 45454631265d9ce82685f1a64e1650ee19c8e121c41db98a22b534c15e543cfa -F ext/wasm/api/sqlite3-api-prologue.js 1fefd40ab21e3dbf46f43b6fafb07f13eb13cc157a884f7c1134caf631ddb3f2 +F ext/wasm/api/sqlite3-api-prologue.js ccd8ece4b4580d2a70996218f28e810d70a86f5e2795f4d4a75f0603af24aef6 F ext/wasm/api/sqlite3-api-worker1.c-pp.js 1041dd645e8e821c082b628cd8d9acf70c667430f9d45167569633ffc7567938 F ext/wasm/api/sqlite3-license-version-header.js 98d90255a12d02214db634e041c8e7f2f133d9361a8ebf000ba9c9af4c6761cc -F ext/wasm/api/sqlite3-opfs-async-proxy.js 912ddb17627e933eb9596d393227f7ea47b1136fc2fb0d957d9979e71de59e81 +F ext/wasm/api/sqlite3-opfs-async-proxy.c-pp.js 0ec326d4c66c7af5a4c79a1be65c07c1e1844e641f133ba026a4a01eb1a37c56 w ext/wasm/api/sqlite3-opfs-async-proxy.js F ext/wasm/api/sqlite3-vfs-helper.c-pp.js 3f828cc66758acb40e9c5b4dcfd87fd478a14c8fb7f0630264e6c7fa0e57515d F ext/wasm/api/sqlite3-vfs-kvvfs.c-pp.js 2ccf4322f42063aefc150972943e750c77f7926b866f1639d40eec05df075b6e F ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js 1575ea6bbcf2da1e6df6892c17521a0c1c1c199a672e9090176ea0b88de48bd9 -F ext/wasm/api/sqlite3-vfs-opfs-wl.c-pp.js deea0f903e8265f58fe57315c35f62170fd65d4730bece7978772c28cead1402 -F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js 88ce2078267a2d1af57525a32d896295f4a8db7664de0e17e82dc9ff006ed8d3 +F ext/wasm/api/sqlite3-vfs-opfs-wl.c-pp.js 1abe919013d759b311e7e3f5c4a326f9027b1b6a0fbe89db5f303e2ff109d5d4 +F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js ab2a73f41a2e109f25fd3159531e801ca14c782c297b2fc1836e6a02b795321b F ext/wasm/api/sqlite3-vtab-helper.c-pp.js 366596d8ff73d4cefb938bbe95bc839d503c3fab6c8335ce4bf52f0d8a7dee81 F ext/wasm/api/sqlite3-wasm.c 45bb20e19b245136711f9b78584371233975811b6560c29ed9b650e225417e29 F ext/wasm/api/sqlite3-worker1-promiser.c-pp.js aa9715f661fb700459a5a6cb1c32a4d6a770723b47aa9ac0e16c2cf87d622a66 @@ -626,7 +627,7 @@ F ext/wasm/index.html 475bc283338749db4e3fbf24cf3f5aa020cc85a1fffb780d400a915fcb F ext/wasm/jaccwabyt/jaccwabyt.js 4e2b797dc170851c9c530c3567679f4aa509eec0fab73b466d945b00b356574b F ext/wasm/jaccwabyt/jaccwabyt.md 6aa90fa1a973d0ad10d077088bea163b241d8470c75eafdef87620a1de1dea41 F ext/wasm/mkdist.sh f8883b077a2ca47cf92e6f0ce305fbf72ca648c3501810125056c4b09c2d5554 x -F ext/wasm/mkwasmbuilds.c 0e9198eb90acae4bcf57cf62d7186f6af5aaac02efdb075a1aded33614b3805a +F ext/wasm/mkwasmbuilds.c b1ed20dfba178e6ccae91b4594ac9439992fd0cb783f202c639ca761f1b1d2a3 F ext/wasm/module-symbols.html e54f42112e0aac2a31f850ab33e7f2630a2ea4f63496f484a12469a2501e07e2 F ext/wasm/scratchpad-wasmfs.html a3d7388f3c4b263676b58b526846e9d02dfcb4014ff29d3a5040935286af5b96 F ext/wasm/scratchpad-wasmfs.mjs 66034b9256b218de59248aad796760a1584c1dd842231505895eff00dbd57c63 @@ -645,7 +646,7 @@ F ext/wasm/tester1.c-pp.html 52d88fe2c6f21a046030a36410b4839b632f4424028197a45a3 F ext/wasm/tester1.c-pp.js 6b946cd6d4da130dbae4a401057716d27117ca02cad2ea8c29ae9c46c675d618 F ext/wasm/tests/opfs/concurrency/index.html 657578a6e9ce1e9b8be951549ed93a6a471f4520a99e5b545928668f4285fb5e F ext/wasm/tests/opfs/concurrency/test.js d08889a5bb6e61937d0b8cbb78c9efbefbf65ad09f510589c779b7cc6a803a88 -F ext/wasm/tests/opfs/concurrency/worker.js 0a8c1a3e6ebb38aabbee24f122693f1fb29d599948915c76906681bb7da1d3d2 +F ext/wasm/tests/opfs/concurrency/worker.js 3e29c07bdd11f4ff9ee50e641e39c254d39243bbb83fb20a255755ee92739d12 F ext/wasm/tests/opfs/sahpool/digest-worker.js b0ab6218588f1f0a6d15a363b493ceaf29bfb87804d9e0165915a9996377cf79 F ext/wasm/tests/opfs/sahpool/digest.html 206d08a34dc8bd570b2581d3d9ab3ecad3201b516a598dd096dcf3cf8cd81df8 F ext/wasm/tests/opfs/sahpool/index.html be736567fd92d3ecb9754c145755037cbbd2bca01385e2732294b53f4c842328 @@ -2189,8 +2190,8 @@ F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee F tool/warnings.sh d924598cf2f55a4ecbc2aeb055c10bd5f48114793e7ba25f9585435da29e7e98 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f F tool/winmain.c 00c8fb88e365c9017db14c73d3c78af62194d9644feaf60e220ab0f411f3604c -P 3343e3aabe465f4ab91dd148dfc5a60346e9afb560c10d1f92aeae98763ec3ce -R f490861a3a61c6b12505aa6e9bec9268 +P 5910ed7ac843aa1fdb103ddcfaf6270d12cb39199205fc0b9f721fbf7fa2f851 +R d999a6bf319e44a9103bde13f0ab87af U stephan -Z 0dd9a22951092896d0ba74d53d905c49 +Z 097c558a7425489e055f6ff0be38c909 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 757a353e6f..9be206dbe2 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5910ed7ac843aa1fdb103ddcfaf6270d12cb39199205fc0b9f721fbf7fa2f851 +1e0b72631aecb0bb72f4089116da221e6c4abf962db589de08132cd52c2be0e2