From: stephan Date: Sun, 18 Sep 2022 02:35:30 +0000 (+0000) Subject: Move the OPFS VFS bits back into api/sqlite3-api-opfs.js. Refactor the OPFS VFS init... X-Git-Tag: version-3.40.0~169^2~102 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=c5313afea7bf38bbe98b5a472b16a8f3efda07c0;p=thirdparty%2Fsqlite.git Move the OPFS VFS bits back into api/sqlite3-api-opfs.js. Refactor the OPFS VFS init process to use a Promise-returning function which the client must call, as that eliminates any uncertainty about when the VFS (necessarily activated asynchronously) actually becomes available to the client. FossilOrigin-Name: 1c660970d0f62bcfd6e698a72b050d99972a1e39f45a5ac24194a190f8f78ab3 --- diff --git a/ext/wasm/GNUmakefile b/ext/wasm/GNUmakefile index dc0ce3681f..20354a6b5f 100644 --- a/ext/wasm/GNUmakefile +++ b/ext/wasm/GNUmakefile @@ -139,14 +139,13 @@ EXPORTED_FUNCTIONS.api: $(EXPORTED_FUNCTIONS.api.in) $(MAKEFILE) cat $(EXPORTED_FUNCTIONS.api.in) > $@ CLEAN_FILES += EXPORTED_FUNCTIONS.api -sqlite3-api.jses := \ - $(dir.api)/sqlite3-api-prologue.js \ - $(dir.common)/whwasmutil.js \ - $(dir.jacc)/jaccwabyt.js \ - $(dir.api)/sqlite3-api-glue.js \ - $(dir.api)/sqlite3-api-oo1.js \ - $(dir.api)/sqlite3-api-worker1.js -#sqlite3-api.jses += $(dir.api)/sqlite3-api-opfs.js +sqlite3-api.jses := $(dir.api)/sqlite3-api-prologue.js +sqlite3-api.jses += $(dir.common)/whwasmutil.js +sqlite3-api.jses += $(dir.jacc)/jaccwabyt.js +sqlite3-api.jses += $(dir.api)/sqlite3-api-glue.js +sqlite3-api.jses += $(dir.api)/sqlite3-api-oo1.js +sqlite3-api.jses += $(dir.api)/sqlite3-api-worker1.js +sqlite3-api.jses += $(dir.api)/sqlite3-api-opfs.js sqlite3-api.jses += $(dir.api)/sqlite3-api-cleanup.js sqlite3-api.js := sqlite3-api.js diff --git a/ext/wasm/api/sqlite3-api-opfs.js b/ext/wasm/api/sqlite3-api-opfs.js index 3faf956c72..af1c6f7608 100644 --- a/ext/wasm/api/sqlite3-api-opfs.js +++ b/ext/wasm/api/sqlite3-api-opfs.js @@ -1,5 +1,5 @@ /* - 2022-07-22 + 2022-09-18 The author disclaims copyright to this source code. In place of a legal notice, here is a blessing: @@ -10,10 +10,30 @@ *********************************************************************** - This file contains extensions to the sqlite3 WASM API related to the - Origin-Private FileSystem (OPFS). It is intended to be appended to - the main JS deliverable somewhere after sqlite3-api-glue.js and - before sqlite3-api-cleanup.js. + This file holds the synchronous half of an sqlite3_vfs + implementation which proxies, in a synchronous fashion, the + asynchronous Origin-Private FileSystem (OPFS) APIs using a second + Worker, implemented in sqlite3-opfs-async-proxy.js. This file is + intended to be appended to the main sqlite3 JS deliverable somewhere + after sqlite3-api-glue.js and before sqlite3-api-cleanup.js. + +*/ + +'use strict'; +self.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ +/** + sqlite3.installOpfsVfs() returns a Promise which, on success, installs + an sqlite3_vfs named "opfs", suitable for use with all sqlite3 APIs + which accept a VFS. It uses the Origin-Private FileSystem API for + all file storage. On error it is rejected with an exception + explaining the problem. Reasons for rejection include, but are + not limited to: + + - The counterpart Worker (see below) could not be loaded. + + - The environment does not support OPFS. That includes when + this function is called from the main window thread. + Significant notes and limitations: @@ -21,375 +41,587 @@ available in bleeding-edge versions of Chrome (v102+, noting that that number will increase as the OPFS API matures). - - The _synchronous_ family of OPFS features (which is what this API - requires) are only available in non-shared Worker threads. This - file tries to detect that case and becomes a no-op if those - features do not seem to be available. -*/ + - The OPFS features used here are only available in dedicated Worker + threads. This file tries to detect that case and becomes a no-op + if those features do not seem to be available. -// FileSystemHandle -// FileSystemDirectoryHandle -// FileSystemFileHandle -// FileSystemFileHandle.prototype.createSyncAccessHandle -self.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ - const warn = console.warn.bind(console), - error = console.error.bind(console); - if(self.window===self || !self.importScripts || !self.FileSystemFileHandle - || !self.FileSystemFileHandle.prototype.createSyncAccessHandle){ - warn("OPFS is not available in this environment."); - return; - }else if(!navigator.storage.getDirectory){ - warn("The OPFS VFS requires navigator.storage.getDirectory."); - }else if(!sqlite3.capi.wasm.bigIntEnabled){ - error("OPFS requires BigInt support but sqlite3.capi.wasm.bigIntEnabled is false."); - return; - } - //warn('self.FileSystemFileHandle =',self.FileSystemFileHandle); - //warn('self.FileSystemFileHandle.prototype =',self.FileSystemFileHandle.prototype); - const toss = (...args)=>{throw new Error(args.join(' '))}; - const capi = sqlite3.capi, - wasm = capi.wasm; - const sqlite3_vfs = capi.sqlite3_vfs - || toss("Missing sqlite3.capi.sqlite3_vfs object."); - const sqlite3_file = capi.sqlite3_file - || toss("Missing sqlite3.capi.sqlite3_file object."); - const sqlite3_io_methods = capi.sqlite3_io_methods - || toss("Missing sqlite3.capi.sqlite3_io_methods object."); - const StructBinder = sqlite3.StructBinder || toss("Missing sqlite3.StructBinder."); - const debug = console.debug.bind(console), - log = console.log.bind(console); - warn("UNDER CONSTRUCTION: setting up OPFS VFS..."); + - It requires the SharedArrayBuffer and Atomics classes, and the + former is only available if the HTTP server emits the so-called + COOP and COEP response headers. These features are required for + proxying OPFS's synchronous API via the synchronous interface + required by the sqlite3_vfs API. - const pDVfs = capi.sqlite3_vfs_find(null)/*pointer to default VFS*/; - const dVfs = pDVfs - ? new sqlite3_vfs(pDVfs) - : null /* dVfs will be null when sqlite3 is built with - SQLITE_OS_OTHER. Though we cannot currently handle - that case, the hope is to eventually be able to. */; - const oVfs = new sqlite3_vfs(); - const oIom = new sqlite3_io_methods(); - oVfs.$iVersion = 2/*yes, two*/; - oVfs.$szOsFile = capi.sqlite3_file.structInfo.sizeof; - oVfs.$mxPathname = 1024/*sure, why not?*/; - oVfs.$zName = wasm.allocCString("opfs"); - oVfs.ondispose = [ - '$zName', oVfs.$zName, - 'cleanup dVfs', ()=>(dVfs ? dVfs.dispose() : null) - ]; - if(dVfs){ - oVfs.$xSleep = dVfs.$xSleep; - oVfs.$xRandomness = dVfs.$xRandomness; - } - // All C-side memory of oVfs is zeroed out, but just to be explicit: - oVfs.$xDlOpen = oVfs.$xDlError = oVfs.$xDlSym = oVfs.$xDlClose = null; + - This function may only be called a single time and it must be + called from the client, as opposed to the library initialization, + in case the client requires a custom path for this API's + "counterpart": this function's argument is the relative URI to + this module's "asynchronous half". When called, this function removes + itself from the sqlite3 object. - /** - Pedantic sidebar about oVfs.ondispose: the entries in that array - are items to clean up when oVfs.dispose() is called, but in this - environment it will never be called. The VFS instance simply - hangs around until the WASM module instance is cleaned up. We - "could" _hypothetically_ clean it up by "importing" an - sqlite3_os_end() impl into the wasm build, but the shutdown order - of the wasm engine and the JS one are undefined so there is no - guaranty that the oVfs instance would be available in one - environment or the other when sqlite3_os_end() is called (_if_ it - gets called at all in a wasm build, which is undefined). - */ + The argument may optionally be a plain object with the following + configuration options: - /** - Installs a StructBinder-bound function pointer member of the - given name and function in the given StructType target object. - It creates a WASM proxy for the given function and arranges for - that proxy to be cleaned up when tgt.dispose() is called. Throws - on the slightest hint of error (e.g. tgt is-not-a StructType, - name does not map to a struct-bound member, etc.). + - proxyUri: as described above - Returns a proxy for this function which is bound to tgt and takes - 2 args (name,func). That function returns the same thing, - permitting calls to be chained. + - verbose (=2): an integer 0-3. 0 disables all logging, 1 enables + logging of errors. 2 enables logging of warnings and errors. 3 + additionally enables debugging info. - If called with only 1 arg, it has no side effects but returns a - func with the same signature as described above. - */ - const installMethod = function callee(tgt, name, func){ - if(!(tgt instanceof StructBinder.StructType)){ - toss("Usage error: target object is-not-a StructType."); - } - if(1===arguments.length){ - return (n,f)=>callee(tgt,n,f); - } - if(!callee.argcProxy){ - callee.argcProxy = function(func,sig){ - return function(...args){ - if(func.length!==arguments.length){ - toss("Argument mismatch. Native signature is:",sig); - } - return func.apply(this, args); - } - }; - callee.removeFuncList = function(){ - if(this.ondispose.__removeFuncList){ - this.ondispose.__removeFuncList.forEach( - (v,ndx)=>{ - if('number'===typeof v){ - try{wasm.uninstallFunction(v)} - catch(e){/*ignore*/} - } - /* else it's a descriptive label for the next number in - the list. */ - } - ); - delete this.ondispose.__removeFuncList; - } - }; - }/*static init*/ - const sigN = tgt.memberSignature(name); - if(sigN.length<2){ - toss("Member",name," is not a function pointer. Signature =",sigN); - } - const memKey = tgt.memberKey(name); - //log("installMethod",tgt, name, sigN); - const fProxy = 1 - // We can remove this proxy middle-man once the VFS is working - ? callee.argcProxy(func, sigN) - : func; - const pFunc = wasm.installFunction(fProxy, tgt.memberSignature(name, true)); - tgt[memKey] = pFunc; - if(!tgt.ondispose) tgt.ondispose = []; - if(!tgt.ondispose.__removeFuncList){ - tgt.ondispose.push('ondispose.__removeFuncList handler', - callee.removeFuncList); - tgt.ondispose.__removeFuncList = []; - } - tgt.ondispose.__removeFuncList.push(memKey, pFunc); - return (n,f)=>callee(tgt, n, f); - }/*installMethod*/; - - /** - Map of sqlite3_file pointers to OPFS handles. - */ - const __opfsHandles = Object.create(null); + - sanityChecks (=false): if true, some basic sanity tests are + run on the OPFS VFS API after it's initialized, before the + returned Promise resolves. - const randomFilename = function f(len=16){ - if(!f._chars){ - f._chars = "abcdefghijklmnopqrstuvwxyz"+ - "ABCDEFGHIJKLMNOPQRSTUVWXYZ"+ - "012346789"; - f._n = f._chars.length; - } - const a = []; - let i = 0; - for( ; i < len; ++i){ - const ndx = Math.random() * (f._n * 64) % f._n | 0; - a[i] = f._chars[ndx]; - } - return a.join(''); + On success, the Promise resolves to the top-most sqlite3 namespace + object. +*/ +sqlite3.installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri){ + const options = (asyncProxyUri && 'object'===asyncProxyUri) ? asyncProxyUri : { + proxyUri: asyncProxyUri }; - - //const rootDir = await navigator.storage.getDirectory(); - - //////////////////////////////////////////////////////////////////////// - // Set up OPFS VFS methods... - let inst = installMethod(oVfs); - inst('xOpen', function(pVfs, zName, pFile, flags, pOutFlags){ - const f = new sqlite3_file(pFile); - f.$pMethods = oIom.pointer; - __opfsHandles[pFile] = f; - f.opfsHandle = null /* TODO */; - if(flags & capi.SQLITE_OPEN_DELETEONCLOSE){ - f.deleteOnClose = true; - } - f.filename = zName ? wasm.cstringToJs(zName) : randomFilename(); - error("OPFS sqlite3_vfs::xOpen is not yet full implemented."); - return capi.SQLITE_IOERR; - }) - ('xFullPathname', function(pVfs,zName,nOut,pOut){ - /* Until/unless we have some notion of "current dir" - in OPFS, simply copy zName to pOut... */ - const i = wasm.cstrncpy(pOut, zName, nOut); - return i{ + if(options.verbose>1) console.warn(logPrefix,...args); + }; + if(self.window===self || + !self.SharedArrayBuffer || + !self.FileSystemHandle || + !self.FileSystemDirectoryHandle || + !self.FileSystemFileHandle || + !self.FileSystemFileHandle.prototype.createSyncAccessHandle || + !navigator.storage.getDirectory){ + warn("This environment does not have OPFS support."); + promiseReject(new Error("This environment does not have OPFS support.")); + return; + } + warn("The OPFS VFS feature is very much experimental and under construction."); + const toss = function(...args){throw new Error(args.join(' '))}; + const log = (...args)=>{ + if(options.verbose>2) console.log(logPrefix,...args); + }; + const error = (...args)=>{ + if(options.verbose>0) console.error(logPrefix,...args); + }; + const capi = sqlite3.capi; + const wasm = capi.wasm; + const sqlite3_vfs = capi.sqlite3_vfs; + const sqlite3_file = capi.sqlite3_file; + const sqlite3_io_methods = capi.sqlite3_io_methods; + const StructBinder = sqlite3.StructBinder; + const W = new Worker(options.proxyUri); + const workerOrigOnError = W.onrror; + W.onerror = function(err){ + promiseReject(new Error("Loading OPFS async Worker failed for unknown reasons.")); + }; + const wMsg = (type,payload)=>W.postMessage({type,payload}); + + /** + State which we send to the async-api Worker or share with it. + This object must initially contain only cloneable or sharable + objects. After the worker's "inited" message arrives, other types + of data may be added to it. + */ + const state = Object.create(null); + state.verbose = options.verbose; + state.fileBufferSize = 1024 * 64 + 8 /* size of fileHandle.sab. 64k = max sqlite3 page size */; + state.fbInt64Offset = state.fileBufferSize - 8 /*spot in fileHandle.sab to store an int64*/; + state.opIds = Object.create(null); + { let i = 0; - for(; i < nOut; ++i) heap[pOut + i] = (Math.random()*255000) & 0xFF; - return i; + state.opIds.xAccess = i++; + state.opIds.xClose = i++; + state.opIds.xDelete = i++; + state.opIds.xFileSize = i++; + state.opIds.xOpen = i++; + state.opIds.xRead = i++; + state.opIds.xSleep = i++; + state.opIds.xSync = i++; + state.opIds.xTruncate = i++; + state.opIds.xWrite = i++; + state.opSAB = new SharedArrayBuffer(i * 4/*sizeof int32*/); + /* The approach of using a single SAB to serialize comms for all + instances may(?) lead to deadlock situations in multi-db + cases. We should probably have one SAB here with a single slot + for locking a per-file initialization step and then allocate a + separate SAB like the above one for each file. That will + require a bit of acrobatics but should be feasible. + */ + } + + state.sq3Codes = Object.create(null); + state.sq3Codes._reverse = Object.create(null); + [ // SQLITE_xxx constants to export to the async worker counterpart... + 'SQLITE_ERROR', 'SQLITE_IOERR', + 'SQLITE_NOTFOUND', 'SQLITE_MISUSE', + 'SQLITE_IOERR_READ', 'SQLITE_IOERR_SHORT_READ', + 'SQLITE_IOERR_WRITE', 'SQLITE_IOERR_FSYNC', + 'SQLITE_IOERR_TRUNCATE', 'SQLITE_IOERR_DELETE', + 'SQLITE_IOERR_ACCESS', 'SQLITE_IOERR_CLOSE' + ].forEach(function(k){ + state.sq3Codes[k] = capi[k] || toss("Maintenance required: not found:",k); + state.sq3Codes._reverse[capi[k]] = k; }); - } - //////////////////////////////////////////////////////////////////////// - // Set up OPFS sqlite3_io_methods... - inst = installMethod(oIom); - inst('xClose', async function(pFile){ - warn("xClose(",arguments,") uses await"); - const f = __opfsHandles[pFile]; - delete __opfsHandles[pFile]; - if(f.opfsHandle){ - await f.opfsHandle.close(); - if(f.deleteOnClose){ - // TODO + const isWorkerErrCode = (n)=>!!state.sq3Codes._reverse[n]; + const opStore = (op,val=-1)=>Atomics.store(state.opSABView, state.opIds[op], val); + const opWait = (op,val=-1)=>Atomics.wait(state.opSABView, state.opIds[op], val); + + /** + Runs the given operation in the async worker counterpart, waits + for its response, and returns the result which the async worker + writes to the given op's index in state.opSABView. The 2nd argument + must be a single object or primitive value, depending on the + given operation's signature in the async API counterpart. + */ + const opRun = (op,args)=>{ + opStore(op); + wMsg(op, args); + opWait(op); + return Atomics.load(state.opSABView, state.opIds[op]); + }; + + /** + Generates a random ASCII string len characters long, intended for + use as a temporary file name. + */ + const randomFilename = function f(len=16){ + if(!f._chars){ + f._chars = "abcdefghijklmnopqrstuvwxyz"+ + "ABCDEFGHIJKLMNOPQRSTUVWXYZ"+ + "012346789"; + f._n = f._chars.length; } - } - f.dispose(); - return 0; - }) - ('xRead', /*i(ppij)*/function(pFile,pDest,n,offset){ - /* int (*xRead)(sqlite3_file*, void*, int iAmt, sqlite3_int64 iOfst) */ - try { - const f = __opfsHandles[pFile]; - const heap = wasm.heap8u(); - const b = new Uint8Array(heap.buffer, pDest, n); - const nRead = f.opfsHandle.read(b, {at: offset}); - if(nRead(dVfs ? dVfs.dispose() : null), + 'cleanup opfsIoMethods', ()=>opfsIoMethods.dispose() + ]; + if(dVfs){ + opfsVfs.$xSleep = dVfs.$xSleep; + opfsVfs.$xRandomness = dVfs.$xRandomness; } - }) - ('xWrite', /*i(ppij)*/function(pFile,pSrc,n,offset){ - /* int (*xWrite)(sqlite3_file*, const void*, int iAmt, sqlite3_int64 iOfst) */ - try { - const f = __opfsHandles[pFile]; - const b = new Uint8Array(wasm.heap8u().buffer, pSrc, n); - const nOut = f.opfsHandle.write(b, {at: offset}); - if(nOutcallee(tgt,n,f); + } + if(!callee.argcProxy){ + callee.argcProxy = function(func,sig){ + return function(...args){ + if(func.length!==arguments.length){ + toss("Argument mismatch. Native signature is:",sig); + } + return func.apply(this, args); + } + }; + callee.removeFuncList = function(){ + if(this.ondispose.__removeFuncList){ + this.ondispose.__removeFuncList.forEach( + (v,ndx)=>{ + if('number'===typeof v){ + try{wasm.uninstallFunction(v)} + catch(e){/*ignore*/} + } + /* else it's a descriptive label for the next number in + the list. */ + } + ); + delete this.ondispose.__removeFuncList; + } + }; + }/*static init*/ + const sigN = tgt.memberSignature(name); + if(sigN.length<2){ + toss("Member",name," is not a function pointer. Signature =",sigN); + } + const memKey = tgt.memberKey(name); + //log("installMethod",tgt, name, sigN); + const fProxy = 1 + // We can remove this proxy middle-man once the VFS is working + ? callee.argcProxy(func, sigN) + : func; + const pFunc = wasm.installFunction(fProxy, tgt.memberSignature(name, true)); + tgt[memKey] = pFunc; + if(!tgt.ondispose) tgt.ondispose = []; + if(!tgt.ondispose.__removeFuncList){ + tgt.ondispose.push('ondispose.__removeFuncList handler', + callee.removeFuncList); + tgt.ondispose.__removeFuncList = []; + } + tgt.ondispose.__removeFuncList.push(memKey, pFunc); + return (n,f)=>callee(tgt, n, f); + }/*installMethod*/; + + /** + Impls for the sqlite3_io_methods methods. Maintenance reminder: + members are in alphabetical order to simplify finding them. + */ + const ioSyncWrappers = { + xCheckReservedLock: function(pFile,pOut){ + // Exclusive lock is automatically acquired when opened + //warn("xCheckReservedLock(",arguments,") is a no-op"); + wasm.setMemValue(pOut,1,'i32'); + return 0; + }, + xClose: function(pFile){ + let rc = 0; + const f = __openFiles[pFile]; + if(f){ + delete __openFiles[pFile]; + rc = opRun('xClose', pFile); + if(f.sq3File) f.sq3File.dispose(); + } + return rc; + }, + xDeviceCharacteristics: function(pFile){ + //debug("xDeviceCharacteristics(",pFile,")"); + return capi.SQLITE_IOCAP_UNDELETABLE_WHEN_OPEN; + }, + xFileControl: function(pFile,op,pArg){ + //debug("xFileControl(",arguments,") is a no-op"); + return capi.SQLITE_NOTFOUND; + }, + xFileSize: function(pFile,pSz64){ + const rc = opRun('xFileSize', pFile); + if(!isWorkerErrCode(rc)){ + const f = __openFiles[pFile]; + wasm.setMemValue(pSz64, f.sabViewFileSize.getBigInt64(0) ,'i64'); + } + return rc; + }, + xLock: function(pFile,lockType){ + //2022-09: OPFS handles lock when opened + //warn("xLock(",arguments,") is a no-op"); + return 0; + }, + xRead: function(pFile,pDest,n,offset){ + /* int (*xRead)(sqlite3_file*, void*, int iAmt, sqlite3_int64 iOfst) */ + const f = __openFiles[pFile]; + let rc; + try { + // FIXME(?): block until we finish copying the xRead result buffer. How? + rc = opRun('xRead',{fid:pFile, n, offset}); + if(0!==rc) return rc; + let i = 0; + for(; i < n; ++i) wasm.setMemValue(pDest + i, f.sabView[i]); + }catch(e){ + error("xRead(",arguments,") failed:",e,f); + rc = capi.SQLITE_IOERR_READ; + } + return rc; + }, + xSync: function(pFile,flags){ + return opRun('xSync', {fid:pFile, flags}); + }, + xTruncate: function(pFile,sz64){ + return opRun('xTruncate', {fid:pFile, size: sz64}); + }, + xUnlock: function(pFile,lockType){ + //2022-09: OPFS handles lock when opened + //warn("xUnlock(",arguments,") is a no-op"); + return 0; + }, + xWrite: function(pFile,pSrc,n,offset){ + /* int (*xWrite)(sqlite3_file*, const void*, int iAmt, sqlite3_int64 iOfst) */ + const f = __openFiles[pFile]; + try { + let i = 0; + // FIXME(?): block from here until we finish the xWrite. How? + for(; i < n; ++i) f.sabView[i] = wasm.getMemValue(pSrc+i); + return opRun('xWrite',{fid:pFile, n, offset}); + }catch(e){ + error("xWrite(",arguments,") failed:",e,f); + return capi.SQLITE_IOERR_WRITE; + } + } + }/*ioSyncWrappers*/; + + /** + Impls for the sqlite3_vfs methods. Maintenance reminder: members + are in alphabetical order to simplify finding them. + */ + const vfsSyncWrappers = { + xAccess: function(pVfs,zName,flags,pOut){ + const rc = opRun('xAccess', wasm.cstringToJs(zName)); + wasm.setMemValue(pOut, rc ? 0 : 1, 'i32'); + return 0; + }, + xCurrentTime: function(pVfs,pOut){ + /* If it turns out that we need to adjust for timezone, see: + https://stackoverflow.com/a/11760121/1458521 */ + wasm.setMemValue(pOut, 2440587.5 + (new Date().getTime()/86400000), + 'double'); + return 0; + }, + xCurrentTimeInt64: function(pVfs,pOut){ + // TODO: confirm that this calculation is correct + wasm.setMemValue(pOut, (2440587.5 * 86400000) + new Date().getTime(), + 'i64'); + return 0; + }, + xDelete: function(pVfs, zName, doSyncDir){ + return opRun('xDelete', {filename: wasm.cstringToJs(zName), syncDir: doSyncDir}); + }, + xFullPathname: function(pVfs,zName,nOut,pOut){ + /* Until/unless we have some notion of "current dir" + in OPFS, simply copy zName to pOut... */ + const i = wasm.cstrncpy(pOut, zName, nOut); + return ipMethods is NULL. */ + if(args.readOnly){ + wasm.setMemValue(pOutFlags, capi.SQLITE_OPEN_READONLY, 'i32'); + } + __openFiles[pFile] = args; + args.sabView = new Uint8Array(args.sab); + args.sabViewFileSize = new DataView(args.sab, state.fbInt64Offset, 8); + args.sq3File = new sqlite3_file(pFile); + args.sq3File.$pMethods = opfsIoMethods.pointer; + args.ba = new Uint8Array(args.sab); + } + return rc; + }/*xOpen()*/ + }/*vfsSyncWrappers*/; + + if(!opfsVfs.$xRandomness){ + /* If the default VFS has no xRandomness(), add a basic JS impl... */ + vfsSyncWrappers.xRandomness = function(pVfs, nOut, pOut){ + const heap = wasm.heap8u(); + let i = 0; + for(; i < nOut; ++i) heap[pOut + i] = (Math.random()*255000) & 0xFF; + return i; + }; } - }) - ('xFileSize', /*i(pp)*/async function(pFile,pSz){ - /* int (*xFileSize)(sqlite3_file*, sqlite3_int64 *pSize) */ - try { - warn("xFileSize(",arguments,") uses await"); - const f = __opfsHandles[pFile]; - const fsz = await f.opfsHandle.getSize(); - capi.wasm.setMemValue(pSz, fsz,'i64'); - return 0; - }catch(e){ - error("xFileSize(",arguments,") failed:",e); - return capi.SQLITE_IOERR_SEEK; + if(!opfsVfs.$xSleep){ + /* If we can inherit an xSleep() impl from the default VFS then + use it, otherwise install one which is certainly less accurate + because it has to go round-trip through the async worker, but + provides the only option for a synchronous sleep() in JS. */ + vfsSyncWrappers.xSleep = (pVfs,ms)=>opRun('xSleep',ms); } - }) - ('xLock', /*i(pi)*/function(pFile,lockType){ - /* int (*xLock)(sqlite3_file*, int) */ - // Opening a handle locks it automatically. - warn("xLock(",arguments,") is a no-op"); - return 0; - }) - ('xUnlock', /*i(pi)*/function(pFile,lockType){ - /* int (*xUnlock)(sqlite3_file*, int) */ - // Opening a handle locks it automatically. - warn("xUnlock(",arguments,") is a no-op"); - return 0; - }) - ('xCheckReservedLock', /*i(pp)*/function(pFile,pOut){ - /* int (*xCheckReservedLock)(sqlite3_file*, int *pResOut) */ - // Exclusive lock is automatically acquired when opened - warn("xCheckReservedLock(",arguments,") is a no-op"); - wasm.setMemValue(pOut,1,'i32'); - return 0; - }) - ('xFileControl', /*i(pip)*/function(pFile,op,pArg){ - /* int (*xFileControl)(sqlite3_file*, int op, void *pArg) */ - debug("xFileControl(",arguments,") is a no-op"); - return capi.SQLITE_NOTFOUND; - }) - ('xDeviceCharacteristics',/*i(p)*/function(pFile){ - /* int (*xDeviceCharacteristics)(sqlite3_file*) */ - debug("xDeviceCharacteristics(",pFile,")"); - return capi.SQLITE_IOCAP_UNDELETABLE_WHEN_OPEN; - }); - // xSectorSize may be NULL - //('xSectorSize', function(pFile){ - // /* int (*xSectorSize)(sqlite3_file*) */ - // log("xSectorSize(",pFile,")"); - // return 4096 /* ==> SQLITE_DEFAULT_SECTOR_SIZE */; - //}) - const rc = capi.sqlite3_vfs_register(oVfs.pointer, 0); - if(rc){ - oVfs.dispose(); - toss("sqlite3_vfs_register(OPFS) failed with rc",rc); - } - capi.sqlite3_vfs_register.addReference(oVfs, oIom); - warn("End of (very incomplete) OPFS setup.", oVfs); - //oVfs.dispose()/*only because we can't yet do anything with it*/; -}); + /* Install the vfs/io_methods into their C-level shared instances... */ + let inst = installMethod(opfsIoMethods); + for(let k of Object.keys(ioSyncWrappers)) inst(k, ioSyncWrappers[k]); + inst = installMethod(opfsVfs); + for(let k of Object.keys(vfsSyncWrappers)) inst(k, vfsSyncWrappers[k]); + + const sanityCheck = async function(){ + const scope = wasm.scopedAllocPush(); + const sq3File = new sqlite3_file(); + try{ + const fid = sq3File.pointer; + const openFlags = capi.SQLITE_OPEN_CREATE + | capi.SQLITE_OPEN_READWRITE + //| capi.SQLITE_OPEN_DELETEONCLOSE + | capi.SQLITE_OPEN_MAIN_DB; + const pOut = wasm.scopedAlloc(8); + const dbFile = "/sanity/check/file"; + const zDbFile = wasm.scopedAllocCString(dbFile); + let rc; + vfsSyncWrappers.xAccess(opfsVfs.pointer, zDbFile, 0, pOut); + rc = wasm.getMemValue(pOut,'i32'); + log("xAccess(",dbFile,") exists ?=",rc); + rc = vfsSyncWrappers.xOpen(opfsVfs.pointer, zDbFile, + fid, openFlags, pOut); + log("open rc =",rc,"state.opSABView[xOpen] =",state.opSABView[state.opIds.xOpen]); + if(isWorkerErrCode(rc)){ + error("open failed with code",rc); + return; + } + vfsSyncWrappers.xAccess(opfsVfs.pointer, zDbFile, 0, pOut); + rc = wasm.getMemValue(pOut,'i32'); + if(!rc) toss("xAccess() failed to detect file."); + rc = ioSyncWrappers.xSync(sq3File.pointer, 0); + if(rc) toss('sync failed w/ rc',rc); + rc = ioSyncWrappers.xTruncate(sq3File.pointer, 1024); + if(rc) toss('truncate failed w/ rc',rc); + wasm.setMemValue(pOut,0,'i64'); + rc = ioSyncWrappers.xFileSize(sq3File.pointer, pOut); + if(rc) toss('xFileSize failed w/ rc',rc); + log("xFileSize says:",wasm.getMemValue(pOut, 'i64')); + rc = ioSyncWrappers.xWrite(sq3File.pointer, zDbFile, 10, 1); + if(rc) toss("xWrite() failed!"); + const readBuf = wasm.scopedAlloc(16); + rc = ioSyncWrappers.xRead(sq3File.pointer, readBuf, 6, 2); + wasm.setMemValue(readBuf+6,0); + let jRead = wasm.cstringToJs(readBuf); + log("xRead() got:",jRead); + if("sanity"!==jRead) toss("Unexpected xRead() value."); + log("xSleep()ing before close()ing..."); + opRun('xSleep',1000); + rc = ioSyncWrappers.xClose(fid); + log("xClose rc =",rc,"opSABView =",state.opSABView); + log("Deleting file:",dbFile); + vfsSyncWrappers.xDelete(opfsVfs.pointer, zDbFile, 0x1234); + vfsSyncWrappers.xAccess(opfsVfs.pointer, zDbFile, 0, pOut); + rc = wasm.getMemValue(pOut,'i32'); + if(rc) toss("Expecting 0 from xAccess(",dbFile,") after xDelete()."); + }finally{ + sq3File.dispose(); + wasm.scopedAllocPop(scope); + } + }/*sanityCheck()*/; + + W.onmessage = function({data}){ + //log("Worker.onmessage:",data); + switch(data.type){ + case 'loaded': + /*Pass our config and shared state on to the async worker.*/ + wMsg('init',state); + break; + case 'inited':{ + /*Indicates that the async partner has received the 'init', + so we now know that the state object is no longer subject to + being copied by a pending postMessage() call.*/ + try { + const rc = capi.sqlite3_vfs_register(opfsVfs.pointer, opfsVfs.$zName); + if(rc){ + opfsVfs.dispose(); + toss("sqlite3_vfs_register(OPFS) failed with rc",rc); + } + if(opfsVfs.pointer !== capi.sqlite3_vfs_find("opfs")){ + toss("BUG: sqlite3_vfs_find() failed for just-installed OPFS VFS"); + } + capi.sqlite3_vfs_register.addReference(opfsVfs, opfsIoMethods); + state.opSABView = new Int32Array(state.opSAB); + if(options.sanityChecks){ + warn("Running sanity checks because of opfs-sanity-check URL arg..."); + sanityCheck(); + } + W.onerror = workerOrigOnError; + promiseResolve(sqlite3); + log("End of OPFS sqlite3_vfs setup.", opfsVfs); + }catch(e){ + error(e); + promiseReject(e); + } + break; + } + default: + promiseReject(e); + error("Unexpected message from the async worker:",data); + break; + } + }; + })/*thePromise*/; + return thePromise; +}/*installOpfsVfs()*/; +sqlite3.installOpfsVfs.defaultProxyUri = "sqlite3-opfs-async-proxy.js"; +}/*sqlite3ApiBootstrap.initializers.push()*/); diff --git a/ext/wasm/x-sync-async.js b/ext/wasm/x-sync-async.js index 203c8eece4..b25f5a2681 100644 --- a/ext/wasm/x-sync-async.js +++ b/ext/wasm/x-sync-async.js @@ -10,11 +10,7 @@ *********************************************************************** - An INCOMPLETE and UNDER CONSTRUCTION experiment for OPFS. - This file holds the synchronous half of an sqlite3_vfs - implementation which proxies, in a synchronous fashion, the - asynchronous OPFS APIs using a second Worker, implemented - in sqlite3-opfs-async-proxy.js. + A testing ground for the OPFS VFS. Summary of how this works: @@ -23,18 +19,12 @@ conventional sqlite3_vfs (except that it's implemented in JS). The methods which require OPFS APIs use a separate worker (hereafter called the OPFS worker) to access that functionality. This worker and that one - use SharedBufferArray + use SharedArrayBuffer. */ 'use strict'; -/** - This function is a placeholder for use in development. When - working, this will be moved into a file named - api/sqlite3-api-opfs.js, or similar, and hooked in to the - sqlite-api build construct. -*/ -const initOpfsVfs = function(sqlite3){ +const tryOpfsVfs = function(sqlite3){ const toss = function(...args){throw new Error(args.join(' '))}; - const logPrefix = "OPFS syncer:"; + const logPrefix = "OPFS tester:"; const log = (...args)=>{ console.log(logPrefix,...args); }; @@ -44,518 +34,20 @@ const initOpfsVfs = function(sqlite3){ const error = (...args)=>{ console.error(logPrefix,...args); }; - - if(self.window===self || - !self.SharedArrayBuffer || - !self.FileSystemHandle || - !self.FileSystemDirectoryHandle || - !self.FileSystemFileHandle || - !self.FileSystemFileHandle.prototype.createSyncAccessHandle || - !navigator.storage.getDirectory){ - warn("This environment does not have OPFS support."); - return; - } - warn("This file is very much experimental and under construction.",self.location.pathname); + log("tryOpfsVfs()"); const capi = sqlite3.capi; - const wasm = capi.wasm; - const sqlite3_vfs = capi.sqlite3_vfs - || toss("Missing sqlite3.capi.sqlite3_vfs object."); - const sqlite3_file = capi.sqlite3_file - || toss("Missing sqlite3.capi.sqlite3_file object."); - const sqlite3_io_methods = capi.sqlite3_io_methods - || toss("Missing sqlite3.capi.sqlite3_io_methods object."); - const StructBinder = sqlite3.StructBinder || toss("Missing sqlite3.StructBinder."); - const thisUrl = new URL(self.location.href); - - const W = new Worker("sqlite3-opfs-async-proxy.js"); - const wMsg = (type,payload)=>W.postMessage({type,payload}); - - /** - State which we send to the async-api Worker or share with it. - This object must initially contain only cloneable or sharable - objects. After the worker's "inited" message arrives, other types - of data may be added to it. - */ - const state = Object.create(null); - state.verbose = thisUrl.searchParams.has('opfs-verbose') ? 3 : 2; - state.fileBufferSize = 1024 * 64 + 8 /* size of fileHandle.sab. 64k = max sqlite3 page size */; - state.fbInt64Offset = state.fileBufferSize - 8 /*spot in fileHandle.sab to store an int64*/; - state.opIds = Object.create(null); - { - let i = 0; - state.opIds.xAccess = i++; - state.opIds.xClose = i++; - state.opIds.xDelete = i++; - state.opIds.xFileSize = i++; - state.opIds.xOpen = i++; - state.opIds.xRead = i++; - state.opIds.xSleep = i++; - state.opIds.xSync = i++; - state.opIds.xTruncate = i++; - state.opIds.xWrite = i++; - state.opSAB = new SharedArrayBuffer(i * 4/*sizeof int32*/); - } - - state.sq3Codes = Object.create(null); - state.sq3Codes._reverse = Object.create(null); - [ // SQLITE_xxx constants to export to the async worker counterpart... - 'SQLITE_ERROR', 'SQLITE_IOERR', - 'SQLITE_NOTFOUND', 'SQLITE_MISUSE', - 'SQLITE_IOERR_READ', 'SQLITE_IOERR_SHORT_READ', - 'SQLITE_IOERR_WRITE', 'SQLITE_IOERR_FSYNC', - 'SQLITE_IOERR_TRUNCATE', 'SQLITE_IOERR_DELETE', - 'SQLITE_IOERR_ACCESS', 'SQLITE_IOERR_CLOSE' - ].forEach(function(k){ - state.sq3Codes[k] = capi[k] || toss("Maintenance required: not found:",k); - state.sq3Codes._reverse[capi[k]] = k; - }); - - const isWorkerErrCode = (n)=>!!state.sq3Codes._reverse[n]; - const opStore = (op,val=-1)=>Atomics.store(state.opSABView, state.opIds[op], val); - const opWait = (op,val=-1)=>Atomics.wait(state.opSABView, state.opIds[op], val); - - /** - Runs the given operation in the async worker counterpart, waits - for its response, and returns the result which the async worker - writes to the given op's index in state.opSABView. The 2nd argument - must be a single object or primitive value, depending on the - given operation's signature in the async API counterpart. - */ - const opRun = (op,args)=>{ - opStore(op); - wMsg(op, args); - opWait(op); - return Atomics.load(state.opSABView, state.opIds[op]); - }; - - /** - Generates a random ASCII string len characters long, intended for - use as a temporary file name. - */ - const randomFilename = function f(len=16){ - if(!f._chars){ - f._chars = "abcdefghijklmnopqrstuvwxyz"+ - "ABCDEFGHIJKLMNOPQRSTUVWXYZ"+ - "012346789"; - f._n = f._chars.length; - } - const a = []; - let i = 0; - for( ; i < len; ++i){ - const ndx = Math.random() * (f._n * 64) % f._n | 0; - a[i] = f._chars[ndx]; - } - return a.join(''); - }; - - /** - Map of sqlite3_file pointers to objects constructed by xOpen(). - */ - const __openFiles = Object.create(null); - - const pDVfs = capi.sqlite3_vfs_find(null)/*pointer to default VFS*/; - const dVfs = pDVfs - ? new sqlite3_vfs(pDVfs) - : null /* dVfs will be null when sqlite3 is built with - SQLITE_OS_OTHER. Though we cannot currently handle - that case, the hope is to eventually be able to. */; - const opfsVfs = new sqlite3_vfs(); - const opfsIoMethods = new sqlite3_io_methods(); - opfsVfs.$iVersion = 2/*yes, two*/; - opfsVfs.$szOsFile = capi.sqlite3_file.structInfo.sizeof; - opfsVfs.$mxPathname = 1024/*sure, why not?*/; - opfsVfs.$zName = wasm.allocCString("opfs"); - // All C-side memory of opfsVfs is zeroed out, but just to be explicit: - opfsVfs.$xDlOpen = opfsVfs.$xDlError = opfsVfs.$xDlSym = opfsVfs.$xDlClose = null; - opfsVfs.ondispose = [ - '$zName', opfsVfs.$zName, - 'cleanup default VFS wrapper', ()=>(dVfs ? dVfs.dispose() : null), - 'cleanup opfsIoMethods', ()=>opfsIoMethods.dispose() - ]; - if(dVfs){ - opfsVfs.$xSleep = dVfs.$xSleep; - opfsVfs.$xRandomness = dVfs.$xRandomness; - } - /** - Pedantic sidebar about opfsVfs.ondispose: the entries in that array - are items to clean up when opfsVfs.dispose() is called, but in this - environment it will never be called. The VFS instance simply - hangs around until the WASM module instance is cleaned up. We - "could" _hypothetically_ clean it up by "importing" an - sqlite3_os_end() impl into the wasm build, but the shutdown order - of the wasm engine and the JS one are undefined so there is no - guaranty that the opfsVfs instance would be available in one - environment or the other when sqlite3_os_end() is called (_if_ it - gets called at all in a wasm build, which is undefined). - */ - - /** - Installs a StructBinder-bound function pointer member of the - given name and function in the given StructType target object. - It creates a WASM proxy for the given function and arranges for - that proxy to be cleaned up when tgt.dispose() is called. Throws - on the slightest hint of error (e.g. tgt is-not-a StructType, - name does not map to a struct-bound member, etc.). - - Returns a proxy for this function which is bound to tgt and takes - 2 args (name,func). That function returns the same thing, - permitting calls to be chained. - - If called with only 1 arg, it has no side effects but returns a - func with the same signature as described above. - */ - const installMethod = function callee(tgt, name, func){ - if(!(tgt instanceof StructBinder.StructType)){ - toss("Usage error: target object is-not-a StructType."); - } - if(1===arguments.length){ - return (n,f)=>callee(tgt,n,f); - } - if(!callee.argcProxy){ - callee.argcProxy = function(func,sig){ - return function(...args){ - if(func.length!==arguments.length){ - toss("Argument mismatch. Native signature is:",sig); - } - return func.apply(this, args); - } - }; - callee.removeFuncList = function(){ - if(this.ondispose.__removeFuncList){ - this.ondispose.__removeFuncList.forEach( - (v,ndx)=>{ - if('number'===typeof v){ - try{wasm.uninstallFunction(v)} - catch(e){/*ignore*/} - } - /* else it's a descriptive label for the next number in - the list. */ - } - ); - delete this.ondispose.__removeFuncList; - } - }; - }/*static init*/ - const sigN = tgt.memberSignature(name); - if(sigN.length<2){ - toss("Member",name," is not a function pointer. Signature =",sigN); - } - const memKey = tgt.memberKey(name); - //log("installMethod",tgt, name, sigN); - const fProxy = 1 - // We can remove this proxy middle-man once the VFS is working - ? callee.argcProxy(func, sigN) - : func; - const pFunc = wasm.installFunction(fProxy, tgt.memberSignature(name, true)); - tgt[memKey] = pFunc; - if(!tgt.ondispose) tgt.ondispose = []; - if(!tgt.ondispose.__removeFuncList){ - tgt.ondispose.push('ondispose.__removeFuncList handler', - callee.removeFuncList); - tgt.ondispose.__removeFuncList = []; - } - tgt.ondispose.__removeFuncList.push(memKey, pFunc); - return (n,f)=>callee(tgt, n, f); - }/*installMethod*/; - - /** - Impls for the sqlite3_io_methods methods. Maintenance reminder: - members are in alphabetical order to simplify finding them. - */ - const ioSyncWrappers = { - xCheckReservedLock: function(pFile,pOut){ - // Exclusive lock is automatically acquired when opened - //warn("xCheckReservedLock(",arguments,") is a no-op"); - wasm.setMemValue(pOut,1,'i32'); - return 0; - }, - xClose: function(pFile){ - let rc = 0; - const f = __openFiles[pFile]; - if(f){ - delete __openFiles[pFile]; - rc = opRun('xClose', pFile); - if(f.sq3File) f.sq3File.dispose(); - } - return rc; - }, - xDeviceCharacteristics: function(pFile){ - //debug("xDeviceCharacteristics(",pFile,")"); - return capi.SQLITE_IOCAP_UNDELETABLE_WHEN_OPEN; - }, - xFileControl: function(pFile,op,pArg){ - //debug("xFileControl(",arguments,") is a no-op"); - return capi.SQLITE_NOTFOUND; - }, - xFileSize: function(pFile,pSz64){ - const rc = opRun('xFileSize', pFile); - if(!isWorkerErrCode(rc)){ - const f = __openFiles[pFile]; - wasm.setMemValue(pSz64, f.sabViewFileSize.getBigInt64(0) ,'i64'); - } - return rc; - }, - xLock: function(pFile,lockType){ - //2022-09: OPFS handles lock when opened - //warn("xLock(",arguments,") is a no-op"); - return 0; - }, - xRead: function(pFile,pDest,n,offset){ - /* int (*xRead)(sqlite3_file*, void*, int iAmt, sqlite3_int64 iOfst) */ - const f = __openFiles[pFile]; - let rc; - try { - // FIXME(?): block until we finish copying the xRead result buffer. How? - rc = opRun('xRead',{fid:pFile, n, offset}); - if(0!==rc) return rc; - let i = 0; - for(; i < n; ++i) wasm.setMemValue(pDest + i, f.sabView[i]); - }catch(e){ - error("xRead(",arguments,") failed:",e,f); - rc = capi.SQLITE_IOERR_READ; - } - return rc; - }, - xSync: function(pFile,flags){ - return opRun('xSync', {fid:pFile, flags}); - }, - xTruncate: function(pFile,sz64){ - return opRun('xTruncate', {fid:pFile, size: sz64}); - }, - xUnlock: function(pFile,lockType){ - //2022-09: OPFS handles lock when opened - //warn("xUnlock(",arguments,") is a no-op"); - return 0; - }, - xWrite: function(pFile,pSrc,n,offset){ - /* int (*xWrite)(sqlite3_file*, const void*, int iAmt, sqlite3_int64 iOfst) */ - const f = __openFiles[pFile]; - try { - let i = 0; - // FIXME(?): block from here until we finish the xWrite. How? - for(; i < n; ++i) f.sabView[i] = wasm.getMemValue(pSrc+i); - return opRun('xWrite',{fid:pFile, n, offset}); - }catch(e){ - error("xWrite(",arguments,") failed:",e,f); - return capi.SQLITE_IOERR_WRITE; - } - } - }/*ioSyncWrappers*/; - - /** - Impls for the sqlite3_vfs methods. Maintenance reminder: members - are in alphabetical order to simplify finding them. - */ - const vfsSyncWrappers = { - xAccess: function(pVfs,zName,flags,pOut){ - const rc = opRun('xAccess', wasm.cstringToJs(zName)); - wasm.setMemValue(pOut, rc ? 0 : 1, 'i32'); - return 0; - }, - xCurrentTime: function(pVfs,pOut){ - /* If it turns out that we need to adjust for timezone, see: - https://stackoverflow.com/a/11760121/1458521 */ - wasm.setMemValue(pOut, 2440587.5 + (new Date().getTime()/86400000), - 'double'); - return 0; - }, - xCurrentTimeInt64: function(pVfs,pOut){ - // TODO: confirm that this calculation is correct - wasm.setMemValue(pOut, (2440587.5 * 86400000) + new Date().getTime(), - 'i64'); - return 0; - }, - xDelete: function(pVfs, zName, doSyncDir){ - return opRun('xDelete', {filename: wasm.cstringToJs(zName), syncDir: doSyncDir}); - }, - xFullPathname: function(pVfs,zName,nOut,pOut){ - /* Until/unless we have some notion of "current dir" - in OPFS, simply copy zName to pOut... */ - const i = wasm.cstrncpy(pOut, zName, nOut); - return ipMethods is NULL. */ - if(args.readOnly){ - wasm.setMemValue(pOutFlags, capi.SQLITE_OPEN_READONLY, 'i32'); - } - __openFiles[pFile] = args; - args.sabView = new Uint8Array(args.sab); - args.sabViewFileSize = new DataView(args.sab, state.fbInt64Offset, 8); - args.sq3File = new sqlite3_file(pFile); - args.sq3File.$pMethods = opfsIoMethods.pointer; - args.ba = new Uint8Array(args.sab); - } - return rc; - }/*xOpen()*/ - }/*vfsSyncWrappers*/; - - if(!opfsVfs.$xRandomness){ - /* If the default VFS has no xRandomness(), add a basic JS impl... */ - vfsSyncWrappers.xRandomness = function(pVfs, nOut, pOut){ - const heap = wasm.heap8u(); - let i = 0; - for(; i < nOut; ++i) heap[pOut + i] = (Math.random()*255000) & 0xFF; - return i; - }; - } - if(!opfsVfs.$xSleep){ - /* If we can inherit an xSleep() impl from the default VFS then - use it, otherwise install one which is certainly less accurate - because it has to go round-trip through the async worker, but - provides the only option for a synchronous sleep() in JS. */ - vfsSyncWrappers.xSleep = (pVfs,ms)=>opRun('xSleep',ms); - } - - /* Install the vfs/io_methods into their C-level shared instances... */ - let inst = installMethod(opfsIoMethods); - for(let k of Object.keys(ioSyncWrappers)) inst(k, ioSyncWrappers[k]); - inst = installMethod(opfsVfs); - for(let k of Object.keys(vfsSyncWrappers)) inst(k, vfsSyncWrappers[k]); - - const sanityCheck = async function(){ - //state.ioBuf = new Uint8Array(state.sabIo); - const scope = wasm.scopedAllocPush(); - const sq3File = new sqlite3_file(); - try{ - const fid = sq3File.pointer; - const openFlags = capi.SQLITE_OPEN_CREATE - | capi.SQLITE_OPEN_READWRITE - //| capi.SQLITE_OPEN_DELETEONCLOSE - | capi.SQLITE_OPEN_MAIN_DB; - const pOut = wasm.scopedAlloc(8); - const dbFile = "/sanity/check/file"; - const zDbFile = wasm.scopedAllocCString(dbFile); - let rc; - vfsSyncWrappers.xAccess(opfsVfs.pointer, zDbFile, 0, pOut); - rc = wasm.getMemValue(pOut,'i32'); - log("xAccess(",dbFile,") exists ?=",rc); - rc = vfsSyncWrappers.xOpen(opfsVfs.pointer, zDbFile, - fid, openFlags, pOut); - log("open rc =",rc,"state.opSABView[xOpen] =",state.opSABView[state.opIds.xOpen]); - if(isWorkerErrCode(rc)){ - error("open failed with code",rc); - return; - } - vfsSyncWrappers.xAccess(opfsVfs.pointer, zDbFile, 0, pOut); - rc = wasm.getMemValue(pOut,'i32'); - if(!rc) toss("xAccess() failed to detect file."); - rc = ioSyncWrappers.xSync(sq3File.pointer, 0); - if(rc) toss('sync failed w/ rc',rc); - rc = ioSyncWrappers.xTruncate(sq3File.pointer, 1024); - if(rc) toss('truncate failed w/ rc',rc); - wasm.setMemValue(pOut,0,'i64'); - rc = ioSyncWrappers.xFileSize(sq3File.pointer, pOut); - if(rc) toss('xFileSize failed w/ rc',rc); - log("xFileSize says:",wasm.getMemValue(pOut, 'i64')); - rc = ioSyncWrappers.xWrite(sq3File.pointer, zDbFile, 10, 1); - if(rc) toss("xWrite() failed!"); - const readBuf = wasm.scopedAlloc(16); - rc = ioSyncWrappers.xRead(sq3File.pointer, readBuf, 6, 2); - wasm.setMemValue(readBuf+6,0); - let jRead = wasm.cstringToJs(readBuf); - log("xRead() got:",jRead); - if("sanity"!==jRead) toss("Unexpected xRead() value."); - log("xSleep()ing before close()ing..."); - opRun('xSleep',1000); - rc = ioSyncWrappers.xClose(fid); - log("xClose rc =",rc,"opSABView =",state.opSABView); - log("Deleting file:",dbFile); - vfsSyncWrappers.xDelete(opfsVfs.pointer, zDbFile, 0x1234); - vfsSyncWrappers.xAccess(opfsVfs.pointer, zDbFile, 0, pOut); - rc = wasm.getMemValue(pOut,'i32'); - if(rc) toss("Expecting 0 from xAccess(",dbFile,") after xDelete()."); - }finally{ - sq3File.dispose(); - wasm.scopedAllocPop(scope); - } - }; - - W.onmessage = function({data}){ - //log("Worker.onmessage:",data); - switch(data.type){ - case 'loaded': - /*Pass our config and shared state on to the async worker.*/ - wMsg('init',state); - break; - case 'inited':{ - /*Indicates that the async partner has received the 'init', - so we now know that the state object is no longer subject to - being copied by a pending postMessage() call.*/ - try { - const rc = capi.sqlite3_vfs_register(opfsVfs.pointer, opfsVfs.$zName); - if(rc){ - opfsVfs.dispose(); - toss("sqlite3_vfs_register(OPFS) failed with rc",rc); - } - if(opfsVfs.pointer !== capi.sqlite3_vfs_find("opfs")){ - toss("BUG: sqlite3_vfs_find() failed for just-installed OPFS VFS"); - } - capi.sqlite3_vfs_register.addReference(opfsVfs, opfsIoMethods); - state.opSABView = new Int32Array(state.opSAB); - - if(thisUrl.searchParams.has('opfs-sanity-check')){ - warn("Running sanity checks because of opfs-sanity-check URL arg..."); - sanityCheck(); - } - warn("End of (very incomplete) OPFS setup.", opfsVfs); - }catch(e){ - error(e); - } - break; - } - default: - error("Unexpected message from the async worker:",data); - break; - } - }; -}/*initOpfsVfs*/ + const pVfs = capi.sqlite3_vfs_find("opfs") || toss("Unexpectedly missing 'opfs' VFS."); + const oVfs = capi.sqlite3_vfs.instanceForPointer(pVfs); + log("OPFS VFS:",pVfs, oVfs); + log("Done!"); +}/*tryOpfsVfs()*/; importScripts('sqlite3.js'); -self.sqlite3InitModule().then((EmscriptenModule)=>initOpfsVfs(EmscriptenModule.sqlite3)); +self.sqlite3InitModule().then((EmscriptenModule)=>{ + EmscriptenModule.sqlite3.installOpfsVfs() + .then((sqlite3)=>tryOpfsVfs(sqlite3)) + .catch((e)=>{ + console.error("Error initializing OPFS VFS:",e); + throw e; + }); +}); diff --git a/manifest b/manifest index 561605cbab..98c1a9df86 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Plug\sOPFS\smethods\sin\sto\stheir\ssqlite3_vfs/io_methods\scounterparts.\sAdd\sURL\sargs\sto\scontrol\sdebug\soutput\sand\srunning\sof\ssanity-checks\sin\sthe\sOPFS\sinit\scode. -D 2022-09-18T00:16:12.445 +C Move\sthe\sOPFS\sVFS\sbits\sback\sinto\sapi/sqlite3-api-opfs.js.\sRefactor\sthe\sOPFS\sVFS\sinit\sprocess\sto\suse\sa\sPromise-returning\sfunction\swhich\sthe\sclient\smust\scall,\sas\sthat\seliminates\sany\suncertainty\sabout\swhen\sthe\sVFS\s(necessarily\sactivated\sasynchronously)\sactually\sbecomes\savailable\sto\sthe\sclient. +D 2022-09-18T02:35:30.998 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -474,7 +474,7 @@ F ext/userauth/user-auth.txt e6641021a9210364665fe625d067617d03f27b04 F ext/userauth/userauth.c 7f00cded7dcaa5d47f54539b290a43d2e59f4b1eb5f447545fa865f002fc80cb F ext/wasm/EXPORTED_FUNCTIONS.fiddle 7fb73f7150ab79d83bb45a67d257553c905c78cd3d693101699243f36c5ae6c3 F ext/wasm/EXPORTED_RUNTIME_METHODS.fiddle a004bd5eeeda6d3b28d16779b7f1a80305bfe009dfc7f0721b042967f0d39d02 -F ext/wasm/GNUmakefile 0323a7597383bf0dab473304f3a8a7e29d49298d92b5413692c012be2dfa84bf +F ext/wasm/GNUmakefile 24e5802cc186b492b3ef6290b64b857e51aa0afaa929b74a68f9fb457c4e8814 F ext/wasm/README.md e1ee1e7c321c6a250bf78a84ca6f5882890a237a450ba5a0649c7a8399194c52 F ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api 150a793a47205b8009ac934f3b6d6ebf67b965c072339aaa25ce808a19e116cc F ext/wasm/api/EXPORTED_RUNTIME_METHODS.sqlite3-api 1ec3c73e7d66e95529c3c64ac3de2470b0e9e7fbf7a5b41261c367cf4f1b7287 @@ -484,7 +484,7 @@ F ext/wasm/api/post-js-header.js 0e853b78db83cb1c06b01663549e0e8b4f377f12f5a2d9a F ext/wasm/api/sqlite3-api-cleanup.js 8564a6077cdcaea9a9f428a019af8a05887f0131e6a2a1e72a7ff1145fadfe77 F ext/wasm/api/sqlite3-api-glue.js 366d580c8e5bf7fcf4c6dee6f646c31f5549bd417ea03a59a0acca00e8ecce30 F ext/wasm/api/sqlite3-api-oo1.js d7526517f7ad3f6bda16ad66d373bbb71b43168deef7af60eda5c9fe873d1387 -F ext/wasm/api/sqlite3-api-opfs.js 130f60cc8f5835f9d77d4f12308bf4c8fb6d9c315009fc7239c5d67ff2bc8c67 +F ext/wasm/api/sqlite3-api-opfs.js 87d98f2449d5790efd7044e492166e4ed767e3320a03ed5a173b2b9364fc4896 F ext/wasm/api/sqlite3-api-prologue.js 48ebca4ae340b0242d4f39bbded01bd0588393c8023628be1c454b4db6f7bd6e F ext/wasm/api/sqlite3-api-worker1.js d33062afa045fd4be01ba4abc266801807472558b862b30056211b00c9c347b4 F ext/wasm/api/sqlite3-wasi.h 25356084cfe0d40458a902afb465df8c21fc4152c1d0a59b563a3fba59a068f9 @@ -534,7 +534,7 @@ F ext/wasm/testing2.html a66951c38137ff1d687df79466351f3c734fa9c6d9cce71d3cf97c2 F ext/wasm/testing2.js 25584bcc30f19673ce13a6f301f89f8820a59dfe044e0c4f2913941f4097fe3c F ext/wasm/wasmfs.make 21a5cf297954a689e0dc2a95299ae158f681cae5e90c10b99d986097815fd42d F ext/wasm/x-sync-async.html d85cb9b1ab398ef5a20ce64a689ce4bf9a347ffe69edd46c2d3dc57b869a8925 -F ext/wasm/x-sync-async.js 2cd04d73ddc515479cc2e4b9246d6da21b3f494020261d47470f4b710c84c0da +F ext/wasm/x-sync-async.js 95142516c0e467480fb87e71d7aab5b8ecee07fe7bd4babb5f5aa9b5b6ace4d0 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60 @@ -2030,8 +2030,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P cd06cc670029763955cf60ffcf944b36d41cb005b859d9b9fd0eea1b6741d0e9 -R af48df928e44c11df9f0c9eb2cf8376e +P a0e93ed20b2463606a63b03ce8ca41ec1fb22886db5c5c898ace86ba24636f70 +R 088ef11f22396ef93f0589b73a5e4797 U stephan -Z 07b2e0eb2359d701cc905c5402ed9c24 +Z 0f7cc3e7ea750e2836a1d35bc86649c9 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 274fb7f0c1..955e61f29c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a0e93ed20b2463606a63b03ce8ca41ec1fb22886db5c5c898ace86ba24636f70 \ No newline at end of file +1c660970d0f62bcfd6e698a72b050d99972a1e39f45a5ac24194a190f8f78ab3 \ No newline at end of file