From: stephan Date: Sat, 17 Sep 2022 20:50:12 +0000 (+0000) Subject: Add the remaining vfs/io_methods wrappers to the OPFS sync/async proxy, but most... X-Git-Tag: version-3.40.0~169^2~107 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=0731554629912621874ec7ebd5ab307e270caef4;p=thirdparty%2Fsqlite.git Add the remaining vfs/io_methods wrappers to the OPFS sync/async proxy, but most are not yet tested. FossilOrigin-Name: 44db9132145b3072488ea91db53f6c06be74544beccad5fd07efd22c0f03dc04 --- diff --git a/ext/wasm/sqlite3-opfs-async-proxy.js b/ext/wasm/sqlite3-opfs-async-proxy.js index 98e2688149..88fdfa603a 100644 --- a/ext/wasm/sqlite3-opfs-async-proxy.js +++ b/ext/wasm/sqlite3-opfs-async-proxy.js @@ -20,267 +20,308 @@ https://github.com/rhashimoto/wa-sqlite/blob/master/src/examples/OriginPrivateFileSystemVFS.js for demonstrating how to use the OPFS APIs. + + This file is to be loaded as a Worker. It does not have any direct + access to the sqlite3 JS/WASM bits, so any bits which it needs (most + notably SQLITE_xxx integer codes) have to be imported into it via an + initialization process. */ 'use strict'; -(function(){ - const toss = function(...args){throw new Error(args.join(' '))}; - if(self.window === self){ - toss("This code cannot run from the main thread.", - "Load it as a Worker from a separate Worker."); - }else if(!navigator.storage.getDirectory){ - toss("This API requires navigator.storage.getDirectory."); - } - const logPrefix = "OPFS worker:"; - const log = (...args)=>{ - console.log(logPrefix,...args); - }; - const warn = (...args)=>{ - console.warn(logPrefix,...args); - }; - const error = (...args)=>{ - console.error(logPrefix,...args); - }; +const toss = function(...args){throw new Error(args.join(' '))}; +if(self.window === self){ + toss("This code cannot run from the main thread.", + "Load it as a Worker from a separate Worker."); +}else if(!navigator.storage.getDirectory){ + toss("This API requires navigator.storage.getDirectory."); +} +/** + Will hold state copied to this object from the syncronous side of + this API. +*/ +const state = Object.create(null); +/** + verbose: - warn("This file is very much experimental and under construction.",self.location.pathname); - const wMsg = (type,payload)=>postMessage({type,payload}); + 0 = no logging output + 1 = only errors + 2 = warnings and errors + 3 = debug, warnings, and errors +*/ +state.verbose = 2; - const state = Object.create(null); - /*state.opSab; - state.sabIO; - state.opBuf; - state.opIds; - state.rootDir;*/ - /** - Map of sqlite3_file pointers (integers) to metadata related to a - given OPFS file handles. The pointers are, in this side of the - interface, opaque file handle IDs provided by the synchronous - part of this constellation. Each value is an object with a structure - demonstrated in the xOpen() impl. - */ - state.openFiles = Object.create(null); +const __logPrefix = "OPFS asyncer:"; +const log = (...args)=>{ + if(state.verbose>2) console.log(__logPrefix,...args); +}; +const warn = (...args)=>{ + if(state.verbose>1) console.warn(__logPrefix,...args); +}; +const error = (...args)=>{ + if(state.verbose) console.error(__logPrefix,...args); +}; - /** - Map of dir names to FileSystemDirectoryHandle objects. - */ - state.dirCache = new Map; +warn("This file is very much experimental and under construction.",self.location.pathname); - const __splitPath = (absFilename)=>{ - const a = absFilename.split('/').filter((v)=>!!v); - return [a, a.pop()]; - }; - /** - Takes the absolute path to a filesystem element. Returns an array - of [handleOfContainingDir, filename]. If the 2nd argument is - truthy then each directory element leading to the file is created - along the way. Throws if any creation or resolution fails. - */ - const getDirForPath = async function f(absFilename, createDirs = false){ - const url = new URL( - absFilename, 'file://xyz' - ) /* use URL to resolve path pieces such as a/../b */; - const [path, filename] = __splitPath(url.pathname); - const allDirs = path.join('/'); - let dh = state.dirCache.get(allDirs); - if(!dh){ - dh = state.rootDir; - for(const dirName of path){ - if(dirName){ - dh = await dh.getDirectoryHandle(dirName, {create: !!createDirs}); - } +/** + Map of sqlite3_file pointers (integers) to metadata related to a + given OPFS file handles. The pointers are, in this side of the + interface, opaque file handle IDs provided by the synchronous + part of this constellation. Each value is an object with a structure + demonstrated in the xOpen() impl. +*/ +const __openFiles = Object.create(null); + +/** + Map of dir names to FileSystemDirectoryHandle objects. +*/ +const __dirCache = new Map; + +/** + Takes the absolute path to a filesystem element. Returns an array + of [handleOfContainingDir, filename]. If the 2nd argument is + truthy then each directory element leading to the file is created + along the way. Throws if any creation or resolution fails. +*/ +const getDirForPath = async function f(absFilename, createDirs = false){ + const url = new URL( + absFilename, 'file://xyz' + ) /* use URL to resolve path pieces such as a/../b */; + const path = url.pathname.split('/').filter((v)=>!!v); + const filename = path.pop(); + const allDirs = '/'+path.join('/'); + let dh = __dirCache.get(allDirs); + if(!dh){ + dh = state.rootDir; + for(const dirName of path){ + if(dirName){ + dh = await dh.getDirectoryHandle(dirName, {create: !!createDirs}); } - state.dirCache.set(allDirs, dh); } - return [dh, filename]; - }; + __dirCache.set(allDirs, dh); + } + return [dh, filename]; +}; - - /** - Generates a random ASCII string len characters long, intended for - use as a temporary file name. - */ - const randomFilename = function f(len=16){ - if(!f._chars){ - f._chars = "abcdefghijklmnopqrstuvwxyz"+ - "ABCDEFGHIJKLMNOPQRSTUVWXYZ"+ - "012346789"; - f._n = f._chars.length; - } - const a = []; - let i = 0; - for( ; i < len; ++i){ - const ndx = Math.random() * (f._n * 64) % f._n | 0; - a[i] = f._chars[ndx]; - } - return a.join(''); - }; - const storeAndNotify = (opName, value)=>{ - log(opName+"() is notify()ing w/ value:",value); - Atomics.store(state.opBuf, state.opIds[opName], value); - Atomics.notify(state.opBuf, state.opIds[opName]); - }; +/** + Stores the given value at the array index reserved for the given op + and then Atomics.notify()'s it. +*/ +const storeAndNotify = (opName, value)=>{ + log(opName+"() is notify()ing w/ value:",value); + Atomics.store(state.opBuf, state.opIds[opName], value); + Atomics.notify(state.opBuf, state.opIds[opName]); +}; - const isInt32 = function(n){ - return ('bigint'!==typeof n /*TypeError: can't convert BigInt to number*/) - && !!(n===(n|0) && n<=2147483647 && n>=-2147483648); - }; - const affirm32Bits = function(n){ - return isInt32(n) || toss("Number is too large (>31 bits):",n); - }; +const isInt32 = function(n){ + return ('bigint'!==typeof n /*TypeError: can't convert BigInt to number*/) + && !!(n===(n|0) && n<=2147483647 && n>=-2147483648); +}; +const affirm32Bits = function(n){ + return isInt32(n) || toss("Number is too large (>31 bits) (FIXME!):",n); +}; - const ioMethods = { - xAccess: async function({filename, exists, readWrite}){ - log("xAccess(",arguments,")"); - const rc = 1; - storeAndNotify('xAccess', rc); - }, - xClose: async function(fid){ - const opName = 'xClose'; - log(opName+"(",arguments[0],")"); - log("state.openFiles",state.openFiles); - const fh = state.openFiles[fid]; - if(fh){ - delete state.openFiles[fid]; - //await fh.close(); - if(fh.accessHandle) await fh.accessHandle.close(); - if(fh.deleteOnClose){ - try{ - await fh.dirHandle.removeEntry(fh.filenamePart); - } - catch(e){ - warn("Ignoring dirHandle.removeEntry() failure of",fh); - } - } - log("state.openFiles",state.openFiles); - storeAndNotify(opName, 0); - }else{ - storeAndNotify(opName, state.errCodes.NotFound); - } - }, - xDelete: async function(filename){ - log("xDelete(",arguments,")"); - storeAndNotify('xClose', 0); - }, - xFileSize: async function(fid){ - log("xFileSize(",arguments,")"); - const fh = state.openFiles[fid]; - const sz = await fh.getSize(); - affirm32Bits(sz); - storeAndNotify('xFileSize', sz | 0); - }, - xOpen: async function({ - fid/*sqlite3_file pointer*/, sab/*file-specific SharedArrayBuffer*/, - filename, - fileType = undefined /*mainDb, mainJournal, etc.*/, - create = false, readOnly = false, deleteOnClose = false, - }){ - const opName = 'xOpen'; - try{ - if(create) readOnly = false; - log(opName+"(",arguments[0],")"); - - let hDir, filenamePart, hFile; - try { - [hDir, filenamePart] = await getDirForPath(filename, !!create); - }catch(e){ - storeAndNotify(opName, state.errCodes.NotFound); - return; - } - hFile = await hDir.getFileHandle(filenamePart, {create: !!create}); - log(opName,"filenamePart =",filenamePart, 'hDir =',hDir); - const fobj = state.openFiles[fid] = Object.create(null); - fobj.filenameAbs = filename; - fobj.filenamePart = filenamePart; - fobj.dirHandle = hDir; - fobj.fileHandle = hFile; - fobj.accessHandle = undefined; - fobj.fileType = fileType; - fobj.sab = sab; - fobj.create = !!create; - fobj.readOnly = !!readOnly; - fobj.deleteOnClose = !!deleteOnClose; +/** + Throws if fh is a file-holding object which is flagged as read-only. +*/ +const affirmNotRO = function(opName,fh){ + if(fh.readOnly) toss(opName+"(): File is read-only: "+fh.filenameAbs); +}; - /** - wa-sqlite, at this point, grabs a SyncAccessHandle and - assigns it to the accessHandle prop of the file state - object, but it's unclear why it does that. - */ - storeAndNotify(opName, 0); +/** + Asynchronous wrappers for sqlite3_vfs and sqlite3_io_methods + methods. Maintenance reminder: members are in alphabetical order + to simplify finding them. +*/ +const vfsAsyncImpls = { + xAccess: async function({filename, exists, readWrite}){ + warn("xAccess(",arguments[0],") is TODO"); + const rc = state.sq3Codes.SQLITE_IOERR; + storeAndNotify('xAccess', rc); + }, + xClose: async function(fid){ + const opName = 'xClose'; + log(opName+"(",arguments[0],")"); + const fh = __openFiles[fid]; + if(fh){ + delete __openFiles[fid]; + if(fh.accessHandle) await fh.accessHandle.close(); + if(fh.deleteOnClose){ + try{ await fh.dirHandle.removeEntry(fh.filenamePart) } + catch(e){ warn("Ignoring dirHandle.removeEntry() failure of",fh,e) } + } + storeAndNotify(opName, 0); + }else{ + storeAndNotify(opName, state.sq3Codes.SQLITE_NOFOUND); + } + }, + xDelete: async function({filename, syncDir/*ignored*/}){ + log("xDelete(",arguments[0],")"); + try { + const [hDir, filenamePart] = await getDirForPath(filename, false); + await hDir.removeEntry(filenamePart); + }catch(e){ + /* Ignoring: _presumably_ the file can't be found. */ + } + storeAndNotify('xDelete', 0); + }, + xFileSize: async function(fid){ + log("xFileSize(",arguments,")"); + const fh = __openFiles[fid]; + let sz; + try{ + sz = await fh.accessHandle.getSize(); + fh.sabViewFileSize.setBigInt64(0, BigInt(sz)); + sz = 0; + }catch(e){ + error("xFileSize():",e, fh); + sz = state.sq3Codes.SQLITE_IOERR; + } + storeAndNotify('xFileSize', sz); + }, + xOpen: async function({ + fid/*sqlite3_file pointer*/, + sab/*file-specific SharedArrayBuffer*/, + filename, + fileType = undefined /*mainDb, mainJournal, etc.*/, + create = false, readOnly = false, deleteOnClose = false + }){ + const opName = 'xOpen'; + try{ + if(create) readOnly = false; + log(opName+"(",arguments[0],")"); + let hDir, filenamePart; + try { + [hDir, filenamePart] = await getDirForPath(filename, !!create); }catch(e){ - error(opName,e); - storeAndNotify(opName, state.errCodes.IO); + storeAndNotify(opName, state.sql3Codes.SQLITE_NOTFOUND); + return; } - }, - xRead: async function({fid,n,offset}){ - log("xRead(",arguments,")"); - affirm32Bits(n + offset); - const fh = state.openFiles[fid]; - storeAndNotify('xRead',fid); - }, - xSleep: async function f({ms}){ - log("xSleep(",arguments[0],")"); - await new Promise((resolve)=>{ - setTimeout(()=>resolve(), ms); - }).finally(()=>storeAndNotify('xSleep',0)); - }, - xSync: async function({fid}){ - log("xSync(",arguments,")"); - const fh = state.openFiles[fid]; - await fh.flush(); - storeAndNotify('xSync',fid); - }, - xTruncate: async function({fid,size}){ - log("xTruncate(",arguments,")"); - affirm32Bits(size); - const fh = state.openFiles[fid]; - fh.truncate(size); - storeAndNotify('xTruncate',fid); - }, - xWrite: async function({fid,src,n,offset}){ - log("xWrite(",arguments,")"); - const fh = state.openFiles[fid]; - storeAndNotify('xWrite',fid); + const hFile = await hDir.getFileHandle(filenamePart, {create: !!create}); + log(opName,"filenamePart =",filenamePart, 'hDir =',hDir); + const fobj = __openFiles[fid] = Object.create(null); + fobj.filenameAbs = filename; + fobj.filenamePart = filenamePart; + fobj.dirHandle = hDir; + fobj.fileHandle = hFile; + fobj.fileType = fileType; + fobj.sab = sab; + fobj.sabViewFileSize = new DataView(sab,state.fbInt64Offset,8); + fobj.create = !!create; + fobj.readOnly = !!readOnly; + fobj.deleteOnClose = !!deleteOnClose; + /** + wa-sqlite, at this point, grabs a SyncAccessHandle and + assigns it to the accessHandle prop of the file state + object, but only for certain cases and it's unclear why it + places that limitation on it. + */ + fobj.accessHandle = await hFile.createSyncAccessHandle(); + storeAndNotify(opName, 0); + }catch(e){ + error(opName,e); + storeAndNotify(opName, state.sq3Codes.SQLITE_IOERR); } - }; - - const onReady = function(){ - self.onmessage = async function({data}){ - log("self.onmessage",data); - switch(data.type){ - case 'init':{ - const opt = data.payload; - state.opSab = opt.opSab; - state.opBuf = new Int32Array(state.opSab); - state.opIds = opt.opIds; - state.errCodes = opt.errCodes; - state.sq3Codes = opt.sq3Codes; - Object.keys(ioMethods).forEach((k)=>{ - if(!state.opIds[k]){ - toss("Maintenance required: missing state.opIds[",k,"]"); - } - }); - log("init state",state); - break; - } - default:{ - const m = ioMethods[data.type] || toss("Unknown message type:",data.type); - try { - await m(data.payload); - }catch(e){ - error("Error handling",data.type+"():",e); - storeAndNotify(data.type, -99); + }, + xRead: async function({fid,n,offset}){ + log("xRead(",arguments[0],")"); + let rc = 0; + const fh = __openFiles[fid]; + try{ + const aRead = new Uint8array(fh.sab, n); + const nRead = fh.accessHandle.read(aRead, {at: offset}); + if(nRead < n){/* Zero-fill remaining bytes */ + new Uint8array(fh.sab).fill(0, nRead, n); + rc = state.sq3Codes.SQLITE_IOERR_SHORT_READ; + } + }catch(e){ + error("xRead() failed",e,fh); + rc = state.sq3Codes.SQLITE_IOERR_READ; + } + storeAndNotify('xRead',rc); + }, + xSleep: async function f(ms){ + log("xSleep(",ms,")"); + await new Promise((resolve)=>{ + setTimeout(()=>resolve(), ms); + }).finally(()=>storeAndNotify('xSleep',0)); + }, + xSync: async function({fid,flags/*ignored*/}){ + log("xSync(",arguments[0],")"); + const fh = __openFiles[fid]; + if(!fh.readOnly && fh.accessHandle) await fh.accessHandle.flush(); + storeAndNotify('xSync',0); + }, + xTruncate: async function({fid,size}){ + log("xTruncate(",arguments[0],")"); + let rc = 0; + const fh = __openFiles[fid]; + try{ + affirmNotRO('xTruncate', fh); + await fh.accessHandle.truncate(size); + }catch(e){ + error("xTruncate():",e,fh); + rc = state.sq3Codes.SQLITE_IOERR_TRUNCATE; + } + storeAndNotify('xTruncate',rc); + }, + xWrite: async function({fid,src,n,offset}){ + log("xWrite(",arguments[0],")"); + let rc; + try{ + const fh = __openFiles[fid]; + affirmNotRO('xWrite', fh); + const nOut = fh.accessHandle.write(new UInt8Array(fh.sab, 0, n), {at: offset}); + rc = (nOut===n) ? 0 : state.sq3Codes.SQLITE_IOERR_WRITE; + }catch(e){ + error("xWrite():",e,fh); + rc = state.sq3Codes.SQLITE_IOERR_WRITE; + } + storeAndNotify('xWrite',rc); + } +}; + +navigator.storage.getDirectory().then(function(d){ + const wMsg = (type)=>postMessage({type}); + state.rootDir = d; + log("state.rootDir =",state.rootDir); + self.onmessage = async function({data}){ + log("self.onmessage()",data); + switch(data.type){ + case 'init':{ + /* Receive shared state from synchronous partner */ + const opt = data.payload; + state.verbose = opt.verbose ?? 2; + state.fileBufferSize = opt.fileBufferSize; + state.fbInt64Offset = opt.fbInt64Offset; + state.opSab = opt.opSab; + state.opBuf = new Int32Array(state.opSab); + state.opIds = opt.opIds; + state.sq3Codes = opt.sq3Codes; + Object.keys(vfsAsyncImpls).forEach((k)=>{ + if(!Number.isFinite(state.opIds[k])){ + toss("Maintenance required: missing state.opIds[",k,"]"); } - break; + }); + log("init state",state); + wMsg('inited'); + break; + } + default:{ + let err; + const m = vfsAsyncImpls[data.type] || toss("Unknown message type:",data.type); + try { + await m(data.payload).catch((e)=>err=e); + }catch(e){ + err = e; } - } - }; - wMsg('ready'); + if(err){ + error("Error handling",data.type+"():",e); + storeAndNotify(data.type, state.sq3Codes.SQLITE_ERROR); + } + break; + } + } }; - - navigator.storage.getDirectory().then(function(d){ - state.rootDir = d; - log("state.rootDir =",state.rootDir); - onReady(); - }); - -})(); + wMsg('loaded'); +}); diff --git a/ext/wasm/x-sync-async.html b/ext/wasm/x-sync-async.html index 4b2e08a31f..ec0a6353b5 100644 --- a/ext/wasm/x-sync-async.html +++ b/ext/wasm/x-sync-async.html @@ -13,16 +13,10 @@
This is an experiment in wrapping the asynchronous OPFS APIs behind a fully synchronous proxy. It is very much incomplete, under construction, and experimental. - See the dev console for all output. + See the dev console for all output.
- - - + diff --git a/ext/wasm/x-sync-async.js b/ext/wasm/x-sync-async.js index fec7efa73f..c3027585b1 100644 --- a/ext/wasm/x-sync-async.js +++ b/ext/wasm/x-sync-async.js @@ -1,5 +1,29 @@ +/* + 2022-09-17 + + The author disclaims copyright to this source code. In place of a + legal notice, here is a blessing: + + * May you do good and not evil. + * May you find forgiveness for yourself and forgive others. + * May you share freely, never taking more than you give. + + *********************************************************************** + + A EXTREMELY INCOMPLETE and UNDER CONSTRUCTION experiment for OPFS. + This file holds the synchronous half of an sqlite3_vfs + implementation which proxies, in a synchronous fashion, the + asynchronous OPFS APIs using a second Worker. +*/ 'use strict'; -const doAtomicsStuff = function(sqlite3){ +/** + This function is a placeholder for use in development. When + working, this will be moved into a file named + api/sqlite3-api-opfs.js, or similar, and hooked in to the + sqlite-api build construct. +*/ +const initOpfsVfs = function(sqlite3){ + const toss = function(...args){throw new Error(args.join(' '))}; const logPrefix = "OPFS syncer:"; const log = (...args)=>{ console.log(logPrefix,...args); @@ -10,46 +34,73 @@ const doAtomicsStuff = function(sqlite3){ const error = (...args)=>{ console.error(logPrefix,...args); }; + warn("This file is very much experimental and under construction.",self.location.pathname); + + const capi = sqlite3.capi; + const wasm = capi.wasm; + const sqlite3_vfs = capi.sqlite3_vfs + || toss("Missing sqlite3.capi.sqlite3_vfs object."); + const sqlite3_file = capi.sqlite3_file + || toss("Missing sqlite3.capi.sqlite3_file object."); + const sqlite3_io_methods = capi.sqlite3_io_methods + || toss("Missing sqlite3.capi.sqlite3_io_methods object."); + const StructBinder = sqlite3.StructBinder || toss("Missing sqlite3.StructBinder."); + const W = new Worker("sqlite3-opfs-async-proxy.js"); const wMsg = (type,payload)=>W.postMessage({type,payload}); - warn("This file is very much experimental and under construction.",self.location.pathname); /** State which we send to the async-api Worker or share with it. This object must initially contain only cloneable or sharable - objects. After the worker's "ready" message arrives, other types + objects. After the worker's "inited" message arrives, other types of data may be added to it. */ const state = Object.create(null); + state.verbose = 3; + state.fileBufferSize = 1024 * 64 + 8 /* size of fileHandle.sab. 64k = max sqlite3 page size */; + state.fbInt64Offset = state.fileBufferSize - 8 /*spot in fileHandle.sab to store an int64*/; state.opIds = Object.create(null); - state.opIds.xAccess = 1; - state.opIds.xClose = 2; - state.opIds.xDelete = 3; - state.opIds.xFileSize = 4; - state.opIds.xOpen = 5; - state.opIds.xRead = 6; - state.opIds.xSync = 7; - state.opIds.xTruncate = 8; - state.opIds.xWrite = 9; - state.opIds.xSleep = 10; - state.opIds.xBlock = 99 /* to block worker while this code is still handling something */; - state.opSab = new SharedArrayBuffer(64); - state.fileBufferSize = 1024 * 65 /* 64k = max sqlite3 page size */; - /* TODO: use SQLITE_xxx err codes. */ - state.errCodes = Object.create(null); - state.errCodes.Error = -100; - state.errCodes.IO = -101; - state.errCodes.NotFound = -102; - state.errCodes.Misuse = -103; - - // TODO: add any SQLITE_xxx symbols we need here. + { + let i = 0; + state.opIds.xAccess = i++; + state.opIds.xClose = i++; + state.opIds.xDelete = i++; + state.opIds.xFileSize = i++; + state.opIds.xOpen = i++; + state.opIds.xRead = i++; + state.opIds.xSleep = i++; + state.opIds.xSync = i++; + state.opIds.xTruncate = i++; + state.opIds.xWrite = i++; + state.opSab = new SharedArrayBuffer(i * 4); + } + state.sq3Codes = Object.create(null); - - const isWorkerErrCode = (n)=>(n<=state.errCodes.Error); + state.sq3Codes._reverse = Object.create(null); + [ // SQLITE_xxx constants to export to the async worker counterpart... + 'SQLITE_ERROR', 'SQLITE_IOERR', + 'SQLITE_NOTFOUND', 'SQLITE_MISUSE', + 'SQLITE_IOERR_READ', 'SQLITE_IOERR_SHORT_READ', + 'SQLITE_IOERR_WRITE', 'SQLITE_IOERR_FSYNC', + 'SQLITE_IOERR_TRUNCATE', 'SQLITE_IOERR_DELETE', + 'SQLITE_IOERR_ACCESS', 'SQLITE_IOERR_CLOSE' + ].forEach(function(k){ + state.sq3Codes[k] = capi[k] || toss("Maintenance required: not found:",k); + state.sq3Codes._reverse[capi[k]] = k; + }); + + const isWorkerErrCode = (n)=>!!state.sq3Codes._reverse[n]; const opStore = (op,val=-1)=>Atomics.store(state.opBuf, state.opIds[op], val); const opWait = (op,val=-1)=>Atomics.wait(state.opBuf, state.opIds[op], val); + /** + Runs the given operation in the async worker counterpart, waits + for its response, and returns the result which the async worker + writes to the given op's index in state.opBuf. The 2nd argument + must be a single object or primitive value, depending on the + given operation's signature in the async API counterpart. + */ const opRun = (op,args)=>{ opStore(op); wMsg(op, args); @@ -63,71 +114,321 @@ const doAtomicsStuff = function(sqlite3){ }); }; + /** + Generates a random ASCII string len characters long, intended for + use as a temporary file name. + */ + const randomFilename = function f(len=16){ + if(!f._chars){ + f._chars = "abcdefghijklmnopqrstuvwxyz"+ + "ABCDEFGHIJKLMNOPQRSTUVWXYZ"+ + "012346789"; + f._n = f._chars.length; + } + const a = []; + let i = 0; + for( ; i < len; ++i){ + const ndx = Math.random() * (f._n * 64) % f._n | 0; + a[i] = f._chars[ndx]; + } + return a.join(''); + }; + + /** + Map of sqlite3_file pointers to objects constructed by xOpen(). + */ + const __openFiles = Object.create(null); + + const pDVfs = capi.sqlite3_vfs_find(null)/*pointer to default VFS*/; + const dVfs = pDVfs + ? new sqlite3_vfs(pDVfs) + : null /* dVfs will be null when sqlite3 is built with + SQLITE_OS_OTHER. Though we cannot currently handle + that case, the hope is to eventually be able to. */; + const opfsVfs = new sqlite3_vfs(); + const opfsIoMethods = new sqlite3_io_methods(); + opfsVfs.$iVersion = 2/*yes, two*/; + opfsVfs.$szOsFile = capi.sqlite3_file.structInfo.sizeof; + opfsVfs.$mxPathname = 1024/*sure, why not?*/; + opfsVfs.$zName = wasm.allocCString("opfs"); + opfsVfs.ondispose = [ + '$zName', opfsVfs.$zName, + 'cleanup dVfs', ()=>(dVfs ? dVfs.dispose() : null) + ]; + if(dVfs){ + opfsVfs.$xSleep = dVfs.$xSleep; + opfsVfs.$xRandomness = dVfs.$xRandomness; + } + // All C-side memory of opfsVfs is zeroed out, but just to be explicit: + opfsVfs.$xDlOpen = opfsVfs.$xDlError = opfsVfs.$xDlSym = opfsVfs.$xDlClose = null; + /** + Pedantic sidebar about opfsVfs.ondispose: the entries in that array + are items to clean up when opfsVfs.dispose() is called, but in this + environment it will never be called. The VFS instance simply + hangs around until the WASM module instance is cleaned up. We + "could" _hypothetically_ clean it up by "importing" an + sqlite3_os_end() impl into the wasm build, but the shutdown order + of the wasm engine and the JS one are undefined so there is no + guaranty that the opfsVfs instance would be available in one + environment or the other when sqlite3_os_end() is called (_if_ it + gets called at all in a wasm build, which is undefined). + */ + + /** + Impls for the sqlite3_io_methods methods. Maintenance reminder: + members are in alphabetical order to simplify finding them. + */ + const ioSyncWrappers = { + xCheckReservedLock: function(pFile,pOut){ + // Exclusive lock is automatically acquired when opened + //warn("xCheckReservedLock(",arguments,") is a no-op"); + wasm.setMemValue(pOut,1,'i32'); + return 0; + }, + xClose: function(pFile){ + let rc = 0; + const f = __openFiles[pFile]; + if(f){ + delete __openFiles[pFile]; + rc = opRun('xClose', pFile); + if(f.sq3File) f.sq3File.dispose(); + } + return rc; + }, + xDeviceCharacteristics: function(pFile){ + //debug("xDeviceCharacteristics(",pFile,")"); + return capi.SQLITE_IOCAP_UNDELETABLE_WHEN_OPEN; + }, + xFileControl: function(pFile,op,pArg){ + //debug("xFileControl(",arguments,") is a no-op"); + return capi.SQLITE_NOTFOUND; + }, + xFileSize: function(pFile,pSz64){ + const rc = opRun('xFileSize', pFile); + if(!isWorkerErrCode(rc)){ + const f = __openFiles[pFile]; + wasm.setMemValue(pSz64, f.sabViewFileSize.getBigInt64(0) ,'i64'); + } + return rc; + }, + xLock: function(pFile,lockType){ + //2022-09: OPFS handles lock when opened + //warn("xLock(",arguments,") is a no-op"); + return 0; + }, + xRead: function(pFile,pDest,n,offset){ + /* int (*xRead)(sqlite3_file*, void*, int iAmt, sqlite3_int64 iOfst) */ + const f = __opfsHandles[pFile]; + try { + // FIXME(?): block until we finish copying the xRead result buffer. How? + let rc = opRun('xRead',{fid:pFile, n, offset}); + if(0!==rc) return rc; + let i = 0; + for(; i < n; ++i) wasm.setMemValue(pDest + i, f.sabView[i]); + }catch(e){ + error("xRead(",arguments,") failed:",e,f); + rc = capi.SQLITE_IOERR_READ; + } + return rc; + }, + xSync: function(pFile,flags){ + return opRun('xSync', {fid:pFile, flags}); + }, + xTruncate: function(pFile,sz64){ + return opRun('xTruncate', {fid:pFile, size: sz64}); + }, + xUnlock: function(pFile,lockType){ + //2022-09: OPFS handles lock when opened + //warn("xUnlock(",arguments,") is a no-op"); + return 0; + }, + xWrite: function(pFile,pSrc,n,offset){ + /* int (*xWrite)(sqlite3_file*, const void*, int iAmt, sqlite3_int64 iOfst) */ + const f = __opfsHandles[pFile]; + try { + let i = 0; + // FIXME(?): block from here until we finish the xWrite. How? + for(; i < n; ++i) f.sabView[i] = wasm.getMemValue(pSrc+i); + return opRun('xWrite',{fid:pFile, n, offset}); + }catch(e){ + error("xWrite(",arguments,") failed:",e,f); + return capi.SQLITE_IOERR_WRITE; + } + } + }/*ioSyncWrappers*/; + + /** + Impls for the sqlite3_vfs methods. Maintenance reminder: members + are in alphabetical order to simplify finding them. + */ const vfsSyncWrappers = { - xOpen: function f(pFile, name, flags, outFlags = {}){ + // TODO: xAccess + xCurrentTime: function(pVfs,pOut){ + /* If it turns out that we need to adjust for timezone, see: + https://stackoverflow.com/a/11760121/1458521 */ + wasm.setMemValue(pOut, 2440587.5 + (new Date().getTime()/86400000), + 'double'); + return 0; + }, + xCurrentTimeInt64: function(pVfs,pOut){ + // TODO: confirm that this calculation is correct + wasm.setMemValue(pOut, (2440587.5 * 86400000) + new Date().getTime(), + 'i64'); + return 0; + }, + xDelete: function(pVfs, zName, doSyncDir){ + return opRun('xDelete', {filename: wasm.cstringToJs(zName), syncDir: doSyncDir}); + }, + xFullPathname: function(pVfs,zName,nOut,pOut){ + /* Until/unless we have some notion of "current dir" + in OPFS, simply copy zName to pOut... */ + const i = wasm.cstrncpy(pOut, zName, nOut); + return ipMethods is NULL. */ + if(args.readOnly){ + wasm.setMemValue(pOutFlags, capi.SQLITE_OPEN_READONLY, 'i32'); + } + __openFiles[pFile] = args; + args.sabView = new Uint8Array(args.sab); + args.sabViewFileSize = new DataView(args.sab, state.fbInt64Offset, 8); + args.sq3File = new sqlite3_file(pFile); + args.sq3File.$pMethods = opfsIoMethods.pointer; args.ba = new Uint8Array(args.sab); - state.openFiles[pFile] = args; } return rc; - }, - xClose: function(pFile){ - let rc = 0; - if(state.openFiles[pFile]){ - delete state.openFiles[pFile]; - rc = opRun('xClose', pFile); - } - return rc; - } - }; + }/*xOpen()*/ + }/*vfsSyncWrappers*/; + if(!opfsVfs.$xRandomness){ + /* If the default VFS has no xRandomness(), add a basic JS impl... */ + vfsSyncWrappers.xRandomness = function(pVfs, nOut, pOut){ + const heap = wasm.heap8u(); + let i = 0; + for(; i < nOut; ++i) heap[pOut + i] = (Math.random()*255000) & 0xFF; + return i; + }; + } + if(!opfsVfs.$xSleep){ + /* If we can inherit an xSleep() impl from the default VFS then + use it, otherwise install one which is certainly less accurate + because it has to go round-trip through the async worker, but + provides the only option for a synchronous sleep() in JS. */ + vfsSyncWrappers.xSleep = (pVfs,ms)=>opRun('xSleep',ms); + } - const doSomething = function(){ + /* + TODO: plug in the above functions in to opfsVfs and opfsIoMethods. + Code for doing so is in api/sqlite3-api-opfs.js. + */ + + const sanityCheck = async function(){ //state.ioBuf = new Uint8Array(state.sabIo); - const fid = 37; - let rc = vfsSyncWrappers.xOpen(fid, "/foo/bar/baz.sqlite3",0, {}); - log("open rc =",rc,"state.opBuf[xOpen] =",state.opBuf[state.opIds.xOpen]); - if(isWorkerErrCode(rc)){ - error("open failed with code",rc); - return; - } - log("xSleep()ing before close()ing..."); - opRun('xSleep',{ms: 1500}); - log("wait()ing before close()ing..."); - wait(1500).then(function(){ - rc = vfsSyncWrappers.xClose(fid); + const scope = wasm.scopedAllocPush(); + const sq3File = new sqlite3_file(); + try{ + const fid = sq3File.pointer; + const openFlags = capi.SQLITE_OPEN_CREATE + | capi.SQLITE_OPEN_READWRITE + | capi.SQLITE_OPEN_DELETEONCLOSE + | capi.SQLITE_OPEN_MAIN_DB; + const pOut = wasm.scopedAlloc(8); + const dbFile = "/sanity/check/file"; + let rc = vfsSyncWrappers.xOpen(opfsVfs.pointer, dbFile, + fid, openFlags, pOut); + log("open rc =",rc,"state.opBuf[xOpen] =",state.opBuf[state.opIds.xOpen]); + if(isWorkerErrCode(rc)){ + error("open failed with code",rc); + return; + } + rc = ioSyncWrappers.xSync(sq3File.pointer, 0); + if(rc) toss('sync failed w/ rc',rc); + rc = ioSyncWrappers.xTruncate(sq3File.pointer, 1024); + if(rc) toss('truncate failed w/ rc',rc); + wasm.setMemValue(pOut,0,'i64'); + rc = ioSyncWrappers.xFileSize(sq3File.pointer, pOut); + if(rc) toss('xFileSize failed w/ rc',rc); + log("xFileSize says:",wasm.getMemValue(pOut, 'i64')); + log("xSleep()ing before close()ing..."); + opRun('xSleep',1500); + rc = ioSyncWrappers.xClose(fid); log("xClose rc =",rc,"opBuf =",state.opBuf); - }); + log("Deleting file:",dbFile); + opRun('xDelete', dbFile); + }finally{ + sq3File.dispose(); + wasm.scopedAllocPop(scope); + } }; + W.onmessage = function({data}){ log("Worker.onmessage:",data); switch(data.type){ - case 'ready': + case 'loaded': + /*Pass our config and shared state on to the async worker.*/ wMsg('init',state); + break; + case 'inited': + /*Indicates that the async partner has received the 'init', + so we now know that the state object is no longer subject to + being copied by a pending postMessage() call.*/ state.opBuf = new Int32Array(state.opSab); - state.openFiles = Object.create(null); - doSomething(); + sanityCheck(); + break; + default: + error("Unexpected message from the async worker:",data); break; } }; -}/*doAtomicsStuff*/ +}/*initOpfsVfs*/ importScripts('sqlite3.js'); -self.sqlite3InitModule().then((EmscriptenModule)=>doAtomicsStuff(EmscriptenModule.sqlite3)); +self.sqlite3InitModule().then((EmscriptenModule)=>initOpfsVfs(EmscriptenModule.sqlite3)); diff --git a/manifest b/manifest index eeba3142ed..cce6558812 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sinitial\sbits\sof\san\sexperimental\sasync-impl-via-synchronous-interface\sproxy\sintended\sto\smarshal\sOPFS\svia\ssqlite3_vfs\sAPI. -D 2022-09-17T15:08:22.642 +C Add\sthe\sremaining\svfs/io_methods\swrappers\sto\sthe\sOPFS\ssync/async\sproxy,\sbut\smost\sare\snot\syet\stested. +D 2022-09-17T20:50:12.684 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -523,7 +523,7 @@ F ext/wasm/speedtest1.html fbb8e4d1639028443f3687a683be660beca6927920545cf6b1fdf F ext/wasm/split-speedtest1-script.sh a3e271938d4d14ee49105eb05567c6a69ba4c1f1293583ad5af0cd3a3779e205 x F ext/wasm/sql/000-mandelbrot.sql 775337a4b80938ac8146aedf88808282f04d02d983d82675bd63d9c2d97a15f0 F ext/wasm/sql/001-sudoku.sql 35b7cb7239ba5d5f193bc05ec379bcf66891bce6f2a5b3879f2f78d0917299b5 -F ext/wasm/sqlite3-opfs-async-proxy.js c42a097dfbb96abef08554b173a47788f5bc1f58c266f859ba01c1fa3ff8327d +F ext/wasm/sqlite3-opfs-async-proxy.js 62024877ad13fdff1834581ca1951ab58bda431e4d548aaaf4506ea54f0ed2de F ext/wasm/sqlite3-worker1-promiser.js 92b8da5f38439ffec459a8215775d30fa498bc0f1ab929ff341fc3dd479660b9 F ext/wasm/sqlite3-worker1.js 0c1e7626304543969c3846573e080c082bf43bcaa47e87d416458af84f340a9e F ext/wasm/testing-worker1-promiser.html 6eaec6e04a56cf24cf4fa8ef49d78ce8905dde1354235c9125dca6885f7ce893 @@ -533,8 +533,8 @@ F ext/wasm/testing1.js 7cd8ab255c238b030d928755ae8e91e7d90a12f2ae601b1b8f7827aaa F ext/wasm/testing2.html a66951c38137ff1d687df79466351f3c734fa9c6d9cce71d3cf97c291b2167e3 F ext/wasm/testing2.js 25584bcc30f19673ce13a6f301f89f8820a59dfe044e0c4f2913941f4097fe3c F ext/wasm/wasmfs.make 21a5cf297954a689e0dc2a95299ae158f681cae5e90c10b99d986097815fd42d -F ext/wasm/x-sync-async.html 283539e4fcca8c60fea18dbf1f1c0df168340145a19123f8fd5b70f41291b36f -F ext/wasm/x-sync-async.js 42da502ea0b89bfa226c7ac7555c0c87d4ab8a10221ea6fadb4f7877c26a5137 +F ext/wasm/x-sync-async.html 717b0d3bee96e49cbd36731bead497ab27a8bf3a3b23dd11e40e61d4ac9e8b80 +F ext/wasm/x-sync-async.js 05c0b49adae0600c5ad12f3325e0873ab1f07b99c2bb017f32b50a4f701490f1 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60 @@ -2030,8 +2030,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P afb79050e635f3c698e51f06c346cbf23b096cfda7d0f1d8e68514ea0c25b7b7 -R dbdee404fc63f84ef61ddb4fd79c0ad1 +P 38da059b472415da52f57de7332fbeb8a91e3add1f4be3ff9c1924b52672f77c +R 8887512b2075b067b4aa7969d08f1316 U stephan -Z 67d0abd2dfaa993776fbe8c511ee53f5 +Z 1bf4c438e48bd1dc1717362060b2f357 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 7a184bb350..7d2261af8e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -38da059b472415da52f57de7332fbeb8a91e3add1f4be3ff9c1924b52672f77c \ No newline at end of file +44db9132145b3072488ea91db53f6c06be74544beccad5fd07efd22c0f03dc04 \ No newline at end of file