https://github.com/rhashimoto/wa-sqlite/blob/master/src/examples/OriginPrivateFileSystemVFS.js
for demonstrating how to use the OPFS APIs.
+
+ This file is to be loaded as a Worker. It does not have any direct
+ access to the sqlite3 JS/WASM bits, so any bits which it needs (most
+ notably SQLITE_xxx integer codes) have to be imported into it via an
+ initialization process.
*/
'use strict';
-(function(){
- const toss = function(...args){throw new Error(args.join(' '))};
- if(self.window === self){
- toss("This code cannot run from the main thread.",
- "Load it as a Worker from a separate Worker.");
- }else if(!navigator.storage.getDirectory){
- toss("This API requires navigator.storage.getDirectory.");
- }
- const logPrefix = "OPFS worker:";
- const log = (...args)=>{
- console.log(logPrefix,...args);
- };
- const warn = (...args)=>{
- console.warn(logPrefix,...args);
- };
- const error = (...args)=>{
- console.error(logPrefix,...args);
- };
+const toss = function(...args){throw new Error(args.join(' '))};
+if(self.window === self){
+ toss("This code cannot run from the main thread.",
+ "Load it as a Worker from a separate Worker.");
+}else if(!navigator.storage.getDirectory){
+ toss("This API requires navigator.storage.getDirectory.");
+}
+/**
+ Will hold state copied to this object from the syncronous side of
+ this API.
+*/
+const state = Object.create(null);
+/**
+ verbose:
- warn("This file is very much experimental and under construction.",self.location.pathname);
- const wMsg = (type,payload)=>postMessage({type,payload});
+ 0 = no logging output
+ 1 = only errors
+ 2 = warnings and errors
+ 3 = debug, warnings, and errors
+*/
+state.verbose = 2;
- const state = Object.create(null);
- /*state.opSab;
- state.sabIO;
- state.opBuf;
- state.opIds;
- state.rootDir;*/
- /**
- Map of sqlite3_file pointers (integers) to metadata related to a
- given OPFS file handles. The pointers are, in this side of the
- interface, opaque file handle IDs provided by the synchronous
- part of this constellation. Each value is an object with a structure
- demonstrated in the xOpen() impl.
- */
- state.openFiles = Object.create(null);
+const __logPrefix = "OPFS asyncer:";
+const log = (...args)=>{
+ if(state.verbose>2) console.log(__logPrefix,...args);
+};
+const warn = (...args)=>{
+ if(state.verbose>1) console.warn(__logPrefix,...args);
+};
+const error = (...args)=>{
+ if(state.verbose) console.error(__logPrefix,...args);
+};
- /**
- Map of dir names to FileSystemDirectoryHandle objects.
- */
- state.dirCache = new Map;
+warn("This file is very much experimental and under construction.",self.location.pathname);
- const __splitPath = (absFilename)=>{
- const a = absFilename.split('/').filter((v)=>!!v);
- return [a, a.pop()];
- };
- /**
- Takes the absolute path to a filesystem element. Returns an array
- of [handleOfContainingDir, filename]. If the 2nd argument is
- truthy then each directory element leading to the file is created
- along the way. Throws if any creation or resolution fails.
- */
- const getDirForPath = async function f(absFilename, createDirs = false){
- const url = new URL(
- absFilename, 'file://xyz'
- ) /* use URL to resolve path pieces such as a/../b */;
- const [path, filename] = __splitPath(url.pathname);
- const allDirs = path.join('/');
- let dh = state.dirCache.get(allDirs);
- if(!dh){
- dh = state.rootDir;
- for(const dirName of path){
- if(dirName){
- dh = await dh.getDirectoryHandle(dirName, {create: !!createDirs});
- }
+/**
+ Map of sqlite3_file pointers (integers) to metadata related to a
+ given OPFS file handles. The pointers are, in this side of the
+ interface, opaque file handle IDs provided by the synchronous
+ part of this constellation. Each value is an object with a structure
+ demonstrated in the xOpen() impl.
+*/
+const __openFiles = Object.create(null);
+
+/**
+ Map of dir names to FileSystemDirectoryHandle objects.
+*/
+const __dirCache = new Map;
+
+/**
+ Takes the absolute path to a filesystem element. Returns an array
+ of [handleOfContainingDir, filename]. If the 2nd argument is
+ truthy then each directory element leading to the file is created
+ along the way. Throws if any creation or resolution fails.
+*/
+const getDirForPath = async function f(absFilename, createDirs = false){
+ const url = new URL(
+ absFilename, 'file://xyz'
+ ) /* use URL to resolve path pieces such as a/../b */;
+ const path = url.pathname.split('/').filter((v)=>!!v);
+ const filename = path.pop();
+ const allDirs = '/'+path.join('/');
+ let dh = __dirCache.get(allDirs);
+ if(!dh){
+ dh = state.rootDir;
+ for(const dirName of path){
+ if(dirName){
+ dh = await dh.getDirectoryHandle(dirName, {create: !!createDirs});
}
- state.dirCache.set(allDirs, dh);
}
- return [dh, filename];
- };
+ __dirCache.set(allDirs, dh);
+ }
+ return [dh, filename];
+};
-
- /**
- Generates a random ASCII string len characters long, intended for
- use as a temporary file name.
- */
- const randomFilename = function f(len=16){
- if(!f._chars){
- f._chars = "abcdefghijklmnopqrstuvwxyz"+
- "ABCDEFGHIJKLMNOPQRSTUVWXYZ"+
- "012346789";
- f._n = f._chars.length;
- }
- const a = [];
- let i = 0;
- for( ; i < len; ++i){
- const ndx = Math.random() * (f._n * 64) % f._n | 0;
- a[i] = f._chars[ndx];
- }
- return a.join('');
- };
- const storeAndNotify = (opName, value)=>{
- log(opName+"() is notify()ing w/ value:",value);
- Atomics.store(state.opBuf, state.opIds[opName], value);
- Atomics.notify(state.opBuf, state.opIds[opName]);
- };
+/**
+ Stores the given value at the array index reserved for the given op
+ and then Atomics.notify()'s it.
+*/
+const storeAndNotify = (opName, value)=>{
+ log(opName+"() is notify()ing w/ value:",value);
+ Atomics.store(state.opBuf, state.opIds[opName], value);
+ Atomics.notify(state.opBuf, state.opIds[opName]);
+};
- const isInt32 = function(n){
- return ('bigint'!==typeof n /*TypeError: can't convert BigInt to number*/)
- && !!(n===(n|0) && n<=2147483647 && n>=-2147483648);
- };
- const affirm32Bits = function(n){
- return isInt32(n) || toss("Number is too large (>31 bits):",n);
- };
+const isInt32 = function(n){
+ return ('bigint'!==typeof n /*TypeError: can't convert BigInt to number*/)
+ && !!(n===(n|0) && n<=2147483647 && n>=-2147483648);
+};
+const affirm32Bits = function(n){
+ return isInt32(n) || toss("Number is too large (>31 bits) (FIXME!):",n);
+};
- const ioMethods = {
- xAccess: async function({filename, exists, readWrite}){
- log("xAccess(",arguments,")");
- const rc = 1;
- storeAndNotify('xAccess', rc);
- },
- xClose: async function(fid){
- const opName = 'xClose';
- log(opName+"(",arguments[0],")");
- log("state.openFiles",state.openFiles);
- const fh = state.openFiles[fid];
- if(fh){
- delete state.openFiles[fid];
- //await fh.close();
- if(fh.accessHandle) await fh.accessHandle.close();
- if(fh.deleteOnClose){
- try{
- await fh.dirHandle.removeEntry(fh.filenamePart);
- }
- catch(e){
- warn("Ignoring dirHandle.removeEntry() failure of",fh);
- }
- }
- log("state.openFiles",state.openFiles);
- storeAndNotify(opName, 0);
- }else{
- storeAndNotify(opName, state.errCodes.NotFound);
- }
- },
- xDelete: async function(filename){
- log("xDelete(",arguments,")");
- storeAndNotify('xClose', 0);
- },
- xFileSize: async function(fid){
- log("xFileSize(",arguments,")");
- const fh = state.openFiles[fid];
- const sz = await fh.getSize();
- affirm32Bits(sz);
- storeAndNotify('xFileSize', sz | 0);
- },
- xOpen: async function({
- fid/*sqlite3_file pointer*/, sab/*file-specific SharedArrayBuffer*/,
- filename,
- fileType = undefined /*mainDb, mainJournal, etc.*/,
- create = false, readOnly = false, deleteOnClose = false,
- }){
- const opName = 'xOpen';
- try{
- if(create) readOnly = false;
- log(opName+"(",arguments[0],")");
-
- let hDir, filenamePart, hFile;
- try {
- [hDir, filenamePart] = await getDirForPath(filename, !!create);
- }catch(e){
- storeAndNotify(opName, state.errCodes.NotFound);
- return;
- }
- hFile = await hDir.getFileHandle(filenamePart, {create: !!create});
- log(opName,"filenamePart =",filenamePart, 'hDir =',hDir);
- const fobj = state.openFiles[fid] = Object.create(null);
- fobj.filenameAbs = filename;
- fobj.filenamePart = filenamePart;
- fobj.dirHandle = hDir;
- fobj.fileHandle = hFile;
- fobj.accessHandle = undefined;
- fobj.fileType = fileType;
- fobj.sab = sab;
- fobj.create = !!create;
- fobj.readOnly = !!readOnly;
- fobj.deleteOnClose = !!deleteOnClose;
+/**
+ Throws if fh is a file-holding object which is flagged as read-only.
+*/
+const affirmNotRO = function(opName,fh){
+ if(fh.readOnly) toss(opName+"(): File is read-only: "+fh.filenameAbs);
+};
- /**
- wa-sqlite, at this point, grabs a SyncAccessHandle and
- assigns it to the accessHandle prop of the file state
- object, but it's unclear why it does that.
- */
- storeAndNotify(opName, 0);
+/**
+ Asynchronous wrappers for sqlite3_vfs and sqlite3_io_methods
+ methods. Maintenance reminder: members are in alphabetical order
+ to simplify finding them.
+*/
+const vfsAsyncImpls = {
+ xAccess: async function({filename, exists, readWrite}){
+ warn("xAccess(",arguments[0],") is TODO");
+ const rc = state.sq3Codes.SQLITE_IOERR;
+ storeAndNotify('xAccess', rc);
+ },
+ xClose: async function(fid){
+ const opName = 'xClose';
+ log(opName+"(",arguments[0],")");
+ const fh = __openFiles[fid];
+ if(fh){
+ delete __openFiles[fid];
+ if(fh.accessHandle) await fh.accessHandle.close();
+ if(fh.deleteOnClose){
+ try{ await fh.dirHandle.removeEntry(fh.filenamePart) }
+ catch(e){ warn("Ignoring dirHandle.removeEntry() failure of",fh,e) }
+ }
+ storeAndNotify(opName, 0);
+ }else{
+ storeAndNotify(opName, state.sq3Codes.SQLITE_NOFOUND);
+ }
+ },
+ xDelete: async function({filename, syncDir/*ignored*/}){
+ log("xDelete(",arguments[0],")");
+ try {
+ const [hDir, filenamePart] = await getDirForPath(filename, false);
+ await hDir.removeEntry(filenamePart);
+ }catch(e){
+ /* Ignoring: _presumably_ the file can't be found. */
+ }
+ storeAndNotify('xDelete', 0);
+ },
+ xFileSize: async function(fid){
+ log("xFileSize(",arguments,")");
+ const fh = __openFiles[fid];
+ let sz;
+ try{
+ sz = await fh.accessHandle.getSize();
+ fh.sabViewFileSize.setBigInt64(0, BigInt(sz));
+ sz = 0;
+ }catch(e){
+ error("xFileSize():",e, fh);
+ sz = state.sq3Codes.SQLITE_IOERR;
+ }
+ storeAndNotify('xFileSize', sz);
+ },
+ xOpen: async function({
+ fid/*sqlite3_file pointer*/,
+ sab/*file-specific SharedArrayBuffer*/,
+ filename,
+ fileType = undefined /*mainDb, mainJournal, etc.*/,
+ create = false, readOnly = false, deleteOnClose = false
+ }){
+ const opName = 'xOpen';
+ try{
+ if(create) readOnly = false;
+ log(opName+"(",arguments[0],")");
+ let hDir, filenamePart;
+ try {
+ [hDir, filenamePart] = await getDirForPath(filename, !!create);
}catch(e){
- error(opName,e);
- storeAndNotify(opName, state.errCodes.IO);
+ storeAndNotify(opName, state.sql3Codes.SQLITE_NOTFOUND);
+ return;
}
- },
- xRead: async function({fid,n,offset}){
- log("xRead(",arguments,")");
- affirm32Bits(n + offset);
- const fh = state.openFiles[fid];
- storeAndNotify('xRead',fid);
- },
- xSleep: async function f({ms}){
- log("xSleep(",arguments[0],")");
- await new Promise((resolve)=>{
- setTimeout(()=>resolve(), ms);
- }).finally(()=>storeAndNotify('xSleep',0));
- },
- xSync: async function({fid}){
- log("xSync(",arguments,")");
- const fh = state.openFiles[fid];
- await fh.flush();
- storeAndNotify('xSync',fid);
- },
- xTruncate: async function({fid,size}){
- log("xTruncate(",arguments,")");
- affirm32Bits(size);
- const fh = state.openFiles[fid];
- fh.truncate(size);
- storeAndNotify('xTruncate',fid);
- },
- xWrite: async function({fid,src,n,offset}){
- log("xWrite(",arguments,")");
- const fh = state.openFiles[fid];
- storeAndNotify('xWrite',fid);
+ const hFile = await hDir.getFileHandle(filenamePart, {create: !!create});
+ log(opName,"filenamePart =",filenamePart, 'hDir =',hDir);
+ const fobj = __openFiles[fid] = Object.create(null);
+ fobj.filenameAbs = filename;
+ fobj.filenamePart = filenamePart;
+ fobj.dirHandle = hDir;
+ fobj.fileHandle = hFile;
+ fobj.fileType = fileType;
+ fobj.sab = sab;
+ fobj.sabViewFileSize = new DataView(sab,state.fbInt64Offset,8);
+ fobj.create = !!create;
+ fobj.readOnly = !!readOnly;
+ fobj.deleteOnClose = !!deleteOnClose;
+ /**
+ wa-sqlite, at this point, grabs a SyncAccessHandle and
+ assigns it to the accessHandle prop of the file state
+ object, but only for certain cases and it's unclear why it
+ places that limitation on it.
+ */
+ fobj.accessHandle = await hFile.createSyncAccessHandle();
+ storeAndNotify(opName, 0);
+ }catch(e){
+ error(opName,e);
+ storeAndNotify(opName, state.sq3Codes.SQLITE_IOERR);
}
- };
-
- const onReady = function(){
- self.onmessage = async function({data}){
- log("self.onmessage",data);
- switch(data.type){
- case 'init':{
- const opt = data.payload;
- state.opSab = opt.opSab;
- state.opBuf = new Int32Array(state.opSab);
- state.opIds = opt.opIds;
- state.errCodes = opt.errCodes;
- state.sq3Codes = opt.sq3Codes;
- Object.keys(ioMethods).forEach((k)=>{
- if(!state.opIds[k]){
- toss("Maintenance required: missing state.opIds[",k,"]");
- }
- });
- log("init state",state);
- break;
- }
- default:{
- const m = ioMethods[data.type] || toss("Unknown message type:",data.type);
- try {
- await m(data.payload);
- }catch(e){
- error("Error handling",data.type+"():",e);
- storeAndNotify(data.type, -99);
+ },
+ xRead: async function({fid,n,offset}){
+ log("xRead(",arguments[0],")");
+ let rc = 0;
+ const fh = __openFiles[fid];
+ try{
+ const aRead = new Uint8array(fh.sab, n);
+ const nRead = fh.accessHandle.read(aRead, {at: offset});
+ if(nRead < n){/* Zero-fill remaining bytes */
+ new Uint8array(fh.sab).fill(0, nRead, n);
+ rc = state.sq3Codes.SQLITE_IOERR_SHORT_READ;
+ }
+ }catch(e){
+ error("xRead() failed",e,fh);
+ rc = state.sq3Codes.SQLITE_IOERR_READ;
+ }
+ storeAndNotify('xRead',rc);
+ },
+ xSleep: async function f(ms){
+ log("xSleep(",ms,")");
+ await new Promise((resolve)=>{
+ setTimeout(()=>resolve(), ms);
+ }).finally(()=>storeAndNotify('xSleep',0));
+ },
+ xSync: async function({fid,flags/*ignored*/}){
+ log("xSync(",arguments[0],")");
+ const fh = __openFiles[fid];
+ if(!fh.readOnly && fh.accessHandle) await fh.accessHandle.flush();
+ storeAndNotify('xSync',0);
+ },
+ xTruncate: async function({fid,size}){
+ log("xTruncate(",arguments[0],")");
+ let rc = 0;
+ const fh = __openFiles[fid];
+ try{
+ affirmNotRO('xTruncate', fh);
+ await fh.accessHandle.truncate(size);
+ }catch(e){
+ error("xTruncate():",e,fh);
+ rc = state.sq3Codes.SQLITE_IOERR_TRUNCATE;
+ }
+ storeAndNotify('xTruncate',rc);
+ },
+ xWrite: async function({fid,src,n,offset}){
+ log("xWrite(",arguments[0],")");
+ let rc;
+ try{
+ const fh = __openFiles[fid];
+ affirmNotRO('xWrite', fh);
+ const nOut = fh.accessHandle.write(new UInt8Array(fh.sab, 0, n), {at: offset});
+ rc = (nOut===n) ? 0 : state.sq3Codes.SQLITE_IOERR_WRITE;
+ }catch(e){
+ error("xWrite():",e,fh);
+ rc = state.sq3Codes.SQLITE_IOERR_WRITE;
+ }
+ storeAndNotify('xWrite',rc);
+ }
+};
+
+navigator.storage.getDirectory().then(function(d){
+ const wMsg = (type)=>postMessage({type});
+ state.rootDir = d;
+ log("state.rootDir =",state.rootDir);
+ self.onmessage = async function({data}){
+ log("self.onmessage()",data);
+ switch(data.type){
+ case 'init':{
+ /* Receive shared state from synchronous partner */
+ const opt = data.payload;
+ state.verbose = opt.verbose ?? 2;
+ state.fileBufferSize = opt.fileBufferSize;
+ state.fbInt64Offset = opt.fbInt64Offset;
+ state.opSab = opt.opSab;
+ state.opBuf = new Int32Array(state.opSab);
+ state.opIds = opt.opIds;
+ state.sq3Codes = opt.sq3Codes;
+ Object.keys(vfsAsyncImpls).forEach((k)=>{
+ if(!Number.isFinite(state.opIds[k])){
+ toss("Maintenance required: missing state.opIds[",k,"]");
}
- break;
+ });
+ log("init state",state);
+ wMsg('inited');
+ break;
+ }
+ default:{
+ let err;
+ const m = vfsAsyncImpls[data.type] || toss("Unknown message type:",data.type);
+ try {
+ await m(data.payload).catch((e)=>err=e);
+ }catch(e){
+ err = e;
}
- }
- };
- wMsg('ready');
+ if(err){
+ error("Error handling",data.type+"():",e);
+ storeAndNotify(data.type, state.sq3Codes.SQLITE_ERROR);
+ }
+ break;
+ }
+ }
};
-
- navigator.storage.getDirectory().then(function(d){
- state.rootDir = d;
- log("state.rootDir =",state.rootDir);
- onReady();
- });
-
-})();
+ wMsg('loaded');
+});
<div>This is an experiment in wrapping the
asynchronous OPFS APIs behind a fully synchronous proxy. It is
very much incomplete, under construction, and experimental.
- See the dev console for all output.
+ <strong>See the dev console for all output.</strong>
</div>
<div id='test-output'>
</div>
- <!--script src="common/whwasmutil.js"></script-->
- <!--script src="common/SqliteTestUtil.js"></script-->
- <script>
-(function(){
- new Worker("x-sync-async.js");
-})();
- </script>
+ <script>new Worker("x-sync-async.js");</script>
</body>
</html>
+/*
+ 2022-09-17
+
+ The author disclaims copyright to this source code. In place of a
+ legal notice, here is a blessing:
+
+ * May you do good and not evil.
+ * May you find forgiveness for yourself and forgive others.
+ * May you share freely, never taking more than you give.
+
+ ***********************************************************************
+
+ A EXTREMELY INCOMPLETE and UNDER CONSTRUCTION experiment for OPFS.
+ This file holds the synchronous half of an sqlite3_vfs
+ implementation which proxies, in a synchronous fashion, the
+ asynchronous OPFS APIs using a second Worker.
+*/
'use strict';
-const doAtomicsStuff = function(sqlite3){
+/**
+ This function is a placeholder for use in development. When
+ working, this will be moved into a file named
+ api/sqlite3-api-opfs.js, or similar, and hooked in to the
+ sqlite-api build construct.
+*/
+const initOpfsVfs = function(sqlite3){
+ const toss = function(...args){throw new Error(args.join(' '))};
const logPrefix = "OPFS syncer:";
const log = (...args)=>{
console.log(logPrefix,...args);
const error = (...args)=>{
console.error(logPrefix,...args);
};
+ warn("This file is very much experimental and under construction.",self.location.pathname);
+
+ const capi = sqlite3.capi;
+ const wasm = capi.wasm;
+ const sqlite3_vfs = capi.sqlite3_vfs
+ || toss("Missing sqlite3.capi.sqlite3_vfs object.");
+ const sqlite3_file = capi.sqlite3_file
+ || toss("Missing sqlite3.capi.sqlite3_file object.");
+ const sqlite3_io_methods = capi.sqlite3_io_methods
+ || toss("Missing sqlite3.capi.sqlite3_io_methods object.");
+ const StructBinder = sqlite3.StructBinder || toss("Missing sqlite3.StructBinder.");
+
const W = new Worker("sqlite3-opfs-async-proxy.js");
const wMsg = (type,payload)=>W.postMessage({type,payload});
- warn("This file is very much experimental and under construction.",self.location.pathname);
/**
State which we send to the async-api Worker or share with it.
This object must initially contain only cloneable or sharable
- objects. After the worker's "ready" message arrives, other types
+ objects. After the worker's "inited" message arrives, other types
of data may be added to it.
*/
const state = Object.create(null);
+ state.verbose = 3;
+ state.fileBufferSize = 1024 * 64 + 8 /* size of fileHandle.sab. 64k = max sqlite3 page size */;
+ state.fbInt64Offset = state.fileBufferSize - 8 /*spot in fileHandle.sab to store an int64*/;
state.opIds = Object.create(null);
- state.opIds.xAccess = 1;
- state.opIds.xClose = 2;
- state.opIds.xDelete = 3;
- state.opIds.xFileSize = 4;
- state.opIds.xOpen = 5;
- state.opIds.xRead = 6;
- state.opIds.xSync = 7;
- state.opIds.xTruncate = 8;
- state.opIds.xWrite = 9;
- state.opIds.xSleep = 10;
- state.opIds.xBlock = 99 /* to block worker while this code is still handling something */;
- state.opSab = new SharedArrayBuffer(64);
- state.fileBufferSize = 1024 * 65 /* 64k = max sqlite3 page size */;
- /* TODO: use SQLITE_xxx err codes. */
- state.errCodes = Object.create(null);
- state.errCodes.Error = -100;
- state.errCodes.IO = -101;
- state.errCodes.NotFound = -102;
- state.errCodes.Misuse = -103;
-
- // TODO: add any SQLITE_xxx symbols we need here.
+ {
+ let i = 0;
+ state.opIds.xAccess = i++;
+ state.opIds.xClose = i++;
+ state.opIds.xDelete = i++;
+ state.opIds.xFileSize = i++;
+ state.opIds.xOpen = i++;
+ state.opIds.xRead = i++;
+ state.opIds.xSleep = i++;
+ state.opIds.xSync = i++;
+ state.opIds.xTruncate = i++;
+ state.opIds.xWrite = i++;
+ state.opSab = new SharedArrayBuffer(i * 4);
+ }
+
state.sq3Codes = Object.create(null);
-
- const isWorkerErrCode = (n)=>(n<=state.errCodes.Error);
+ state.sq3Codes._reverse = Object.create(null);
+ [ // SQLITE_xxx constants to export to the async worker counterpart...
+ 'SQLITE_ERROR', 'SQLITE_IOERR',
+ 'SQLITE_NOTFOUND', 'SQLITE_MISUSE',
+ 'SQLITE_IOERR_READ', 'SQLITE_IOERR_SHORT_READ',
+ 'SQLITE_IOERR_WRITE', 'SQLITE_IOERR_FSYNC',
+ 'SQLITE_IOERR_TRUNCATE', 'SQLITE_IOERR_DELETE',
+ 'SQLITE_IOERR_ACCESS', 'SQLITE_IOERR_CLOSE'
+ ].forEach(function(k){
+ state.sq3Codes[k] = capi[k] || toss("Maintenance required: not found:",k);
+ state.sq3Codes._reverse[capi[k]] = k;
+ });
+
+ const isWorkerErrCode = (n)=>!!state.sq3Codes._reverse[n];
const opStore = (op,val=-1)=>Atomics.store(state.opBuf, state.opIds[op], val);
const opWait = (op,val=-1)=>Atomics.wait(state.opBuf, state.opIds[op], val);
+ /**
+ Runs the given operation in the async worker counterpart, waits
+ for its response, and returns the result which the async worker
+ writes to the given op's index in state.opBuf. The 2nd argument
+ must be a single object or primitive value, depending on the
+ given operation's signature in the async API counterpart.
+ */
const opRun = (op,args)=>{
opStore(op);
wMsg(op, args);
});
};
+ /**
+ Generates a random ASCII string len characters long, intended for
+ use as a temporary file name.
+ */
+ const randomFilename = function f(len=16){
+ if(!f._chars){
+ f._chars = "abcdefghijklmnopqrstuvwxyz"+
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZ"+
+ "012346789";
+ f._n = f._chars.length;
+ }
+ const a = [];
+ let i = 0;
+ for( ; i < len; ++i){
+ const ndx = Math.random() * (f._n * 64) % f._n | 0;
+ a[i] = f._chars[ndx];
+ }
+ return a.join('');
+ };
+
+ /**
+ Map of sqlite3_file pointers to objects constructed by xOpen().
+ */
+ const __openFiles = Object.create(null);
+
+ const pDVfs = capi.sqlite3_vfs_find(null)/*pointer to default VFS*/;
+ const dVfs = pDVfs
+ ? new sqlite3_vfs(pDVfs)
+ : null /* dVfs will be null when sqlite3 is built with
+ SQLITE_OS_OTHER. Though we cannot currently handle
+ that case, the hope is to eventually be able to. */;
+ const opfsVfs = new sqlite3_vfs();
+ const opfsIoMethods = new sqlite3_io_methods();
+ opfsVfs.$iVersion = 2/*yes, two*/;
+ opfsVfs.$szOsFile = capi.sqlite3_file.structInfo.sizeof;
+ opfsVfs.$mxPathname = 1024/*sure, why not?*/;
+ opfsVfs.$zName = wasm.allocCString("opfs");
+ opfsVfs.ondispose = [
+ '$zName', opfsVfs.$zName,
+ 'cleanup dVfs', ()=>(dVfs ? dVfs.dispose() : null)
+ ];
+ if(dVfs){
+ opfsVfs.$xSleep = dVfs.$xSleep;
+ opfsVfs.$xRandomness = dVfs.$xRandomness;
+ }
+ // All C-side memory of opfsVfs is zeroed out, but just to be explicit:
+ opfsVfs.$xDlOpen = opfsVfs.$xDlError = opfsVfs.$xDlSym = opfsVfs.$xDlClose = null;
+ /**
+ Pedantic sidebar about opfsVfs.ondispose: the entries in that array
+ are items to clean up when opfsVfs.dispose() is called, but in this
+ environment it will never be called. The VFS instance simply
+ hangs around until the WASM module instance is cleaned up. We
+ "could" _hypothetically_ clean it up by "importing" an
+ sqlite3_os_end() impl into the wasm build, but the shutdown order
+ of the wasm engine and the JS one are undefined so there is no
+ guaranty that the opfsVfs instance would be available in one
+ environment or the other when sqlite3_os_end() is called (_if_ it
+ gets called at all in a wasm build, which is undefined).
+ */
+
+ /**
+ Impls for the sqlite3_io_methods methods. Maintenance reminder:
+ members are in alphabetical order to simplify finding them.
+ */
+ const ioSyncWrappers = {
+ xCheckReservedLock: function(pFile,pOut){
+ // Exclusive lock is automatically acquired when opened
+ //warn("xCheckReservedLock(",arguments,") is a no-op");
+ wasm.setMemValue(pOut,1,'i32');
+ return 0;
+ },
+ xClose: function(pFile){
+ let rc = 0;
+ const f = __openFiles[pFile];
+ if(f){
+ delete __openFiles[pFile];
+ rc = opRun('xClose', pFile);
+ if(f.sq3File) f.sq3File.dispose();
+ }
+ return rc;
+ },
+ xDeviceCharacteristics: function(pFile){
+ //debug("xDeviceCharacteristics(",pFile,")");
+ return capi.SQLITE_IOCAP_UNDELETABLE_WHEN_OPEN;
+ },
+ xFileControl: function(pFile,op,pArg){
+ //debug("xFileControl(",arguments,") is a no-op");
+ return capi.SQLITE_NOTFOUND;
+ },
+ xFileSize: function(pFile,pSz64){
+ const rc = opRun('xFileSize', pFile);
+ if(!isWorkerErrCode(rc)){
+ const f = __openFiles[pFile];
+ wasm.setMemValue(pSz64, f.sabViewFileSize.getBigInt64(0) ,'i64');
+ }
+ return rc;
+ },
+ xLock: function(pFile,lockType){
+ //2022-09: OPFS handles lock when opened
+ //warn("xLock(",arguments,") is a no-op");
+ return 0;
+ },
+ xRead: function(pFile,pDest,n,offset){
+ /* int (*xRead)(sqlite3_file*, void*, int iAmt, sqlite3_int64 iOfst) */
+ const f = __opfsHandles[pFile];
+ try {
+ // FIXME(?): block until we finish copying the xRead result buffer. How?
+ let rc = opRun('xRead',{fid:pFile, n, offset});
+ if(0!==rc) return rc;
+ let i = 0;
+ for(; i < n; ++i) wasm.setMemValue(pDest + i, f.sabView[i]);
+ }catch(e){
+ error("xRead(",arguments,") failed:",e,f);
+ rc = capi.SQLITE_IOERR_READ;
+ }
+ return rc;
+ },
+ xSync: function(pFile,flags){
+ return opRun('xSync', {fid:pFile, flags});
+ },
+ xTruncate: function(pFile,sz64){
+ return opRun('xTruncate', {fid:pFile, size: sz64});
+ },
+ xUnlock: function(pFile,lockType){
+ //2022-09: OPFS handles lock when opened
+ //warn("xUnlock(",arguments,") is a no-op");
+ return 0;
+ },
+ xWrite: function(pFile,pSrc,n,offset){
+ /* int (*xWrite)(sqlite3_file*, const void*, int iAmt, sqlite3_int64 iOfst) */
+ const f = __opfsHandles[pFile];
+ try {
+ let i = 0;
+ // FIXME(?): block from here until we finish the xWrite. How?
+ for(; i < n; ++i) f.sabView[i] = wasm.getMemValue(pSrc+i);
+ return opRun('xWrite',{fid:pFile, n, offset});
+ }catch(e){
+ error("xWrite(",arguments,") failed:",e,f);
+ return capi.SQLITE_IOERR_WRITE;
+ }
+ }
+ }/*ioSyncWrappers*/;
+
+ /**
+ Impls for the sqlite3_vfs methods. Maintenance reminder: members
+ are in alphabetical order to simplify finding them.
+ */
const vfsSyncWrappers = {
- xOpen: function f(pFile, name, flags, outFlags = {}){
+ // TODO: xAccess
+ xCurrentTime: function(pVfs,pOut){
+ /* If it turns out that we need to adjust for timezone, see:
+ https://stackoverflow.com/a/11760121/1458521 */
+ wasm.setMemValue(pOut, 2440587.5 + (new Date().getTime()/86400000),
+ 'double');
+ return 0;
+ },
+ xCurrentTimeInt64: function(pVfs,pOut){
+ // TODO: confirm that this calculation is correct
+ wasm.setMemValue(pOut, (2440587.5 * 86400000) + new Date().getTime(),
+ 'i64');
+ return 0;
+ },
+ xDelete: function(pVfs, zName, doSyncDir){
+ return opRun('xDelete', {filename: wasm.cstringToJs(zName), syncDir: doSyncDir});
+ },
+ xFullPathname: function(pVfs,zName,nOut,pOut){
+ /* Until/unless we have some notion of "current dir"
+ in OPFS, simply copy zName to pOut... */
+ const i = wasm.cstrncpy(pOut, zName, nOut);
+ return i<nOut ? 0 : capi.SQLITE_CANTOPEN
+ /*CANTOPEN is required by the docs but SQLITE_RANGE would be a closer match*/;
+ },
+ xGetLastError: function(pVfs,nOut,pOut){
+ /* TODO: store exception.message values from the async
+ partner in a dedicated SharedArrayBuffer, noting that we'd have
+ to encode them... TextEncoder can do that for us. */
+ warn("OPFS xGetLastError() has nothing sensible to return.");
+ return 0;
+ },
+ xOpen: function f(pVfs, zName, pFile, flags, pOutFlags){
if(!f._){
f._ = {
- // TODO: map openFlags to args.fileType names.
+ fileTypes: {
+ SQLITE_OPEN_MAIN_DB: 'mainDb',
+ SQLITE_OPEN_MAIN_JOURNAL: 'mainJournal',
+ SQLITE_OPEN_TEMP_DB: 'tempDb',
+ SQLITE_OPEN_TEMP_JOURNAL: 'tempJournal',
+ SQLITE_OPEN_TRANSIENT_DB: 'transientDb',
+ SQLITE_OPEN_SUBJOURNAL: 'subjournal',
+ SQLITE_OPEN_SUPER_JOURNAL: 'superJournal',
+ SQLITE_OPEN_WAL: 'wal'
+ },
+ getFileType: function(filename,oflags){
+ const ft = f._.fileTypes;
+ for(let k of Object.keys(ft)){
+ if(oflags & capi[k]) return ft[k];
+ }
+ warn("Cannot determine fileType based on xOpen() flags for file",filename);
+ return '???';
+ }
};
}
+ if(0===zName){
+ zName = randomFilename();
+ }else if('number'===typeof zName){
+ zName = wasm.cstringToJs(zName);
+ }
const args = Object.create(null);
args.fid = pFile;
- args.filename = name;
+ args.filename = zName;
args.sab = new SharedArrayBuffer(state.fileBufferSize);
- args.fileType = undefined /*TODO: populate based on SQLITE_OPEN_xxx */;
- // TODO: populate args object based on flags:
- // args.create, args.readOnly, args.deleteOnClose
- args.create = true;
- args.deleteOnClose = true;
+ args.fileType = f._.getFileType(args.filename, flags);
+ args.create = !!(flags & capi.SQLITE_OPEN_CREATE);
+ args.deleteOnClose = !!(flags & capi.SQLITE_OPEN_DELETEONCLOSE);
+ args.readOnly = !!(flags & capi.SQLITE_OPEN_READONLY);
const rc = opRun('xOpen', args);
if(!rc){
- outFlags.readOnly = args.readOnly;
+ /* Recall that sqlite3_vfs::xClose() will be called, even on
+ error, unless pFile->pMethods is NULL. */
+ if(args.readOnly){
+ wasm.setMemValue(pOutFlags, capi.SQLITE_OPEN_READONLY, 'i32');
+ }
+ __openFiles[pFile] = args;
+ args.sabView = new Uint8Array(args.sab);
+ args.sabViewFileSize = new DataView(args.sab, state.fbInt64Offset, 8);
+ args.sq3File = new sqlite3_file(pFile);
+ args.sq3File.$pMethods = opfsIoMethods.pointer;
args.ba = new Uint8Array(args.sab);
- state.openFiles[pFile] = args;
}
return rc;
- },
- xClose: function(pFile){
- let rc = 0;
- if(state.openFiles[pFile]){
- delete state.openFiles[pFile];
- rc = opRun('xClose', pFile);
- }
- return rc;
- }
- };
+ }/*xOpen()*/
+ }/*vfsSyncWrappers*/;
+ if(!opfsVfs.$xRandomness){
+ /* If the default VFS has no xRandomness(), add a basic JS impl... */
+ vfsSyncWrappers.xRandomness = function(pVfs, nOut, pOut){
+ const heap = wasm.heap8u();
+ let i = 0;
+ for(; i < nOut; ++i) heap[pOut + i] = (Math.random()*255000) & 0xFF;
+ return i;
+ };
+ }
+ if(!opfsVfs.$xSleep){
+ /* If we can inherit an xSleep() impl from the default VFS then
+ use it, otherwise install one which is certainly less accurate
+ because it has to go round-trip through the async worker, but
+ provides the only option for a synchronous sleep() in JS. */
+ vfsSyncWrappers.xSleep = (pVfs,ms)=>opRun('xSleep',ms);
+ }
- const doSomething = function(){
+ /*
+ TODO: plug in the above functions in to opfsVfs and opfsIoMethods.
+ Code for doing so is in api/sqlite3-api-opfs.js.
+ */
+
+ const sanityCheck = async function(){
//state.ioBuf = new Uint8Array(state.sabIo);
- const fid = 37;
- let rc = vfsSyncWrappers.xOpen(fid, "/foo/bar/baz.sqlite3",0, {});
- log("open rc =",rc,"state.opBuf[xOpen] =",state.opBuf[state.opIds.xOpen]);
- if(isWorkerErrCode(rc)){
- error("open failed with code",rc);
- return;
- }
- log("xSleep()ing before close()ing...");
- opRun('xSleep',{ms: 1500});
- log("wait()ing before close()ing...");
- wait(1500).then(function(){
- rc = vfsSyncWrappers.xClose(fid);
+ const scope = wasm.scopedAllocPush();
+ const sq3File = new sqlite3_file();
+ try{
+ const fid = sq3File.pointer;
+ const openFlags = capi.SQLITE_OPEN_CREATE
+ | capi.SQLITE_OPEN_READWRITE
+ | capi.SQLITE_OPEN_DELETEONCLOSE
+ | capi.SQLITE_OPEN_MAIN_DB;
+ const pOut = wasm.scopedAlloc(8);
+ const dbFile = "/sanity/check/file";
+ let rc = vfsSyncWrappers.xOpen(opfsVfs.pointer, dbFile,
+ fid, openFlags, pOut);
+ log("open rc =",rc,"state.opBuf[xOpen] =",state.opBuf[state.opIds.xOpen]);
+ if(isWorkerErrCode(rc)){
+ error("open failed with code",rc);
+ return;
+ }
+ rc = ioSyncWrappers.xSync(sq3File.pointer, 0);
+ if(rc) toss('sync failed w/ rc',rc);
+ rc = ioSyncWrappers.xTruncate(sq3File.pointer, 1024);
+ if(rc) toss('truncate failed w/ rc',rc);
+ wasm.setMemValue(pOut,0,'i64');
+ rc = ioSyncWrappers.xFileSize(sq3File.pointer, pOut);
+ if(rc) toss('xFileSize failed w/ rc',rc);
+ log("xFileSize says:",wasm.getMemValue(pOut, 'i64'));
+ log("xSleep()ing before close()ing...");
+ opRun('xSleep',1500);
+ rc = ioSyncWrappers.xClose(fid);
log("xClose rc =",rc,"opBuf =",state.opBuf);
- });
+ log("Deleting file:",dbFile);
+ opRun('xDelete', dbFile);
+ }finally{
+ sq3File.dispose();
+ wasm.scopedAllocPop(scope);
+ }
};
+
W.onmessage = function({data}){
log("Worker.onmessage:",data);
switch(data.type){
- case 'ready':
+ case 'loaded':
+ /*Pass our config and shared state on to the async worker.*/
wMsg('init',state);
+ break;
+ case 'inited':
+ /*Indicates that the async partner has received the 'init',
+ so we now know that the state object is no longer subject to
+ being copied by a pending postMessage() call.*/
state.opBuf = new Int32Array(state.opSab);
- state.openFiles = Object.create(null);
- doSomething();
+ sanityCheck();
+ break;
+ default:
+ error("Unexpected message from the async worker:",data);
break;
}
};
-}/*doAtomicsStuff*/
+}/*initOpfsVfs*/
importScripts('sqlite3.js');
-self.sqlite3InitModule().then((EmscriptenModule)=>doAtomicsStuff(EmscriptenModule.sqlite3));
+self.sqlite3InitModule().then((EmscriptenModule)=>initOpfsVfs(EmscriptenModule.sqlite3));
-C Add\sinitial\sbits\sof\san\sexperimental\sasync-impl-via-synchronous-interface\sproxy\sintended\sto\smarshal\sOPFS\svia\ssqlite3_vfs\sAPI.
-D 2022-09-17T15:08:22.642
+C Add\sthe\sremaining\svfs/io_methods\swrappers\sto\sthe\sOPFS\ssync/async\sproxy,\sbut\smost\sare\snot\syet\stested.
+D 2022-09-17T20:50:12.684
F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
F ext/wasm/split-speedtest1-script.sh a3e271938d4d14ee49105eb05567c6a69ba4c1f1293583ad5af0cd3a3779e205 x
F ext/wasm/sql/000-mandelbrot.sql 775337a4b80938ac8146aedf88808282f04d02d983d82675bd63d9c2d97a15f0
F ext/wasm/sql/001-sudoku.sql 35b7cb7239ba5d5f193bc05ec379bcf66891bce6f2a5b3879f2f78d0917299b5
-F ext/wasm/sqlite3-opfs-async-proxy.js c42a097dfbb96abef08554b173a47788f5bc1f58c266f859ba01c1fa3ff8327d
+F ext/wasm/sqlite3-opfs-async-proxy.js 62024877ad13fdff1834581ca1951ab58bda431e4d548aaaf4506ea54f0ed2de
F ext/wasm/sqlite3-worker1-promiser.js 92b8da5f38439ffec459a8215775d30fa498bc0f1ab929ff341fc3dd479660b9
F ext/wasm/sqlite3-worker1.js 0c1e7626304543969c3846573e080c082bf43bcaa47e87d416458af84f340a9e
F ext/wasm/testing-worker1-promiser.html 6eaec6e04a56cf24cf4fa8ef49d78ce8905dde1354235c9125dca6885f7ce893
F ext/wasm/testing2.html a66951c38137ff1d687df79466351f3c734fa9c6d9cce71d3cf97c291b2167e3
F ext/wasm/testing2.js 25584bcc30f19673ce13a6f301f89f8820a59dfe044e0c4f2913941f4097fe3c
F ext/wasm/wasmfs.make 21a5cf297954a689e0dc2a95299ae158f681cae5e90c10b99d986097815fd42d
-F ext/wasm/x-sync-async.html 283539e4fcca8c60fea18dbf1f1c0df168340145a19123f8fd5b70f41291b36f
-F ext/wasm/x-sync-async.js 42da502ea0b89bfa226c7ac7555c0c87d4ab8a10221ea6fadb4f7877c26a5137
+F ext/wasm/x-sync-async.html 717b0d3bee96e49cbd36731bead497ab27a8bf3a3b23dd11e40e61d4ac9e8b80
+F ext/wasm/x-sync-async.js 05c0b49adae0600c5ad12f3325e0873ab1f07b99c2bb017f32b50a4f701490f1
F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x
F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8
F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60
F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P afb79050e635f3c698e51f06c346cbf23b096cfda7d0f1d8e68514ea0c25b7b7
-R dbdee404fc63f84ef61ddb4fd79c0ad1
+P 38da059b472415da52f57de7332fbeb8a91e3add1f4be3ff9c1924b52672f77c
+R 8887512b2075b067b4aa7969d08f1316
U stephan
-Z 67d0abd2dfaa993776fbe8c511ee53f5
+Z 1bf4c438e48bd1dc1717362060b2f357
# Remove this line to create a well-formed Fossil manifest.
-38da059b472415da52f57de7332fbeb8a91e3add1f4be3ff9c1924b52672f77c
\ No newline at end of file
+44db9132145b3072488ea91db53f6c06be74544beccad5fd07efd22c0f03dc04
\ No newline at end of file
projects / thirdparty / sqlite.git / commitdiff
