--- /dev/null
+/*
+ 2022-09-16
+
+ The author disclaims copyright to this source code. In place of a
+ legal notice, here is a blessing:
+
+ * May you do good and not evil.
+ * May you find forgiveness for yourself and forgive others.
+ * May you share freely, never taking more than you give.
+
+ ***********************************************************************
+
+ A EXTREMELY INCOMPLETE and UNDER CONSTRUCTION experiment for OPFS: a
+ Worker which manages asynchronous OPFS handles on behalf of a
+ synchronous API which controls it via a combination of Worker
+ messages, SharedArrayBuffer, and Atomics.
+
+ Highly indebted to:
+
+ https://github.com/rhashimoto/wa-sqlite/blob/master/src/examples/OriginPrivateFileSystemVFS.js
+
+ for demonstrating how to use the OPFS APIs.
+*/
+'use strict';
+(function(){
+ const toss = function(...args){throw new Error(args.join(' '))};
+ if(self.window === self){
+ toss("This code cannot run from the main thread.",
+ "Load it as a Worker from a separate Worker.");
+ }else if(!navigator.storage.getDirectory){
+ toss("This API requires navigator.storage.getDirectory.");
+ }
+ const logPrefix = "OPFS worker:";
+ const log = (...args)=>{
+ console.log(logPrefix,...args);
+ };
+ const warn = (...args)=>{
+ console.warn(logPrefix,...args);
+ };
+ const error = (...args)=>{
+ console.error(logPrefix,...args);
+ };
+
+ warn("This file is very much experimental and under construction.",self.location.pathname);
+ const wMsg = (type,payload)=>postMessage({type,payload});
+
+ const state = Object.create(null);
+ /*state.opSab;
+ state.sabIO;
+ state.opBuf;
+ state.opIds;
+ state.rootDir;*/
+ /**
+ Map of sqlite3_file pointers (integers) to metadata related to a
+ given OPFS file handles. The pointers are, in this side of the
+ interface, opaque file handle IDs provided by the synchronous
+ part of this constellation. Each value is an object with a structure
+ demonstrated in the xOpen() impl.
+ */
+ state.openFiles = Object.create(null);
+
+ /**
+ Map of dir names to FileSystemDirectoryHandle objects.
+ */
+ state.dirCache = new Map;
+
+ const __splitPath = (absFilename)=>{
+ const a = absFilename.split('/').filter((v)=>!!v);
+ return [a, a.pop()];
+ };
+ /**
+ Takes the absolute path to a filesystem element. Returns an array
+ of [handleOfContainingDir, filename]. If the 2nd argument is
+ truthy then each directory element leading to the file is created
+ along the way. Throws if any creation or resolution fails.
+ */
+ const getDirForPath = async function f(absFilename, createDirs = false){
+ const url = new URL(
+ absFilename, 'file://xyz'
+ ) /* use URL to resolve path pieces such as a/../b */;
+ const [path, filename] = __splitPath(url.pathname);
+ const allDirs = path.join('/');
+ let dh = state.dirCache.get(allDirs);
+ if(!dh){
+ dh = state.rootDir;
+ for(const dirName of path){
+ if(dirName){
+ dh = await dh.getDirectoryHandle(dirName, {create: !!createDirs});
+ }
+ }
+ state.dirCache.set(allDirs, dh);
+ }
+ return [dh, filename];
+ };
+
+
+ /**
+ Generates a random ASCII string len characters long, intended for
+ use as a temporary file name.
+ */
+ const randomFilename = function f(len=16){
+ if(!f._chars){
+ f._chars = "abcdefghijklmnopqrstuvwxyz"+
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZ"+
+ "012346789";
+ f._n = f._chars.length;
+ }
+ const a = [];
+ let i = 0;
+ for( ; i < len; ++i){
+ const ndx = Math.random() * (f._n * 64) % f._n | 0;
+ a[i] = f._chars[ndx];
+ }
+ return a.join('');
+ };
+
+ const storeAndNotify = (opName, value)=>{
+ log(opName+"() is notify()ing w/ value:",value);
+ Atomics.store(state.opBuf, state.opIds[opName], value);
+ Atomics.notify(state.opBuf, state.opIds[opName]);
+ };
+
+ const isInt32 = function(n){
+ return ('bigint'!==typeof n /*TypeError: can't convert BigInt to number*/)
+ && !!(n===(n|0) && n<=2147483647 && n>=-2147483648);
+ };
+ const affirm32Bits = function(n){
+ return isInt32(n) || toss("Number is too large (>31 bits):",n);
+ };
+
+ const ioMethods = {
+ xAccess: async function({filename, exists, readWrite}){
+ log("xAccess(",arguments,")");
+ const rc = 1;
+ storeAndNotify('xAccess', rc);
+ },
+ xClose: async function(fid){
+ const opName = 'xClose';
+ log(opName+"(",arguments[0],")");
+ log("state.openFiles",state.openFiles);
+ const fh = state.openFiles[fid];
+ if(fh){
+ delete state.openFiles[fid];
+ //await fh.close();
+ if(fh.accessHandle) await fh.accessHandle.close();
+ if(fh.deleteOnClose){
+ try{
+ await fh.dirHandle.removeEntry(fh.filenamePart);
+ }
+ catch(e){
+ warn("Ignoring dirHandle.removeEntry() failure of",fh);
+ }
+ }
+ log("state.openFiles",state.openFiles);
+ storeAndNotify(opName, 0);
+ }else{
+ storeAndNotify(opName, state.errCodes.NotFound);
+ }
+ },
+ xDelete: async function(filename){
+ log("xDelete(",arguments,")");
+ storeAndNotify('xClose', 0);
+ },
+ xFileSize: async function(fid){
+ log("xFileSize(",arguments,")");
+ const fh = state.openFiles[fid];
+ const sz = await fh.getSize();
+ affirm32Bits(sz);
+ storeAndNotify('xFileSize', sz | 0);
+ },
+ xOpen: async function({
+ fid/*sqlite3_file pointer*/, sab/*file-specific SharedArrayBuffer*/,
+ filename,
+ fileType = undefined /*mainDb, mainJournal, etc.*/,
+ create = false, readOnly = false, deleteOnClose = false,
+ }){
+ const opName = 'xOpen';
+ try{
+ if(create) readOnly = false;
+ log(opName+"(",arguments[0],")");
+
+ let hDir, filenamePart, hFile;
+ try {
+ [hDir, filenamePart] = await getDirForPath(filename, !!create);
+ }catch(e){
+ storeAndNotify(opName, state.errCodes.NotFound);
+ return;
+ }
+ hFile = await hDir.getFileHandle(filenamePart, {create: !!create});
+ log(opName,"filenamePart =",filenamePart, 'hDir =',hDir);
+ const fobj = state.openFiles[fid] = Object.create(null);
+ fobj.filenameAbs = filename;
+ fobj.filenamePart = filenamePart;
+ fobj.dirHandle = hDir;
+ fobj.fileHandle = hFile;
+ fobj.accessHandle = undefined;
+ fobj.fileType = fileType;
+ fobj.sab = sab;
+ fobj.create = !!create;
+ fobj.readOnly = !!readOnly;
+ fobj.deleteOnClose = !!deleteOnClose;
+
+ /**
+ wa-sqlite, at this point, grabs a SyncAccessHandle and
+ assigns it to the accessHandle prop of the file state
+ object, but it's unclear why it does that.
+ */
+ storeAndNotify(opName, 0);
+ }catch(e){
+ error(opName,e);
+ storeAndNotify(opName, state.errCodes.IO);
+ }
+ },
+ xRead: async function({fid,n,offset}){
+ log("xRead(",arguments,")");
+ affirm32Bits(n + offset);
+ const fh = state.openFiles[fid];
+ storeAndNotify('xRead',fid);
+ },
+ xSleep: async function f({ms}){
+ log("xSleep(",arguments[0],")");
+ await new Promise((resolve)=>{
+ setTimeout(()=>resolve(), ms);
+ }).finally(()=>storeAndNotify('xSleep',0));
+ },
+ xSync: async function({fid}){
+ log("xSync(",arguments,")");
+ const fh = state.openFiles[fid];
+ await fh.flush();
+ storeAndNotify('xSync',fid);
+ },
+ xTruncate: async function({fid,size}){
+ log("xTruncate(",arguments,")");
+ affirm32Bits(size);
+ const fh = state.openFiles[fid];
+ fh.truncate(size);
+ storeAndNotify('xTruncate',fid);
+ },
+ xWrite: async function({fid,src,n,offset}){
+ log("xWrite(",arguments,")");
+ const fh = state.openFiles[fid];
+ storeAndNotify('xWrite',fid);
+ }
+ };
+
+ const onReady = function(){
+ self.onmessage = async function({data}){
+ log("self.onmessage",data);
+ switch(data.type){
+ case 'init':{
+ const opt = data.payload;
+ state.opSab = opt.opSab;
+ state.opBuf = new Int32Array(state.opSab);
+ state.opIds = opt.opIds;
+ state.errCodes = opt.errCodes;
+ state.sq3Codes = opt.sq3Codes;
+ Object.keys(ioMethods).forEach((k)=>{
+ if(!state.opIds[k]){
+ toss("Maintenance required: missing state.opIds[",k,"]");
+ }
+ });
+ log("init state",state);
+ break;
+ }
+ default:{
+ const m = ioMethods[data.type] || toss("Unknown message type:",data.type);
+ try {
+ await m(data.payload);
+ }catch(e){
+ error("Error handling",data.type+"():",e);
+ storeAndNotify(data.type, -99);
+ }
+ break;
+ }
+ }
+ };
+ wMsg('ready');
+ };
+
+ navigator.storage.getDirectory().then(function(d){
+ state.rootDir = d;
+ log("state.rootDir =",state.rootDir);
+ onReady();
+ });
+
+})();
--- /dev/null
+'use strict';
+const doAtomicsStuff = function(sqlite3){
+ const logPrefix = "OPFS syncer:";
+ const log = (...args)=>{
+ console.log(logPrefix,...args);
+ };
+ const warn = (...args)=>{
+ console.warn(logPrefix,...args);
+ };
+ const error = (...args)=>{
+ console.error(logPrefix,...args);
+ };
+ const W = new Worker("sqlite3-opfs-async-proxy.js");
+ const wMsg = (type,payload)=>W.postMessage({type,payload});
+ warn("This file is very much experimental and under construction.",self.location.pathname);
+
+ /**
+ State which we send to the async-api Worker or share with it.
+ This object must initially contain only cloneable or sharable
+ objects. After the worker's "ready" message arrives, other types
+ of data may be added to it.
+ */
+ const state = Object.create(null);
+ state.opIds = Object.create(null);
+ state.opIds.xAccess = 1;
+ state.opIds.xClose = 2;
+ state.opIds.xDelete = 3;
+ state.opIds.xFileSize = 4;
+ state.opIds.xOpen = 5;
+ state.opIds.xRead = 6;
+ state.opIds.xSync = 7;
+ state.opIds.xTruncate = 8;
+ state.opIds.xWrite = 9;
+ state.opIds.xSleep = 10;
+ state.opIds.xBlock = 99 /* to block worker while this code is still handling something */;
+ state.opSab = new SharedArrayBuffer(64);
+ state.fileBufferSize = 1024 * 65 /* 64k = max sqlite3 page size */;
+ /* TODO: use SQLITE_xxx err codes. */
+ state.errCodes = Object.create(null);
+ state.errCodes.Error = -100;
+ state.errCodes.IO = -101;
+ state.errCodes.NotFound = -102;
+ state.errCodes.Misuse = -103;
+
+ // TODO: add any SQLITE_xxx symbols we need here.
+ state.sq3Codes = Object.create(null);
+
+ const isWorkerErrCode = (n)=>(n<=state.errCodes.Error);
+
+ const opStore = (op,val=-1)=>Atomics.store(state.opBuf, state.opIds[op], val);
+ const opWait = (op,val=-1)=>Atomics.wait(state.opBuf, state.opIds[op], val);
+
+ const opRun = (op,args)=>{
+ opStore(op);
+ wMsg(op, args);
+ opWait(op);
+ return Atomics.load(state.opBuf, state.opIds[op]);
+ };
+
+ const wait = (ms,value)=>{
+ return new Promise((resolve)=>{
+ setTimeout(()=>resolve(value), ms);
+ });
+ };
+
+ const vfsSyncWrappers = {
+ xOpen: function f(pFile, name, flags, outFlags = {}){
+ if(!f._){
+ f._ = {
+ // TODO: map openFlags to args.fileType names.
+ };
+ }
+ const args = Object.create(null);
+ args.fid = pFile;
+ args.filename = name;
+ args.sab = new SharedArrayBuffer(state.fileBufferSize);
+ args.fileType = undefined /*TODO: populate based on SQLITE_OPEN_xxx */;
+ // TODO: populate args object based on flags:
+ // args.create, args.readOnly, args.deleteOnClose
+ args.create = true;
+ args.deleteOnClose = true;
+ const rc = opRun('xOpen', args);
+ if(!rc){
+ outFlags.readOnly = args.readOnly;
+ args.ba = new Uint8Array(args.sab);
+ state.openFiles[pFile] = args;
+ }
+ return rc;
+ },
+ xClose: function(pFile){
+ let rc = 0;
+ if(state.openFiles[pFile]){
+ delete state.openFiles[pFile];
+ rc = opRun('xClose', pFile);
+ }
+ return rc;
+ }
+ };
+
+
+ const doSomething = function(){
+ //state.ioBuf = new Uint8Array(state.sabIo);
+ const fid = 37;
+ let rc = vfsSyncWrappers.xOpen(fid, "/foo/bar/baz.sqlite3",0, {});
+ log("open rc =",rc,"state.opBuf[xOpen] =",state.opBuf[state.opIds.xOpen]);
+ if(isWorkerErrCode(rc)){
+ error("open failed with code",rc);
+ return;
+ }
+ log("xSleep()ing before close()ing...");
+ opRun('xSleep',{ms: 1500});
+ log("wait()ing before close()ing...");
+ wait(1500).then(function(){
+ rc = vfsSyncWrappers.xClose(fid);
+ log("xClose rc =",rc,"opBuf =",state.opBuf);
+ });
+ };
+
+ W.onmessage = function({data}){
+ log("Worker.onmessage:",data);
+ switch(data.type){
+ case 'ready':
+ wMsg('init',state);
+ state.opBuf = new Int32Array(state.opSab);
+ state.openFiles = Object.create(null);
+ doSomething();
+ break;
+ }
+ };
+}/*doAtomicsStuff*/
+
+importScripts('sqlite3.js');
+self.sqlite3InitModule().then((EmscriptenModule)=>doAtomicsStuff(EmscriptenModule.sqlite3));
-C Merge\skv-vfs\sbranch\sinto\sfiddle-opfs\sbranch.\sAdjust\sspeedtest1\s--size\sflags\sto\saccount\sfor\snew\ssize\slimit.
-D 2022-09-16T20:16:50.240
+C Add\sinitial\sbits\sof\san\sexperimental\sasync-impl-via-synchronous-interface\sproxy\sintended\sto\smarshal\sOPFS\svia\ssqlite3_vfs\sAPI.
+D 2022-09-17T15:08:22.642
F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
F ext/wasm/api/sqlite3-api-cleanup.js 8564a6077cdcaea9a9f428a019af8a05887f0131e6a2a1e72a7ff1145fadfe77
F ext/wasm/api/sqlite3-api-glue.js 366d580c8e5bf7fcf4c6dee6f646c31f5549bd417ea03a59a0acca00e8ecce30
F ext/wasm/api/sqlite3-api-oo1.js d7526517f7ad3f6bda16ad66d373bbb71b43168deef7af60eda5c9fe873d1387
-F ext/wasm/api/sqlite3-api-opfs.js 011799db398157cbd254264b6ebae00d7234b93d0e9e810345f213a5774993c0
+F ext/wasm/api/sqlite3-api-opfs.js 130f60cc8f5835f9d77d4f12308bf4c8fb6d9c315009fc7239c5d67ff2bc8c67
F ext/wasm/api/sqlite3-api-prologue.js 48ebca4ae340b0242d4f39bbded01bd0588393c8023628be1c454b4db6f7bd6e
F ext/wasm/api/sqlite3-api-worker1.js d33062afa045fd4be01ba4abc266801807472558b862b30056211b00c9c347b4
F ext/wasm/api/sqlite3-wasi.h 25356084cfe0d40458a902afb465df8c21fc4152c1d0a59b563a3fba59a068f9
F ext/wasm/fiddle/fiddle-worker.js bccf46045be8824752876f3eec01c223be0616ccac184bffd0024cfe7a3262b8
F ext/wasm/fiddle/fiddle.html 550c5aafce40bd218de9bf26192749f69f9b10bc379423ecd2e162bcef885c08
F ext/wasm/fiddle/fiddle.js 4ffcfc9a235beebaddec689a549e9e0dfad6dca5c1f0b41f03468d7e76480686
-F ext/wasm/index.html 095b9a8cee9aac2654c23686ead22f3452b89d581fb41d34d47b6548546b5365
+F ext/wasm/index.html 8365e47e2aff1829923f0481292948487b27a0755fde9c0d3ad15f7ea0118992
F ext/wasm/jaccwabyt/jaccwabyt.js 0d7f32817456a0f3937fcfd934afeb32154ca33580ab264dab6c285e6dbbd215
F ext/wasm/jaccwabyt/jaccwabyt.md 447cc02b598f7792edaa8ae6853a7847b8178a18ed356afacbdbf312b2588106
F ext/wasm/jaccwabyt/jaccwabyt_test.c 39e4b865a33548f943e2eb9dd0dc8d619a80de05d5300668e9960fff30d0d36f
F ext/wasm/split-speedtest1-script.sh a3e271938d4d14ee49105eb05567c6a69ba4c1f1293583ad5af0cd3a3779e205 x
F ext/wasm/sql/000-mandelbrot.sql 775337a4b80938ac8146aedf88808282f04d02d983d82675bd63d9c2d97a15f0
F ext/wasm/sql/001-sudoku.sql 35b7cb7239ba5d5f193bc05ec379bcf66891bce6f2a5b3879f2f78d0917299b5
+F ext/wasm/sqlite3-opfs-async-proxy.js c42a097dfbb96abef08554b173a47788f5bc1f58c266f859ba01c1fa3ff8327d
F ext/wasm/sqlite3-worker1-promiser.js 92b8da5f38439ffec459a8215775d30fa498bc0f1ab929ff341fc3dd479660b9
F ext/wasm/sqlite3-worker1.js 0c1e7626304543969c3846573e080c082bf43bcaa47e87d416458af84f340a9e
F ext/wasm/testing-worker1-promiser.html 6eaec6e04a56cf24cf4fa8ef49d78ce8905dde1354235c9125dca6885f7ce893
F ext/wasm/testing2.html a66951c38137ff1d687df79466351f3c734fa9c6d9cce71d3cf97c291b2167e3
F ext/wasm/testing2.js 25584bcc30f19673ce13a6f301f89f8820a59dfe044e0c4f2913941f4097fe3c
F ext/wasm/wasmfs.make 21a5cf297954a689e0dc2a95299ae158f681cae5e90c10b99d986097815fd42d
+F ext/wasm/x-sync-async.html 283539e4fcca8c60fea18dbf1f1c0df168340145a19123f8fd5b70f41291b36f
+F ext/wasm/x-sync-async.js 42da502ea0b89bfa226c7ac7555c0c87d4ab8a10221ea6fadb4f7877c26a5137
F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x
F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8
F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60
F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 52d1b185b9f6cee1eb3dec436f47e0f52e4621a127abfad8c27f92fd78147889 ef54961ce69fddb4cfeeff0860288de2858a6f7a5aa396691e8e99933eb9af54
-R e35abbf6d1d9b2b828d30426a9a9f018
+P afb79050e635f3c698e51f06c346cbf23b096cfda7d0f1d8e68514ea0c25b7b7
+R dbdee404fc63f84ef61ddb4fd79c0ad1
U stephan
-Z 0ddb259c4784fc46ece386ccc85309e5
+Z 67d0abd2dfaa993776fbe8c511ee53f5
# Remove this line to create a well-formed Fossil manifest.
projects / thirdparty / sqlite.git / commitdiff
