From: stephan Date: Mon, 12 Sep 2022 16:09:50 +0000 (+0000) Subject: Merge kv-vfs branch into fiddle-opfs branch to add kvvfs-based wasm build and demo. X-Git-Tag: version-3.40.0~169^2~123 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=5360f5fcff10bb4f3ff292a2686f6cfc22a1993d;p=thirdparty%2Fsqlite.git Merge kv-vfs branch into fiddle-opfs branch to add kvvfs-based wasm build and demo. FossilOrigin-Name: a7d8b26acd3c1ae344369e4d70804c0cab45272c0983cfd32d616a0a7b28acb9 --- 5360f5fcff10bb4f3ff292a2686f6cfc22a1993d diff --cc ext/wasm/GNUmakefile index dccbbbe5d1,3133d1230b..fd95b013ca --- a/ext/wasm/GNUmakefile +++ b/ext/wasm/GNUmakefile @@@ -310,76 -262,8 +310,78 @@@ wasm: $(sqlite3.js # End main Emscripten-based module build ######################################################################## -include kvvfs.make +######################################################################## +# batch-runner.js... +dir.sql := sql +speedtest1 := ../../speedtest1 +speedtest1.c := ../../test/speedtest1.c +speedtest1.sql := $(dir.sql)/speedtest1.sql +$(speedtest1): + $(MAKE) -C ../.. speedtest1 +$(speedtest1.sql): $(speedtest1) + $(speedtest1) --script $@ +batch-runner.list: $(MAKEFILE) $(speedtest1.sql) $(dir.sql)/000-mandelbrot.sql + bash split-speedtest1-script.sh $(dir.sql)/speedtest1.sql + ls -1 $(dir.sql)/*.sql | grep -v speedtest1.sql | sort > $@ +clean-batch: + rm -f batch-runner.list $(dir.sql)/speedtest1*.sql +# ^^^ we don't do this along with 'clean' because we clean/rebuild on +# a regular basis with different -Ox flags and rebuilding the batch +# pieces each time is an unnecessary time sink. +batch: batch-runner.list +all: batch +# end batch-runner.js +######################################################################## +# speedtest1.js... +emcc.speedtest1-flags := -g $(emcc_opt) +ifneq (0,$(ENABLE_WASMFS)) + emcc.speedtest1-flags += -pthread -sWASMFS -sPTHREAD_POOL_SIZE=2 + emcc.speedtest1-flags += -DSQLITE_WASM_OPFS +endif +emcc.speedtest1-flags += -sINVOKE_RUN=0 +#emcc.speedtest1-flags += --no-entry +emcc.speedtest1-flags += -flto +emcc.speedtest1-flags += -sABORTING_MALLOC +emcc.speedtest1-flags += -sINITIAL_MEMORY=128450560 +emcc.speedtest1-flags += -sSTRICT_JS +emcc.speedtest1-flags += $(emcc.environment) +emcc.speedtest1-flags += -sMODULARIZE +emcc.speedtest1-flags += -sEXPORT_NAME=sqlite3Speedtest1InitModule +emcc.speedtest1-flags += -Wno-limited-postlink-optimizations +emcc.speedtest1-flags += -sEXPORTED_FUNCTIONS=_main,_malloc,_free,_sqlite3_wasm_vfs_unlink,_sqlite3_wasm_init_opfs +emcc.speedtest1-flags += -sDYNAMIC_EXECUTION=0 +emcc.speedtest1-flags += --minify 0 + +speedtest1.js := speedtest1.js +speedtest1.wasm := $(subst .js,.wasm,$(speedtest1.js)) +$(speedtest1.js): emcc.cflags+= +# speedtest1 notes re. sqlite3-wasm.o vs sqlite3-wasm.c: building against +# the latter (predictably) results in a slightly faster binary, but we're +# close enough to the target speed requirements that the 500ms makes a +# difference. +$(speedtest1.js): $(speedtest1.c) $(sqlite3-wasm.c) $(MAKEFILE) + $(emcc.bin) \ + $(emcc.speedtest1-flags) \ + -I. -I$(dir.top) \ + -DSQLITE_THREADSAFE=0 \ + -DSQLITE_TEMP_STORE=3 \ + -DSQLITE_OMIT_UTF16 \ + -DSQLITE_OMIT_DEPRECATED \ + -DSQLITE_OMIT_SHARED_CACHE \ + '-DSQLITE_DEFAULT_UNIX_VFS="unix-none"' \ + -DSQLITE_SPEEDTEST1_WASM \ + -o $@ $(speedtest1.c) $(sqlite3-wasm.c) -lm +ifneq (,$(wasm-strip)) + $(wasm-strip) $(speedtest1.wasm) +endif + ls -la $@ $(speedtest1.wasm) + +speedtest1: $(speedtest1.js) +all: $(speedtest1.js) +CLEAN_FILES += $(speedtest1.js) $(speedtest1.wasm) +# end speedtest1.js ++######################################################################## + ######################################################################## # fiddle_remote is the remote destination for the fiddle app. It # must be a [user@]HOST:/path for rsync. @@@ -402,3 -286,3 +404,5 @@@ push-fiddle: $(fiddle_files rsync -va fiddle/ $(fiddle_remote) # end fiddle remote push ######################################################################## ++ ++include kvvfs.make diff --cc ext/wasm/api/sqlite3-api-cleanup.js index 01aba213ed,a2f921a5d7..1b57cdc5de --- a/ext/wasm/api/sqlite3-api-cleanup.js +++ b/ext/wasm/api/sqlite3-api-cleanup.js @@@ -11,45 -11,34 +11,45 @@@ *********************************************************************** This file is the tail end of the sqlite3-api.js constellation, - intended to be appended after all other files so that it can clean - up any global systems temporarily used for setting up the API's - various subsystems. + intended to be appended after all other sqlite3-api-*.js files so + that it can finalize any setup and clean up any global symbols + temporarily used for setting up the API's various subsystems. */ 'use strict'; -self.sqlite3.postInit.forEach( - self.importScripts/*global is a Worker*/ - ? function(f){ - /** We try/catch/report for the sake of failures which happen in - a Worker, as those exceptions can otherwise get completely - swallowed, leading to confusing downstream errors which have - nothing to do with this failure. */ - try{ f(self, self.sqlite3) } - catch(e){ - console.error("Error in postInit() function:",e); - throw e; - } - } - : (f)=>f(self, self.sqlite3) -); -delete self.sqlite3.postInit; -if(self.location && +self.location.port > 1024){ - console.warn("Installing sqlite3 bits as global S for dev-testing purposes."); - self.S = self.sqlite3; +if('undefined' !== typeof Module){ // presumably an Emscripten build + /** + Install a suitable default configuration for sqlite3ApiBootstrap(). + */ + const SABC = self.sqlite3ApiBootstrap.defaultConfig; + SABC.Module = Module /* ==> Currently needs to be exposed here for test code. NOT part + of the public API. */; + SABC.exports = Module['asm']; + SABC.memory = Module.wasmMemory /* gets set if built with -sIMPORT_MEMORY */; + + /** + For current (2022-08-22) purposes, automatically call + sqlite3ApiBootstrap(). That decision will be revisited at some + point, as we really want client code to be able to call this to + configure certain parts. Clients may modify + self.sqlite3ApiBootstrap.defaultConfig to tweak the default + configuration used by a no-args call to sqlite3ApiBootstrap(). + */ + //console.warn("self.sqlite3ApiConfig = ",self.sqlite3ApiConfig); + const sqlite3 = self.sqlite3ApiBootstrap(); + delete self.sqlite3ApiBootstrap; + + if(self.location && +self.location.port > 1024){ - console.warn("Installing sqlite3 bits as global S for dev-testing purposes."); ++ console.warn("Installing sqlite3 bits as global S for local dev/test purposes."); + self.S = sqlite3; + } + + /* Clean up temporary references to our APIs... */ + delete sqlite3.capi.util /* arguable, but these are (currently) internal-use APIs */; + //console.warn("Module.sqlite3 =",Module.sqlite3); + Module.sqlite3 = sqlite3 /* Currently needed by test code and sqlite3-worker1.js */; +}else{ + console.warn("This is not running in an Emscripten module context, so", + "self.sqlite3ApiBootstrap() is _not_ being called due to lack", + "of config info for the WASM environment.", + "It must be called manually."); } -/* Clean up temporary global-scope references to our APIs... */ -self.sqlite3.config.Module.sqlite3 = self.sqlite3 -/* ^^^^ Currently needed by test code and Worker API setup */; -delete self.sqlite3.capi.util /* arguable, but these are (currently) internal-use APIs */; -delete self.sqlite3 /* clean up our global-scope reference */; -//console.warn("Module.sqlite3 =",Module.sqlite3); diff --cc ext/wasm/api/sqlite3-api-oo1.js index aafc04a2d5,9e54733966..af179d1fe1 --- a/ext/wasm/api/sqlite3-api-oo1.js +++ b/ext/wasm/api/sqlite3-api-oo1.js @@@ -67,9 -64,9 +67,77 @@@ self.sqlite3ApiBootstrap.initializers.p this.name = 'SQLite3Error'; } }; - const toss3 = (...args)=>{throw new SQLite3Error(args)}; + const toss3 = (...args)=>{throw new SQLite3Error(...args)}; sqlite3.SQLite3Error = SQLite3Error; ++ // Documented in DB.checkRc() ++ const checkSqlite3Rc = function(dbPtr, sqliteResultCode){ ++ if(sqliteResultCode){ ++ if(dbPtr instanceof DB) dbPtr = dbPtr.pointer; ++ throw new SQLite3Error( ++ "sqlite result code",sqliteResultCode+":", ++ (dbPtr ++ ? capi.sqlite3_errmsg(dbPtr) ++ : capi.sqlite3_errstr(sqliteResultCode)) ++ ); ++ } ++ }; ++ ++ /** ++ A proxy for DB class constructors. It must be called with the ++ being-construct DB object as its "this". ++ */ ++ const dbCtorHelper = function ctor(fn=':memory:', flags='c', vfsName){ ++ if(!ctor._name2vfs){ ++ // Map special filenames which we handle here (instead of in C) ++ // to some helpful metadata... ++ ctor._name2vfs = Object.create(null); ++ const isWorkerThread = (self.window===self /*===running in main window*/) ++ ? false ++ : (n)=>toss3("The VFS for",n,"is only available in the main window thread.") ++ ctor._name2vfs[':localStorage:'] = { ++ vfs: 'kvvfs', ++ filename: isWorkerThread || (()=>'local') ++ }; ++ ctor._name2vfs[':sessionStorage:'] = { ++ vfs: 'kvvfs', ++ filename: isWorkerThread || (()=>'session') ++ }; ++ } ++ if('string'!==typeof fn){ ++ toss3("Invalid filename for DB constructor."); ++ } ++ const vfsCheck = ctor._name2vfs[fn]; ++ if(vfsCheck){ ++ vfsName = vfsCheck.vfs; ++ fn = vfsCheck.filename(fn); ++ } ++ let ptr, oflags = 0; ++ if( flags.indexOf('c')>=0 ){ ++ oflags |= capi.SQLITE_OPEN_CREATE | capi.SQLITE_OPEN_READWRITE; ++ } ++ if( flags.indexOf('w')>=0 ) oflags |= capi.SQLITE_OPEN_READWRITE; ++ if( 0===oflags ) oflags |= capi.SQLITE_OPEN_READONLY; ++ oflags |= capi.SQLITE_OPEN_EXRESCODE; ++ const stack = capi.wasm.scopedAllocPush(); ++ try { ++ const ppDb = capi.wasm.scopedAllocPtr() /* output (sqlite3**) arg */; ++ const pVfsName = vfsName ? capi.wasm.scopedAllocCString(vfsName) : 0; ++ const rc = capi.sqlite3_open_v2(fn, ppDb, oflags, pVfsName); ++ ptr = capi.wasm.getPtrValue(ppDb); ++ checkSqlite3Rc(ptr, rc); ++ }catch( e ){ ++ if( ptr ) capi.sqlite3_close_v2(ptr); ++ throw e; ++ }finally{ ++ capi.wasm.scopedAllocPop(stack); ++ } ++ this.filename = fn; ++ __ptrMap.set(this, ptr); ++ __stmtMap.set(this, Object.create(null)); ++ __udfMap.set(this, Object.create(null)); ++ }; ++ /** The DB class provides a high-level OO wrapper around an sqlite3 db handle. @@@ -83,61 -80,39 +151,48 @@@ not resolve to real filenames, but "" uses an on-storage temporary database and requires that the VFS support that. - The db is currently opened with a fixed set of flags: - (SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | - SQLITE_OPEN_EXRESCODE). This API will change in the future - permit the caller to provide those flags via an additional - argument. + The second argument specifies the open/create mode for the + database. It must be string containing a sequence of letters (in + any order, but case sensitive) specifying the mode: + + - "c" => create if it does not exist, else fail if it does not + exist. Implies the "w" flag. + + - "w" => write. Implies "r": a db cannot be write-only. + + - "r" => read-only if neither "w" nor "c" are provided, else it + is ignored. + + If "w" is not provided, the db is implicitly read-only, noting that + "rc" is meaningless + + Any other letters are currently ignored. The default is + "c". These modes are ignored for the special ":memory:" and "" + names. + - The final argument is currently unimplemented but will eventually - be used to specify an optional sqlite3 VFS implementation name, - as for the final argument to sqlite3_open_v2(). ++ The final argument is analogous to the final argument of ++ sqlite3_open_v2(): the name of an sqlite3 VFS. Pass a falsy value, ++ or not at all, to use the default. If passed a value, it must ++ be the string name of a VFS For purposes of passing a DB instance to C-style sqlite3 - functions, its read-only `pointer` property holds its `sqlite3*` - pointer value. That property can also be used to check whether - this DB instance is still open. + functions, the DB object's read-only `pointer` property holds its + `sqlite3*` pointer value. That property can also be used to check + whether this DB instance is still open. ++ ++ ++ EXPERIMENTAL: in the main window thread, the filenames ++ ":localStorage:" and ":sessionStorage:" are special: they cause ++ the db to use either localStorage or sessionStorage for storing ++ the database. In this mode, only a single database is permitted ++ in each storage object. This feature is experimental and subject ++ to any number of changes (including outright removal). This ++ support requires a specific build of sqlite3, the existence of ++ which can be determined at runtime by checking for a non-0 return ++ value from sqlite3.capi.sqlite3_vfs_find("kvvfs"). */ - const DB = function ctor(fn=':memory:', flags='c', vtab="not yet implemented"){ - const DB = function ctor(fn=':memory:'){ -- if('string'!==typeof fn){ -- toss3("Invalid filename for DB constructor."); - } - let ptr, oflags = 0; - if( flags.indexOf('c')>=0 ){ - oflags |= capi.SQLITE_OPEN_CREATE | capi.SQLITE_OPEN_READWRITE; -- } - if( flags.indexOf('w')>=0 ) oflags |= capi.SQLITE_OPEN_READWRITE; - if( 0===oflags ) oflags |= capi.SQLITE_OPEN_READONLY; - oflags |= capi.SQLITE_OPEN_EXRESCODE; -- const stack = capi.wasm.scopedAllocPush(); - let ptr; -- try { -- const ppDb = capi.wasm.scopedAllocPtr() /* output (sqlite3**) arg */; - const rc = capi.sqlite3_open_v2(fn, ppDb, oflags, null); - ptr = capi.wasm.getPtrValue(ppDb); - const rc = capi.sqlite3_open_v2(fn, ppDb, capi.SQLITE_OPEN_READWRITE - | capi.SQLITE_OPEN_CREATE - | capi.SQLITE_OPEN_EXRESCODE, null); - ptr = capi.wasm.getMemValue(ppDb, '*'); -- ctor.checkRc(ptr, rc); - }catch( e ){ - if( ptr ) capi.sqlite3_close_v2(ptr); - }catch(e){ - if(ptr) capi.sqlite3_close_v2(ptr); -- throw e; - }finally{ - capi.wasm.scopedAllocPop(stack); -- } - finally{capi.wasm.scopedAllocPop(stack);} -- this.filename = fn; -- __ptrMap.set(this, ptr); -- __stmtMap.set(this, Object.create(null)); -- __udfMap.set(this, Object.create(null)); ++ const DB = function ctor(fn=':memory:', flags='c', vfsName){ ++ dbCtorHelper.apply(this, Array.prototype.slice.call(arguments)); }; /** @@@ -232,6 -194,6 +287,8 @@@ }else if(args[0] && 'object'===typeof args[0]){ out.opt = args[0]; out.sql = out.opt.sql; ++ }else if(Array.isArray(args[0])){ ++ out.sql = args[0]; } break; case 2: @@@ -285,27 -234,24 +342,17 @@@ }; /** - Expects to be given a DB instance or an `sqlite3*` pointer, and an - sqlite3 API result code. If the result code is not falsy, this - function throws an SQLite3Error with an error message from - sqlite3_errmsg(), using dbPtr as the db handle. Note that if it's - passed a non-error code like SQLITE_ROW or SQLITE_DONE, it will - still throw but the error string might be "Not an error." The - various non-0 non-error codes need to be checked for in client - code where they are expected. + Expects to be given a DB instance or an `sqlite3*` pointer (may + be null) and an sqlite3 API result code. If the result code is + not falsy, this function throws an SQLite3Error with an error + message from sqlite3_errmsg(), using dbPtr as the db handle, or + sqlite3_errstr() if dbPtr is falsy. Note that if it's passed a + non-error code like SQLITE_ROW or SQLITE_DONE, it will still + throw but the error string might be "Not an error." The various + non-0 non-error codes need to be checked for in + client code where they are expected. */ -- DB.checkRc = function(dbPtr, sqliteResultCode){ -- if(sqliteResultCode){ -- if(dbPtr instanceof DB) dbPtr = dbPtr.pointer; - throw new SQLite3Error( - throw new SQLite3Error([ -- "sqlite result code",sqliteResultCode+":", - (dbPtr - ? capi.sqlite3_errmsg(dbPtr) - : capi.sqlite3_errstr(sqliteResultCode)) - ); - capi.sqlite3_errmsg(dbPtr) || "Unknown db error." - ].join(' ')); -- } -- }; ++ DB.checkRc = checkSqlite3Rc; DB.prototype = { /** @@@ -1531,6 -1434,5 +1578,20 @@@ }, DB, Stmt - }/*SQLite3 object*/; -})(self); + }/*oo1 object*/; ++ ++ if( self.window===self && 0!==capi.sqlite3_vfs_find('kvvfs') ){ ++ /* In the main window thread, add a couple of convenience proxies ++ for localStorage and sessionStorage DBs... */ ++ let klass = sqlite3.oo1.LocalStorageDb = function(){ ++ dbCtorHelper.call(this, 'local', 'c', 'kvvfs'); ++ }; ++ klass.prototype = DB.prototype; ++ ++ klass = sqlite3.oo1.SessionStorageDb = function(){ ++ dbCtorHelper.call(this, 'session', 'c', 'kvvfs'); ++ }; ++ klass.prototype = DB.prototype; ++ } +}); + diff --cc ext/wasm/api/sqlite3-api-prologue.js index 7959d047c9,60ed61477e..17dcd42289 --- a/ext/wasm/api/sqlite3-api-prologue.js +++ b/ext/wasm/api/sqlite3-api-prologue.js @@@ -671,119 -576,18 +671,125 @@@ self.sqlite3ApiBootstrap = function sql ["sqlite3_total_changes64", "i64", ["sqlite3*"]] ]; + /** + Functions which are intended solely for API-internal use by the + WASM components, not client code. These get installed into + capi.wasm. + */ + capi.wasm.bindingSignatures.wasm = [ + ["sqlite3_wasm_vfs_unlink", "int", "string"] + ]; + + /** State for sqlite3_web_persistent_dir(). */ + let __persistentDir; + /** + An experiment. Do not use. + + If the wasm environment has a persistent storage directory, + its path is returned by this function. If it does not then + it returns "" (noting that "" is a falsy value). + + The first time this is called, this function inspects the current + environment to determine whether persistence filesystem support + is available and, if it is, enables it (if needed). + + TODOs and caveats: + + - If persistent storage is available at the root of the virtual + filesystem, this interface cannot currently distinguish that + from the lack of persistence. That case cannot currently (with + WASMFS/OPFS) happen, but it is conceivably possible in future + environments or non-browser runtimes (none of which are yet + supported targets). + */ + capi.sqlite3_web_persistent_dir = function(){ + if(undefined !== __persistentDir) return __persistentDir; + // If we have no OPFS, there is no persistent dir + const pdir = config.persistentDirName; + if(!pdir + || !self.FileSystemHandle + || !self.FileSystemDirectoryHandle + || !self.FileSystemFileHandle){ + return __persistentDir = ""; + } + try{ - if(pdir && 0===this.wasm.xCallWrapped( ++ if(pdir && 0===capi.wasm.xCallWrapped( + 'sqlite3_wasm_init_opfs', 'i32', ['string'], pdir + )){ + /** OPFS does not support locking and will trigger errors if + we try to lock. We don't _really_ want to + _unconditionally_ install a non-locking sqlite3 VFS as the + default, but we do so here for simplicy's sake for the + time being. That said: locking is a no-op on all of the + current WASM storage, so this isn't (currently) as bad as + it may initially seem. */ + const pVfs = sqlite3.capi.sqlite3_vfs_find("unix-none"); + if(pVfs){ + capi.sqlite3_vfs_register(pVfs,1); + console.warn("Installed 'unix-none' as the default sqlite3 VFS."); + } + return __persistentDir = pdir; + }else{ + return __persistentDir = ""; + } + }catch(e){ + // sqlite3_wasm_init_opfs() is not available + return __persistentDir = ""; + } - }.bind(capi); ++ }; + + /** + Returns true if sqlite3.capi.sqlite3_web_persistent_dir() is a + non-empty string and the given name has that string as its + prefix, else returns false. + */ + capi.sqlite3_web_filename_is_persistent = function(name){ - const p = this.sqlite3_web_persistent_dir(); ++ const p = capi.sqlite3_web_persistent_dir(); + return (p && name) ? name.startsWith(p) : false; - }.bind(capi); ++ }; ++ ++ if(0===capi.wasm.exports.sqlite3_vfs_find(0)){ ++ /* Assume that sqlite3_initialize() has not yet been called. ++ This will be the case in an SQLITE_OS_KV build. */ ++ capi.wasm.exports.sqlite3_initialize(); ++ } + /* The remainder of the API will be set up in later steps. */ - return { + const sqlite3 = { + WasmAllocError: WasmAllocError, capi, - postInit: [ - /* some pieces of the API may install functions into this array, - and each such function will be called, passed (self,sqlite3), - at the very end of the API load/init process, where self is - the current global object and sqlite3 is the object returned - from sqlite3ApiBootstrap(). This array will be removed at the - end of the API setup process. */], - /** Config is needed downstream for gluing pieces together. It - will be removed at the end of the API setup process. */ config }; + sqlite3ApiBootstrap.initializers.forEach((f)=>f(sqlite3)); + delete sqlite3ApiBootstrap.initializers; + sqlite3ApiBootstrap.sqlite3 = sqlite3; + return sqlite3; }/*sqlite3ApiBootstrap()*/; +/** + self.sqlite3ApiBootstrap.initializers is an internal detail used by + the various pieces of the sqlite3 API's amalgamation process. It + must not be modified by client code except when plugging such code + into the amalgamation process. + + Each component of the amalgamation is expected to append a function + to this array. When sqlite3ApiBootstrap() is called for the first + time, each such function will be called (in their appended order) + and passed the sqlite3 namespace object, into which they can install + their features (noting that most will also require that certain + features alread have been installed). At the end of that process, + this array is deleted. +*/ +self.sqlite3ApiBootstrap.initializers = []; +/** + Client code may assign sqlite3ApiBootstrap.defaultConfig an + object-type value before calling sqlite3ApiBootstrap() (without + arguments) in order to tell that call to use this object as its + default config value. The intention of this is to provide + downstream clients with a reasonably flexible approach for plugging in + an environment-suitable configuration without having to define a new + global-scope symbol. +*/ +self.sqlite3ApiBootstrap.defaultConfig = Object.create(null); +/** Placeholder: gets installed by the first call to + self.sqlite3ApiBootstrap(). */ +self.sqlite3ApiBootstrap.sqlite3 = undefined; diff --cc ext/wasm/api/sqlite3-wasm.c index df60a4a24c,6a81da3e5f..2a505f19ab --- a/ext/wasm/api/sqlite3-wasm.c +++ b/ext/wasm/api/sqlite3-wasm.c @@@ -446,303 -411,3 +446,82 @@@ const char * sqlite3_wasm_enum_json(voi #undef outf #undef lenCheck } + +/* +** This function is NOT part of the sqlite3 public API. It is strictly +** for use by the sqlite project's own JS/WASM bindings. +** +** This function invokes the xDelete method of the default VFS, +** passing on the given filename. If zName is NULL, no default VFS is +** found, or it has no xDelete method, SQLITE_MISUSE is returned, else +** the result of the xDelete() call is returned. +*/ +WASM_KEEP +int sqlite3_wasm_vfs_unlink(const char * zName){ + int rc = SQLITE_MISUSE /* ??? */; + sqlite3_vfs * const pVfs = sqlite3_vfs_find(0); + if( zName && pVfs && pVfs->xDelete ){ + rc = pVfs->xDelete(pVfs, zName, 1); + } + return rc; +} + +#if defined(__EMSCRIPTEN__) && defined(SQLITE_WASM_OPFS) +#include +#include + +/* +** This function is NOT part of the sqlite3 public API. It is strictly +** for use by the sqlite project's own JS/WASM bindings, specifically +** only when building with Emscripten's WASMFS support. +** +** This function should only be called if the JS side detects the +** existence of the Origin-Private FileSystem (OPFS) APIs in the +** client. The first time it is called, this function instantiates a +** WASMFS backend impl for OPFS. On success, subsequent calls are +** no-ops. +** +** This function may be passed a "mount point" name, which must have a +** leading "/" and is currently restricted to a single path component, +** e.g. "/foo" is legal but "/foo/" and "/foo/bar" are not. If it is +** NULL or empty, it defaults to "/persistent". +** +** Returns 0 on success, SQLITE_NOMEM if instantiation of the backend +** object fails, SQLITE_IOERR if mkdir() of the zMountPoint dir in +** the virtual FS fails. In builds compiled without SQLITE_WASM_OPFS +** defined, SQLITE_NOTFOUND is returned without side effects. +*/ +WASM_KEEP +int sqlite3_wasm_init_opfs(const char *zMountPoint){ + static backend_t pOpfs = 0; + if( !zMountPoint || !*zMountPoint ) zMountPoint = "/persistent"; + if( !pOpfs ){ + pOpfs = wasmfs_create_opfs_backend(); + if( pOpfs ){ + emscripten_console_log("Created WASMFS OPFS backend."); + } + } + /** It's not enough to instantiate the backend. We have to create a + mountpoint in the VFS and attach the backend to it. */ + if( pOpfs && 0!=access(zMountPoint, F_OK) ){ + /* mkdir() simply hangs when called from fiddle app. Cause is + not yet determined but the hypothesis is an init-order + issue. */ + /* Note that this check and is not robust but it will + hypothetically suffice for the transient wasm-based virtual + filesystem we're currently running in. */ + const int rc = wasmfs_create_directory(zMountPoint, 0777, pOpfs); + emscripten_console_logf("OPFS mkdir(%s) rc=%d", zMountPoint, rc); + if(rc) return SQLITE_IOERR; + } + return pOpfs ? 0 : SQLITE_NOMEM; +} +#else +WASM_KEEP +int sqlite3_wasm_init_opfs(void){ + return SQLITE_NOTFOUND; +} +#endif /* __EMSCRIPTEN__ && SQLITE_WASM_OPFS */ + - #if defined(__EMSCRIPTEN__) // && defined(SQLITE_OS_KV) - #include - #include - - #ifndef KVSTORAGE_KEY_SZ - /* We can remove this once kvvfs and this bit is merged. */ - # define KVSTORAGE_KEY_SZ 32 - static void kvstorageMakeKey( - const char *zClass, - const char *zKeyIn, - char *zKeyOut - ){ - sqlite3_snprintf(KVSTORAGE_KEY_SZ, zKeyOut, "kvvfs-%s-%s", zClass, zKeyIn); - } - #endif - - /* - ** An internal level of indirection for accessing the static - ** kvstorageMakeKey() from EM_JS()-generated functions. This must be - ** made available for export via Emscripten but is not intended to be - ** used from client code. If called with a NULL zKeyOut it is a no-op. - ** It returns KVSTORAGE_KEY_SZ, so JS code (which cannot see that - ** constant) may call it with NULL arguments to get the size of the - ** allocation they'll need for a kvvfs key. - ** - ** Maintenance reminder: Emscripten will install this in the Module - ** init scope and will prefix its name with "_". - */ - WASM_KEEP - int sqlite3_wasm__kvvfsMakeKey(const char *zClass, - const char *zKeyIn, - char *zKeyOut){ - if(zKeyOut) kvstorageMakeKey(zClass, zKeyIn, zKeyOut); - return KVSTORAGE_KEY_SZ; - } - - #if 0 - /* - ** Alternately, we can implement kvstorageMakeKey() in JS in such a - ** way that it's visible to kvstorageWrite/Delete/Read() but not the - ** rest of the world. This impl is considerably more verbose than the - ** C impl because writing directly to memory requires more code in - ** JS. Though more verbose, this approach enables removal of - ** sqlite3_wasm__kvvfsMakeKey(). The only catch is that the - ** KVSTORAGE_KEY_SZ constant has to be hard-coded into this function. - */ - EM_JS(void, kvstorageMakeKeyJS, - (const char *zClass, const char *zKeyIn, char *zKeyOut),{ - const max = 32; - if(!arguments.length) return max; - let n = 0, i = 0, ch = 0; - // Write key prefix to dest... - if(0){ - const prefix = "kvvfs-"; - for(i in prefix) setValue(zKeyOut+(n++), prefix.charCodeAt(i)); - }else{ - // slightly optimized but less readable... - setValue(zKeyOut + (n++), 107/*'k'*/); - setValue(zKeyOut + (n++), 118/*'v'*/); - setValue(zKeyOut + (n++), 118/*'v'*/); - setValue(zKeyOut + (n++), 102/*'f'*/); - setValue(zKeyOut + (n++), 115/*'s'*/); - setValue(zKeyOut + (n++), 45/*'-'*/); - } - // Write zClass to dest... - for(i = 0; n < max && (ch = getValue(zClass+i)); ++n, ++i){ - setValue(zKeyOut + n, ch); - } - // Write "-" separator to dest... - if(n nV + 1) nBuf = nV + 1; - HEAPU8.copyWithin(zBuf, zV, zV + nBuf - 1); - setValue( zBuf + nBuf - 1, 0 ); - return nBuf - 1; - }catch(e){ - console.error("kvstorageRead()",e); - return -2; - }finally{ - stackRestore(stack); - } - }); - - /* - ** This function exists for (1) WASM testing purposes and (2) as a - ** hook to get Emscripten to export several EM_JS()-generated - ** functions. It is not part of the public API and its signature - ** and semantics may change at any time. - */ - WASM_KEEP - int sqlite3_wasm__emjs_keep(int whichOp){ - int rc = 0; - const char * zClass = "session"; - const char * zKey = "hello"; - switch( whichOp ){ - case 0: break; - case 1: - kvstorageWrite(zClass, zKey, "world"); - break; - case 2: { - char buffer[128] = {0}; - char * zBuf = &buffer[0]; - rc = kvstorageRead(zClass, zKey, zBuf, (int)sizeof(buffer)); - emscripten_console_logf("kvstorageRead()=%d %s\n", rc, zBuf); - break; - } - case 3: - kvstorageDelete(zClass, zKey); - break; - case 4: - kvstorageMakeKeyOnJSStack(0,0) /* force Emscripten to include this */; - break; - default: break; - } - return rc; - } - #endif /* ifdef __EMSCRIPTEN__ (kvvfs method impls) */ + +#undef WASM_KEEP diff --cc ext/wasm/index.html index def70cce03,0000000000..435040434f mode 100644,000000..100644 --- a/ext/wasm/index.html +++ b/ext/wasm/index.html @@@ -1,54 -1,0 +1,56 @@@ + + + + + + + + sqlite3 WASM Testing Page Index + + +
sqlite3 WASM test pages
+
+
Below is the list of test pages for the sqlite3 WASM + builds. All of them require that this directory have been + "make"d first. The intent is that this page be run + using:
+ 
althttpd -page index.html
+
and the individual tests be started in their own tab.
+
Warnings and Caveats: +
    +
  • Some of these pages require that + the web server emit the so-called COOP and COEP headers. The + default build of althttpd does not. +
    • +
  • Whether or not WASMFS/OPFS support is enabled on any given + page may depend on build-time options which are off by + default because they currently (as of 2022-09-08) break + with Worker-based pages. +
    • +
+
+
The tests... +
    +
  • testing1: sanity tests of the core APIs and surrounding utility code.
    • +
  • testing2: Worker-based test of OO API #1.
    • +
  • testing-worker1-promiser: + tests for the Promise-based wrapper of the Worker-based API.
    • +
  • batch-runner: runs batches of SQL exported from speedtest1.
    • +
  • speedtest1: a main-thread WASM build of speedtest1.
    • +
  • speedtest1-worker: an interactive Worker-thread variant of speedtest1.
    • +
  • demo-oo1: demonstration of the OO API #1.
    • ++
  • kvvfs1: very basic demo of using the key-value vfs for storing ++ a persistent db in JS localStorage or sessionStorage.
    • + +
+
+ + + + diff --cc ext/wasm/kvvfs.make index 0000000000,83a2691370..a65f2faf26 mode 000000,100644..100644 --- a/ext/wasm/kvvfs.make +++ b/ext/wasm/kvvfs.make @@@ -1,0 -1,82 +1,84 @@@ + #!/usr/bin/make + #^^^^ help emacs select makefile mode + # + # This is a sub-make for building a standalone kvvfs-based + # sqlite3.wasm. It is intended to be "include"d from the main + # GNUMakefile. + # + # Notable potential TODOs: + # + # - Trim down a custom sqlite3-api.js for this build. We can elimate + # the jaccwabyt dependency, for example, because this build won't + # make use of the VFS bits. Similarly, we can eliminate or replace + # parts of the OO1 API, or provide a related API which manages + # singletons of the localStorage/sessionStorage instances. + # + ######################################################################## + MAKEFILE.kvvfs := $(lastword $(MAKEFILE_LIST)) + + kvvfs.js := sqlite3-kvvfs.js + kvvfs.wasm := sqlite3-kvvfs.wasm + kvvfs.wasm.c := $(dir.api)/sqlite3-wasm.c + + CLEAN_FILES += $(kvvfs.js) $(kvvfs.wasm) + + ######################################################################## + # emcc flags for .c/.o/.wasm. + kvvfs.flags = + #kvvfs.flags += -v # _very_ loud but also informative about what it's doing + + ######################################################################## + # emcc flags for .c/.o. + kvvfs.cflags := -kvvfs.cflags += -std=c99 -fPIC ++kvvfs.cflags += -std=c99 -fPIC -g + kvvfs.cflags += -I. -I$(dir.top) + kvvfs.cflags += -DSQLITE_OS_KV=1 $(SQLITE_OPT) + + ######################################################################## + # emcc flags specific to building the final .js/.wasm file... + kvvfs.jsflags := -fPIC + kvvfs.jsflags += --no-entry ++kvvfs.jsflags += --minify 0 + kvvfs.jsflags += -sENVIRONMENT=web + kvvfs.jsflags += -sMODULARIZE + kvvfs.jsflags += -sSTRICT_JS + kvvfs.jsflags += -sDYNAMIC_EXECUTION=0 + kvvfs.jsflags += -sNO_POLYFILL + kvvfs.jsflags += -sEXPORTED_FUNCTIONS=@$(dir.api)/EXPORTED_FUNCTIONS.sqlite3-api + kvvfs.jsflags += -sEXPORTED_RUNTIME_METHODS=FS,wasmMemory,allocateUTF8OnStack + # wasmMemory ==> for -sIMPORTED_MEMORY + # allocateUTF8OnStack ==> kvvfs internals + kvvfs.jsflags += -sUSE_CLOSURE_COMPILER=0 + kvvfs.jsflags += -sIMPORTED_MEMORY + #kvvfs.jsflags += -sINITIAL_MEMORY=13107200 + #kvvfs.jsflags += -sTOTAL_STACK=4194304 + kvvfs.jsflags += -sEXPORT_NAME=sqlite3InitModule + kvvfs.jsflags += -sGLOBAL_BASE=4096 # HYPOTHETICALLY keep func table indexes from overlapping w/ heap addr. + kvvfs.jsflags += --post-js=$(post-js.js) + #kvvfs.jsflags += -sFILESYSTEM=0 # only for experimentation. sqlite3 needs the FS API + # Perhaps the kvvfs build doesn't? + #kvvfs.jsflags += -sABORTING_MALLOC + kvvfs.jsflags += -sALLOW_MEMORY_GROWTH + kvvfs.jsflags += -sALLOW_TABLE_GROWTH + kvvfs.jsflags += -Wno-limited-postlink-optimizations + # ^^^^^ it likes to warn when we have "limited optimizations" via the -g3 flag. + kvvfs.jsflags += -sERROR_ON_UNDEFINED_SYMBOLS=0 + kvvfs.jsflags += -sLLD_REPORT_UNDEFINED + #kvvfs.jsflags += --import-undefined + kvvfs.jsflags += -sMEMORY64=0 + ifneq (0,$(enable_bigint)) + kvvfs.jsflags += -sWASM_BIGINT + endif + + $(kvvfs.js): $(MAKEFILE) $(MAKEFILE.kvvfs) $(kvvfs.wasm.c) \ + EXPORTED_FUNCTIONS.api \ + $(post-js.js) + $(emcc.bin) -o $@ $(emcc_opt) $(emcc.flags) $(kvvfs.cflags) $(kvvfs.jsflags) $(kvvfs.wasm.c) + chmod -x $(kvvfs.wasm) + ifneq (,$(wasm-strip)) + $(wasm-strip) $(kvvfs.wasm) + endif + @ls -la $@ $(kvvfs.wasm) + + kvvfs: $(kvvfs.js) ++all: kvvfs diff --cc ext/wasm/kvvfs1.html index 0000000000,0657920c53..773de0a603 mode 000000,100644..100644 --- a/ext/wasm/kvvfs1.html +++ b/ext/wasm/kvvfs1.html @@@ -1,0 -1,34 +1,43 @@@ + + + + + + + + + sqlite3-kvvfs.js tests + + +
sqlite3-kvvfs.js tests
+ +
+ +
Initializing app...
+
+ On a slow internet connection this may take a moment. If this + message displays for "a long time", intialization may have + failed and the JavaScript console may contain clues as to why. +
+
+
Downloading...
+
+ +
-
Everything on this page happens in the dev console.
++
Everything on this page happens in the dev console. TODOs for this demo include, ++ but are not necessarily limited to: ++ ++
    ++
  • UI controls to switch between localStorage and sessionStorage
    • ++
  • Button to clear storage.
    • ++
  • Button to dump the current db contents.
    • ++ ++
++
+
+
+ + + + + diff --cc ext/wasm/kvvfs1.js index 0000000000,f56f4874e0..169fcc8bdf mode 000000,100644..100644 --- a/ext/wasm/kvvfs1.js +++ b/ext/wasm/kvvfs1.js @@@ -1,0 -1,84 +1,73 @@@ + /* + 2022-09-12 + + The author disclaims copyright to this source code. In place of a + legal notice, here is a blessing: + + * May you do good and not evil. + * May you find forgiveness for yourself and forgive others. + * May you share freely, never taking more than you give. + + *********************************************************************** + + A basic test script for sqlite3-kvvfs.wasm. This file must be run in + main JS thread and sqlite3-kvvfs.js must have been loaded before it. + */ + 'use strict'; + (function(){ + const T = self.SqliteTestUtil; + const toss = function(...args){throw new Error(args.join(' '))}; + const debug = console.debug.bind(console); + const eOutput = document.querySelector('#test-output'); + const log = console.log.bind(console) + const logHtml = function(...args){ + log.apply(this, args); + const ln = document.createElement('div'); + ln.append(document.createTextNode(args.join(' '))); + eOutput.append(ln); + }; + + const runTests = function(Module){ + //log("Module",Module); + const sqlite3 = Module.sqlite3, + capi = sqlite3.capi, + oo = sqlite3.oo1, + wasm = capi.wasm; + log("Loaded module:",capi.sqlite3_libversion(), capi.sqlite3_sourceid()); + log("Build options:",wasm.compileOptionUsed()); - self.S = sqlite3; - T.assert(0 === capi.sqlite3_vfs_find(null)); - S.capi.sqlite3_initialize(); - T.assert( Number.isFinite( capi.sqlite3_vfs_find(null) ) ); - const stores = { - local: localStorage, - session: sessionStorage - }; - const cleanupStore = function(n){ - const s = stores[n]; - const isKv = (key)=>key.startsWith('kvvfs-'+n); - let i, k, toRemove = []; - for( i = 0; (k = s.key(i)); ++i) { - if(isKv(k)) toRemove.push(k); - } - toRemove.forEach((k)=>s.removeItem(k)); - }; - const dbStorage = 1 ? 'session' : 'local'; - const db = new oo.DB(dbStorage); ++ T.assert( 0 !== capi.sqlite3_vfs_find(null) ); ++ ++ const dbStorage = 1 ? ':sessionStorage:' : ':localStorage:'; ++ /** ++ The names ':sessionStorage:' and ':localStorage:' are handled ++ via the DB class constructor, not the C level. In the C API, ++ the names "local" and "session" are the current (2022-09-12) ++ names for those keys, but that is subject to change. ++ */ ++ const db = new oo.DB( dbStorage ); ++ log("Storage backend:",db.filename /* note that the name was internally translated */); + try { + db.exec("create table if not exists t(a)"); + if(undefined===db.selectValue("select a from t limit 1")){ - log("New db. Populating.."); ++ log("New db. Populating. This DB will persist across page reloads."); + db.exec("insert into t(a) values(1),(2),(3)"); + }else{ + log("Found existing table data:"); + db.exec({ + sql: "select * from t order by a", + rowMode: 0, + callback: function(v){log(v)} + }); + } + }finally{ - const n = db.filename; + db.close(); - //cleanupStore(n); + } - - log("Init done. Proceed from the dev console."); ++ log("End of demo."); + }; + + sqlite3InitModule(self.sqlite3TestModule).then(function(theModule){ + console.warn("Installing Emscripten module as global EM for dev console access."); + self.EM = theModule; + runTests(theModule); + }); + })(); diff --cc manifest index cdac3ba49c,601e86a73d..c182819a97 --- a/manifest +++ b/manifest @@@ -1,9 -1,9 +1,9 @@@ - C Minor\scleanups\sand\sdocumentation\sin\sthe\swasm\spieces. - D 2022-09-11T16:59:40.674 -C Fix\suninitialized\svariable\sin\srollback-journal\sprocessing\sin\sos_kv.c -D 2022-09-12T15:59:35.676 ++C Merge\skv-vfs\sbranch\sinto\sfiddle-opfs\sbranch\sto\sadd\skvvfs-based\swasm\sbuild\sand\sdemo. ++D 2022-09-12T16:09:50.625 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 - F Makefile.in 918d18842ba16dac5ebd5f7913aa8fb4f39879c881deced83362ed44a14ab4c6 -F Makefile.in ee179f405fd5f8845473f888517c4ada46099306c33ae1f27dd1aef53fe8e867 ++F Makefile.in 50e421194df031f669667fdb238c54959ecbea5a0b97dd3ed776cffbeea926d5 F Makefile.linux-gcc f609543700659711fbd230eced1f01353117621dccae7b9fb70daa64236c5241 F Makefile.msc d547a2fdba38a1c6cd1954977d0b0cc017f5f8fbfbc65287bf8d335808938016 F README.md 8b8df9ca852aeac4864eb1e400002633ee6db84065bd01b78c33817f97d31f5e @@@ -472,59 -472,43 +472,62 @@@ F ext/session/test_session.c f433f68a8a F ext/userauth/sqlite3userauth.h 7f3ea8c4686db8e40b0a0e7a8e0b00fac13aa7a3 F ext/userauth/user-auth.txt e6641021a9210364665fe625d067617d03f27b04 F ext/userauth/userauth.c 7f00cded7dcaa5d47f54539b290a43d2e59f4b1eb5f447545fa865f002fc80cb -F ext/wasm/EXPORTED_FUNCTIONS.fiddle 7fb73f7150ab79d83bb45a67d257553c905c78cd3d693101699243f36c5ae6c3 +F ext/wasm/EXPORTED_FUNCTIONS.fiddle db7a4602f043cf4a5e4135be3609a487f9f1c83f05778bfbdf93766be4541b96 F ext/wasm/EXPORTED_RUNTIME_METHODS.fiddle a004bd5eeeda6d3b28d16779b7f1a80305bfe009dfc7f0721b042967f0d39d02 - F ext/wasm/GNUmakefile 18b80a063c684b0ca44b96221fc2c23647d0196f757c83c7a572b853562d8ac9 -F ext/wasm/GNUmakefile 12a672ab9125dc860457c2853f7651b98517e424d7a0e9714c89b28c5ff73800 -F ext/wasm/README.md 4b00ae7c7d93c4591251245f0996a319e2651361013c98d2efb0b026771b7331 -F ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api c5eaceabb9e759aaae7d3101a4a3e542f96ab2c99d89a80ce20ec18c23115f33 ++F ext/wasm/GNUmakefile 6e642a0dc7ac43d9287e8f31c80ead469ddc7475a6d4ab7ac3b1feefcd4f7279 +F ext/wasm/README.md e1ee1e7c321c6a250bf78a84ca6f5882890a237a450ba5a0649c7a8399194c52 +F ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api 1dfd067b3cbd9a49cb204097367cf2f8fe71b5a3b245d9d82a24779fd4ac2394 F ext/wasm/api/EXPORTED_RUNTIME_METHODS.sqlite3-api 1ec3c73e7d66e95529c3c64ac3de2470b0e9e7fbf7a5b41261c367cf4f1b7287 -F ext/wasm/api/README.md b6d0fb64bfdf7bf9ce6938ea4104228f6f5bbef600f5d910b2f8c8694195988c +F ext/wasm/api/README.md d876597edd2b9542b6ea031adaaff1c042076fde7b670b1dc6d8a87b28a6631b F ext/wasm/api/post-js-footer.js b64319261d920211b8700004d08b956a6c285f3b0bba81456260a713ed04900c F ext/wasm/api/post-js-header.js 0e853b78db83cb1c06b01663549e0e8b4f377f12f5a2d9a4a06cb776c003880b - F ext/wasm/api/sqlite3-api-cleanup.js 1a12e64060c2cb0defd34656a76a9b1d7ed58459c290249bb31567c806fd44de -F ext/wasm/api/sqlite3-api-cleanup.js 149fd63a0400cd1d69548887ffde2ed89c13283384a63c2e9fcfc695e38a9e11 -F ext/wasm/api/sqlite3-api-glue.js 82c09f49c69984009ba5af2b628e67cc26c5dd203d383cd3091d40dab4e6514b -F ext/wasm/api/sqlite3-api-oo1.js e9612cb704c0563c5d71ed2a8dccd95bf6394fa4de3115d1b978dc269c49ab02 -F ext/wasm/api/sqlite3-api-opfs.js c93cdd14f81a26b3a64990515ee05c7e29827fbc8fba4e4c2fef3a37a984db89 -F ext/wasm/api/sqlite3-api-prologue.js 0fb0703d2d8ac89fa2d4dd8f9726b0ea226b8708ac34e5b482df046e147de0eb -F ext/wasm/api/sqlite3-api-worker.js 1124f404ecdf3c14d9f829425cef778cd683911a9883f0809a463c3c7773c9fd ++F ext/wasm/api/sqlite3-api-cleanup.js 101919ec261644e2f6f0a59952fd9612127b69ea99b493277b2789ea478f9b6b +F ext/wasm/api/sqlite3-api-glue.js 2bf536a38cde324cf352bc2c575f8e22c6d204d667c0eda5a254ba45318914bc - F ext/wasm/api/sqlite3-api-oo1.js b06a1ac982c7d433396b8304550ce1493a63671a3bf653c3b5646a9075e0ca41 ++F ext/wasm/api/sqlite3-api-oo1.js a9d8892be246548a9978ace506d108954aa13eb5ce25332975c8377953804ff3 +F ext/wasm/api/sqlite3-api-opfs.js 011799db398157cbd254264b6ebae00d7234b93d0e9e810345f213a5774993c0 - F ext/wasm/api/sqlite3-api-prologue.js afd753be49ecba4b5bfc3e54929826a01eaf96ac2d8cec575d683e3c5bcc8464 ++F ext/wasm/api/sqlite3-api-prologue.js 9e37ce4dfd74926d0df80dd7e72e33085db4bcee48e2c21236039be416a7dff2 +F ext/wasm/api/sqlite3-api-worker1.js d33062afa045fd4be01ba4abc266801807472558b862b30056211b00c9c347b4 F ext/wasm/api/sqlite3-wasi.h 25356084cfe0d40458a902afb465df8c21fc4152c1d0a59b563a3fba59a068f9 - F ext/wasm/api/sqlite3-wasm.c ab8abf26708238a17840cad224bec1511efde6e5b182068f3abf079ba1d83ef6 -F ext/wasm/api/sqlite3-wasm.c 8585793ca8311c7a0618b7e00ed2b3729799c20664a51f196258576e3d475c9e -F ext/wasm/api/sqlite3-worker.js 1325ca8d40129a82531902a3a077b795db2eeaee81746e5a0c811a04b415fa7f -F ext/wasm/common/SqliteTestUtil.js e41a1406f18da9224523fad0c48885caf995b56956a5b9852909c0989e687e90 ++F ext/wasm/api/sqlite3-wasm.c bf4637cf28463cada4b25f09651943c7ece004b253ef39b7ab68eaa60662aa09 +F ext/wasm/batch-runner.html 23209ade7981acce7ecd79d6eff9f4c5a4e8b14ae867ac27cd89b230be640fa6 +F ext/wasm/batch-runner.js 2abd146d3e3a66128ac0a2cc39bfd01e9811c9511fa10ec927d6649795f1ee50 +F ext/wasm/common/SqliteTestUtil.js 529161a624265ba84271a52db58da022649832fa1c71309fb1e02cc037327a2b F ext/wasm/common/emscripten.css 3d253a6fdb8983a2ac983855bfbdd4b6fa1ff267c28d69513dd6ef1f289ada3f -F ext/wasm/common/testing.css 572cf1ffae0b6eb7ca63684d3392bf350217a07b90e7a896e4fa850700c989b0 -F ext/wasm/common/whwasmutil.js 3d9deda1be718e2b10e2b6b474ba6ba857d905be314201ae5b3df5eef79f66aa +F ext/wasm/common/testing.css 3a5143699c2b73a85b962271e1a9b3241b30d90e30d895e4f55665e648572962 +F ext/wasm/common/whwasmutil.js f7282ef36c9625330d4e6e82d1beec6678cd101e95e7108cd85db587a788c145 +F ext/wasm/demo-oo1.html 75646855b38405d82781246fd08c852a2b3bee05dd9f0fe10ab655a8cffb79aa +F ext/wasm/demo-oo1.js 477f230cce3455e701431436d892d8c6bfea2bdf1ddcdd32a273e2f4bb339801 F ext/wasm/fiddle/emscripten.css 3d253a6fdb8983a2ac983855bfbdd4b6fa1ff267c28d69513dd6ef1f289ada3f -F ext/wasm/fiddle/fiddle-worker.js 88bc2193a6cb6a3f04d8911bed50a4401fe6f277de7a71ba833865ab64a1b4ae +F ext/wasm/fiddle/fiddle-worker.js bccf46045be8824752876f3eec01c223be0616ccac184bffd0024cfe7a3262b8 F ext/wasm/fiddle/fiddle.html 550c5aafce40bd218de9bf26192749f69f9b10bc379423ecd2e162bcef885c08 -F ext/wasm/fiddle/fiddle.js 812f9954cc7c4b191884ad171f36fcf2d0112d0a7ecfdf6087896833a0c079a8 -F ext/wasm/jaccwabyt/jaccwabyt.js 99b424b4d467d4544e82615b58e2fe07532a898540bf9de2a985f3c21e7082b2 +F ext/wasm/fiddle/fiddle.js 4ffcfc9a235beebaddec689a549e9e0dfad6dca5c1f0b41f03468d7e76480686 - F ext/wasm/index.html 4f635f986dbc7518280abe0ef537ba41682e35f160fac35a0745cf6c4d223b62 ++F ext/wasm/index.html 5876ae0442bef5b37fec9a45ee0722798e47ef727723ada742d33554845afa6a +F ext/wasm/jaccwabyt/jaccwabyt.js 0d7f32817456a0f3937fcfd934afeb32154ca33580ab264dab6c285e6dbbd215 F ext/wasm/jaccwabyt/jaccwabyt.md 447cc02b598f7792edaa8ae6853a7847b8178a18ed356afacbdbf312b2588106 F ext/wasm/jaccwabyt/jaccwabyt_test.c 39e4b865a33548f943e2eb9dd0dc8d619a80de05d5300668e9960fff30d0d36f F ext/wasm/jaccwabyt/jaccwabyt_test.exports 5ff001ef975c426ffe88d7d8a6e96ec725e568d2c2307c416902059339c06f19 -F ext/wasm/kvvfs.make 7cc9cf10e744c3ba523c3eaf5c4af47028f3a5bb76db304ea8044a9b2a9d496f -F ext/wasm/kvvfs1.html 2acb241a6110a4ec581adbf07a23d5fc2ef9c7142aa9d60856732a102abc5016 -F ext/wasm/kvvfs1.js 46afaf4faba041bf938355627bc529854295e561f49db3a240c914e75a529338 -F ext/wasm/testing1.html 0bf3ff224628c1f1e3ed22a2dc1837c6c73722ad8c0ad9c8e6fb9e6047667231 -F ext/wasm/testing1.js cba7134901a965743fa9289d82447ab71de4690b1ee5d06f6cb83e8b569d7943 -F ext/wasm/testing2.html 73e5048e666fd6fb28b6e635677a9810e1e139c599ddcf28d687c982134b92b8 -F ext/wasm/testing2.js d37433c601f88ed275712c1cfc92d3fb36c7c22e1ed8c7396fb2359e42238ebc ++F ext/wasm/kvvfs.make dba616578bf91a76370a46494dd68a09c6dff5beb6d5561e2db65a27216e9630 ++F ext/wasm/kvvfs1.html b8304cd5c7e7ec32c3b15521a95c322d6efdb8d22b3c4156123545dc54e07583 ++F ext/wasm/kvvfs1.js a5075f98ffecd7d32348697db991fc61342d89aa20651034d1572af61890fb8b +F ext/wasm/scratchpad-opfs-main.html 4565cf194e66188190d35f70e82553e2e2d72b9809b73c94ab67b8cfd14d2e0c +F ext/wasm/scratchpad-opfs-main.js 69e960e9161f6412fd0c30f355d4112f1894d6609eb431e2d16d207d1380518e +F ext/wasm/scratchpad-opfs-worker.html 66c1d15d678f3bd306373d76b61c6c8aef988f61f4a8dd40185d452f9c6d2bf5 +F ext/wasm/scratchpad-opfs-worker.js 3ec2868c669713145c76eb5877c64a1b20741f741817b87c907a154b676283a9 +F ext/wasm/scratchpad-opfs-worker2.js 5f2237427ac537b8580b1c659ff14ad2621d1694043eaaf41ae18dbfef2e48c0 +F ext/wasm/speedtest1-worker.html 6b5fda04d0b69e8c2651689356cb0c28fd33aa1a82b03dcbc8b0d68fbd7ed57f +F ext/wasm/speedtest1-worker.js 356b9953add4449acf199793db9b76b11ee016021918d8daffd19f08ec68d305 +F ext/wasm/speedtest1.html 8f61cbe68300acca25dd9fa74dce79b774786e2b4feeb9bcbc46e1cefbfa6262 +F ext/wasm/split-speedtest1-script.sh a3e271938d4d14ee49105eb05567c6a69ba4c1f1293583ad5af0cd3a3779e205 x +F ext/wasm/sql/000-mandelbrot.sql 775337a4b80938ac8146aedf88808282f04d02d983d82675bd63d9c2d97a15f0 +F ext/wasm/sql/001-sudoku.sql 35b7cb7239ba5d5f193bc05ec379bcf66891bce6f2a5b3879f2f78d0917299b5 +F ext/wasm/sqlite3-worker1-promiser.js 92b8da5f38439ffec459a8215775d30fa498bc0f1ab929ff341fc3dd479660b9 +F ext/wasm/sqlite3-worker1.js 0c1e7626304543969c3846573e080c082bf43bcaa47e87d416458af84f340a9e +F ext/wasm/testing-worker1-promiser.html 6eaec6e04a56cf24cf4fa8ef49d78ce8905dde1354235c9125dca6885f7ce893 +F ext/wasm/testing-worker1-promiser.js c62b5879339eef0b21aebd9d75bc125c86530edc17470afff18077f931cb704a +F ext/wasm/testing1.html 528001c7e32ee567abc195aa071fd9820cc3c8ffc9c8a39a75e680db05f0c409 +F ext/wasm/testing1.js 2def7a86c52ff28b145cb86188d5c7a49d5993f9b78c50d140e1c31551220955 +F ext/wasm/testing2.html a66951c38137ff1d687df79466351f3c734fa9c6d9cce71d3cf97c291b2167e3 +F ext/wasm/testing2.js 25584bcc30f19673ce13a6f301f89f8820a59dfe044e0c4f2913941f4097fe3c F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60 @@@ -590,8 -574,9 +593,9 @@@ F src/notify.c 89a97dc854c3aa62ad5f384e F src/os.c 0eb831ba3575af5277e47f4edd14fdfc90025c67eb25ce5cda634518d308d4e9 F src/os.h 1ff5ae51d339d0e30d8a9d814f4b8f8e448169304d83a7ed9db66a65732f3e63 F src/os_common.h b2f4707a603e36811d9b1a13278bffd757857b85 - F src/os_setup.h 0dbaea40a7d36bf311613d31342e0b99e2536586 - F src/os_unix.c 102f7e5c5b59c18ea3dbc929dc3be8acb3afc0e0b6ad572e032335c9c27f44f1 -F src/os_kv.c 2b4c04a470c05fe95a84d2ba3a5eb4874f0dbaa12da3a47f221ee3beec7eeda0 ++F src/os_kv.c a188e92dac693b1c1b512d93b0c4dc85c1baad11e322b01121f87057996e4d11 + F src/os_setup.h 0711dbc4678f3ac52d7fe736951b6384a0615387c4ba5135a4764e4e31f4b6a6 + F src/os_unix.c d6322b78130d995160bb9cfb7850678ad6838b08c1d13915461b33326a406c04 F src/os_win.c e9454cb141908e8eef2102180bad353a36480612d5b736e4c2bd5777d9b25a34 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a F src/pager.c 6176d9752eb580419e8fef4592dc417a6b00ddfd43ee22f818819bf8840ceee8 @@@ -2019,8 -2004,8 +2023,8 @@@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a9 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 - P cdbf09fa1b0c93aeb3222a157de33a4688ae629c2b829ffff0f1f62364c5ae1c - R 9d278bdeb3b04c59e81ef94ad61886ea -P 250a935aeb94d3fadec0d3fe22de85de4e658e2fdb3be3aa9a8bbc8f7b7d8414 -R b0d44b8764d44e0d25567b7049186ee6 -U drh -Z b805cc7ee3bf0477637e784d78058472 ++P 4e6ce329872eb733ba2f7f7879747c52761ae97790fd8ed169a25a79854cc3d9 e49682c5eac91958f143e639c5656ca54560d14f5805d514bf4aa0c206e63844 ++R f15733833cf5e73dce86b4365f218581 +U stephan - Z 1e71f5bdda12caaeb1881e6b3d2fda00 ++Z f2a01bf4c99986993a2e00b39e93be73 # Remove this line to create a well-formed Fossil manifest. diff --cc manifest.uuid index 0742ec8643,3839586165..bde2102af8 --- a/manifest.uuid +++ b/manifest.uuid @@@ -1,1 -1,1 +1,1 @@@ - 4e6ce329872eb733ba2f7f7879747c52761ae97790fd8ed169a25a79854cc3d9 -e49682c5eac91958f143e639c5656ca54560d14f5805d514bf4aa0c206e63844 ++a7d8b26acd3c1ae344369e4d70804c0cab45272c0983cfd32d616a0a7b28acb9 diff --cc src/os_kv.c index 0000000000,9260852dbe..a14dc5c543 mode 000000,100644..100644 --- a/src/os_kv.c +++ b/src/os_kv.c @@@ -1,0 -1,1077 +1,1077 @@@ + /* + ** 2022-09-06 + ** + ** The author disclaims copyright to this source code. In place of + ** a legal notice, here is a blessing: + ** + ** May you do good and not evil. + ** May you find forgiveness for yourself and forgive others. + ** May you share freely, never taking more than you give. + ** + ****************************************************************************** + ** + ** This file contains an experimental VFS layer that operates on a + ** Key/Value storage engine where both keys and values must be pure + ** text. + */ + #include + #if SQLITE_OS_KV + + /***************************************************************************** + ** Debugging logic + */ + + /* SQLITE_KV_TRACE() is used for tracing calls to kvstorage routines. */ + #if 0 + #define SQLITE_KV_TRACE(X) printf X; + #else + #define SQLITE_KV_TRACE(X) + #endif + + /* SQLITE_KV_LOG() is used for tracing calls to the VFS interface */ + #if 0 + #define SQLITE_KV_LOG(X) printf X; + #else + #define SQLITE_KV_LOG(X) + #endif + + + /* + ** Forward declaration of objects used by this VFS implementation + */ + typedef struct KVVfsFile KVVfsFile; + + /* A single open file. There are only two files represented by this + ** VFS - the database and the rollback journal. + */ + struct KVVfsFile { + sqlite3_file base; /* IO methods */ + const char *zClass; /* Storage class */ + int isJournal; /* True if this is a journal file */ + unsigned int nJrnl; /* Space allocated for aJrnl[] */ + char *aJrnl; /* Journal content */ + int szPage; /* Last known page size */ + sqlite3_int64 szDb; /* Database file size. -1 means unknown */ + }; + + /* + ** Methods for KVVfsFile + */ + static int kvvfsClose(sqlite3_file*); + static int kvvfsReadDb(sqlite3_file*, void*, int iAmt, sqlite3_int64 iOfst); + static int kvvfsReadJrnl(sqlite3_file*, void*, int iAmt, sqlite3_int64 iOfst); + static int kvvfsWriteDb(sqlite3_file*,const void*,int iAmt, sqlite3_int64); + static int kvvfsWriteJrnl(sqlite3_file*,const void*,int iAmt, sqlite3_int64); + static int kvvfsTruncateDb(sqlite3_file*, sqlite3_int64 size); + static int kvvfsTruncateJrnl(sqlite3_file*, sqlite3_int64 size); + static int kvvfsSyncDb(sqlite3_file*, int flags); + static int kvvfsSyncJrnl(sqlite3_file*, int flags); + static int kvvfsFileSizeDb(sqlite3_file*, sqlite3_int64 *pSize); + static int kvvfsFileSizeJrnl(sqlite3_file*, sqlite3_int64 *pSize); + static int kvvfsLock(sqlite3_file*, int); + static int kvvfsUnlock(sqlite3_file*, int); + static int kvvfsCheckReservedLock(sqlite3_file*, int *pResOut); + static int kvvfsFileControl(sqlite3_file*, int op, void *pArg); + static int kvvfsSectorSize(sqlite3_file*); + static int kvvfsDeviceCharacteristics(sqlite3_file*); + + /* + ** Methods for sqlite3_vfs + */ + static int kvvfsOpen(sqlite3_vfs*, const char *, sqlite3_file*, int , int *); + static int kvvfsDelete(sqlite3_vfs*, const char *zName, int syncDir); + static int kvvfsAccess(sqlite3_vfs*, const char *zName, int flags, int *); + static int kvvfsFullPathname(sqlite3_vfs*, const char *zName, int, char *zOut); + static void *kvvfsDlOpen(sqlite3_vfs*, const char *zFilename); + static int kvvfsRandomness(sqlite3_vfs*, int nByte, char *zOut); + static int kvvfsSleep(sqlite3_vfs*, int microseconds); + static int kvvfsCurrentTime(sqlite3_vfs*, double*); + static int kvvfsCurrentTimeInt64(sqlite3_vfs*, sqlite3_int64*); + + static sqlite3_vfs kvvfs_vfs = { + 1, /* iVersion */ + sizeof(KVVfsFile), /* szOsFile */ + 1024, /* mxPathname */ + 0, /* pNext */ + "kvvfs", /* zName */ + 0, /* pAppData */ + kvvfsOpen, /* xOpen */ + kvvfsDelete, /* xDelete */ + kvvfsAccess, /* xAccess */ + kvvfsFullPathname, /* xFullPathname */ + kvvfsDlOpen, /* xDlOpen */ + 0, /* xDlError */ + 0, /* xDlSym */ + 0, /* xDlClose */ + kvvfsRandomness, /* xRandomness */ + kvvfsSleep, /* xSleep */ + kvvfsCurrentTime, /* xCurrentTime */ + 0, /* xGetLastError */ + kvvfsCurrentTimeInt64 /* xCurrentTimeInt64 */ + }; + + /* Methods for sqlite3_file objects referencing a database file + */ + static sqlite3_io_methods kvvfs_db_io_methods = { + 1, /* iVersion */ + kvvfsClose, /* xClose */ + kvvfsReadDb, /* xRead */ + kvvfsWriteDb, /* xWrite */ + kvvfsTruncateDb, /* xTruncate */ + kvvfsSyncDb, /* xSync */ + kvvfsFileSizeDb, /* xFileSize */ + kvvfsLock, /* xLock */ + kvvfsUnlock, /* xUnlock */ + kvvfsCheckReservedLock, /* xCheckReservedLock */ + kvvfsFileControl, /* xFileControl */ + kvvfsSectorSize, /* xSectorSize */ + kvvfsDeviceCharacteristics, /* xDeviceCharacteristics */ + 0, /* xShmMap */ + 0, /* xShmLock */ + 0, /* xShmBarrier */ + 0, /* xShmUnmap */ + 0, /* xFetch */ + 0 /* xUnfetch */ + }; + + /* Methods for sqlite3_file objects referencing a rollback journal + */ + static sqlite3_io_methods kvvfs_jrnl_io_methods = { + 1, /* iVersion */ + kvvfsClose, /* xClose */ + kvvfsReadJrnl, /* xRead */ + kvvfsWriteJrnl, /* xWrite */ + kvvfsTruncateJrnl, /* xTruncate */ + kvvfsSyncJrnl, /* xSync */ + kvvfsFileSizeJrnl, /* xFileSize */ + kvvfsLock, /* xLock */ + kvvfsUnlock, /* xUnlock */ + kvvfsCheckReservedLock, /* xCheckReservedLock */ + kvvfsFileControl, /* xFileControl */ + kvvfsSectorSize, /* xSectorSize */ + kvvfsDeviceCharacteristics, /* xDeviceCharacteristics */ + 0, /* xShmMap */ + 0, /* xShmLock */ + 0, /* xShmBarrier */ + 0, /* xShmUnmap */ + 0, /* xFetch */ + 0 /* xUnfetch */ + }; + + /****** Storage subsystem **************************************************/ + #include + #include + #include + + /* Forward declarations for the low-level storage engine + */ + #define KVSTORAGE_KEY_SZ 32 + + /* Expand the key name with an appropriate prefix and put the result + ** zKeyOut[]. The zKeyOut[] buffer is assumed to hold at least + ** KVSTORAGE_KEY_SZ bytes. + */ + static void kvstorageMakeKey( + const char *zClass, + const char *zKeyIn, + char *zKeyOut + ){ + sqlite3_snprintf(KVSTORAGE_KEY_SZ, zKeyOut, "kvvfs-%s-%s", zClass, zKeyIn); + } + + #ifdef __EMSCRIPTEN__ + /* Provide Emscripten-based impls of kvstorageWrite/Read/Delete()... */ + #include + #include + + /* + ** WASM_KEEP is identical to EMSCRIPTEN_KEEPALIVE but is not + ** Emscripten-specific. It explicitly includes marked functions for + ** export into the target wasm file without requiring explicit listing + ** of those functions in Emscripten's -sEXPORTED_FUNCTIONS=... list + ** (or equivalent in other build platforms). Any function with neither + ** this attribute nor which is listed as an explicit export will not + ** be exported from the wasm file (but may still be used internally + ** within the wasm file). + ** + ** The functions in this file (sqlite3-wasm.c) which require exporting + ** are marked with this flag. They may also be added to any explicit + ** build-time export list but need not be. All of these APIs are + ** intended for use only within the project's own JS/WASM code, and + ** not by client code, so an argument can be made for reducing their + ** visibility by not including them in any build-time export lists. + ** + ** 2022-09-11: it's not yet _proven_ that this approach works in + ** non-Emscripten builds. If not, such builds will need to export + ** those using the --export=... wasm-ld flag (or equivalent). As of + ** this writing we are tied to Emscripten for various reasons + ** and cannot test the library with other build environments. + */ + #define WASM_KEEP __attribute__((used,visibility("default"))) + /* + ** An internal level of indirection for accessing the static + ** kvstorageMakeKey() from EM_JS()-generated functions. This must be + ** made available for export via Emscripten but is not intended to be + ** used from client code. If called with a NULL zKeyOut it is a no-op. + ** It returns KVSTORAGE_KEY_SZ, so JS code (which cannot see that + ** constant) may call it with NULL arguments to get the size of the + ** allocation they'll need for a kvvfs key. + ** + ** Maintenance reminder: Emscripten will install this in the Module + ** init scope and will prefix its name with "_". + */ + WASM_KEEP + int sqlite3_wasm__kvvfsMakeKey(const char *zClass, const char *zKeyIn, + char *zKeyOut){ + if( 0!=zKeyOut ) kvstorageMakeKey(zClass, zKeyIn, zKeyOut); + return KVSTORAGE_KEY_SZ; + } + /* + ** Internal helper for kvstorageWrite/Read/Delete() which creates a + ** storage key for the given zClass/zKeyIn combination. Returns a + ** pointer to the key: a C string allocated on the WASM stack, or 0 if + ** allocation fails. It is up to the caller to save/restore the stack + ** before/after this operation. + */ + EM_JS(const char *, kvstorageMakeKeyOnJSStack, + (const char *zClass, const char *zKeyIn),{ + if( 0==zClass || 0==zKeyIn) return 0; + const zXKey = stackAlloc(_sqlite3_wasm__kvvfsMakeKey(0,0,0)); + if(zXKey) _sqlite3_wasm__kvvfsMakeKey(zClass, zKeyIn, zXKey); + return zXKey; + }); + + /* + ** JS impl of kvstorageWrite(). Main docs are in the C impl. This impl + ** writes zData to the global sessionStorage (if zClass starts with + ** 's') or localStorage, using a storage key derived from zClass and + ** zKey. + */ + EM_JS(int, kvstorageWrite, + (const char *zClass, const char *zKey, const char *zData),{ + const stack = stackSave(); + try { + const zXKey = kvstorageMakeKeyOnJSStack(zClass,zKey); + if(!zXKey) return 1/*OOM*/; + const jKey = UTF8ToString(zXKey); + /** + We could simplify this function and eliminate the + kvstorageMakeKey() symbol acrobatics if we'd simply hard-code + the key algo into the 3 functions which need it: + + const jKey = "kvvfs-"+UTF8ToString(zClass)+"-"+UTF8ToString(zKey); + */ + ((115/*=='s'*/===getValue(zClass)) + ? sessionStorage : localStorage).setItem(jKey, UTF8ToString(zData)); + }catch(e){ + console.error("kvstorageWrite()",e); + return 1; // Can't access SQLITE_xxx from here + }finally{ + stackRestore(stack); + } + return 0; + }); + + /* + ** JS impl of kvstorageDelete(). Main docs are in the C impl. This + ** impl generates a key derived from zClass and zKey, and removes the + ** matching entry (if any) from global sessionStorage (if zClass + ** starts with 's') or localStorage. + */ + EM_JS(int, kvstorageDelete, + (const char *zClass, const char *zKey),{ + const stack = stackSave(); + try { + const zXKey = kvstorageMakeKeyOnJSStack(zClass,zKey); + if(!zXKey) return 1/*OOM*/; + _sqlite3_wasm__kvvfsMakeKey(zClass, zKey, zXKey); + const jKey = UTF8ToString(zXKey); + ((115/*=='s'*/===getValue(zClass)) + ? sessionStorage : localStorage).removeItem(jKey); + }catch(e){ + console.error("kvstorageDelete()",e); + return 1; + }finally{ + stackRestore(stack); + } + return 0; + }); + + /* + ** JS impl of kvstorageRead(). Main docs are in the C impl. This impl + ** reads its data from the global sessionStorage (if zClass starts + ** with 's') or localStorage, using a storage key derived from zClass + ** and zKey. + */ + EM_JS(int, kvstorageRead, + (const char *zClass, const char *zKey, char *zBuf, int nBuf),{ + const stack = stackSave(); + try { + const zXKey = kvstorageMakeKeyOnJSStack(zClass,zKey); + if(!zXKey) return -3/*OOM*/; + const jKey = UTF8ToString(zXKey); + const jV = ((115/*=='s'*/===getValue(zClass)) + ? sessionStorage : localStorage).getItem(jKey); + if(!jV) return -1; + const nV = jV.length /* Note that we are relying 100% on v being + ASCII so that jV.length is equal to the + C-string's byte length. */; + if(nBuf<=0) return nV; + else if(1===nBuf){ + setValue(zBuf, 0); + return nV; + } + const zV = allocateUTF8OnStack(jV); + if(nBuf > nV + 1) nBuf = nV + 1; + HEAPU8.copyWithin(zBuf, zV, zV + nBuf - 1); + setValue( zBuf + nBuf - 1, 0 ); + return nBuf - 1; + }catch(e){ + console.error("kvstorageRead()",e); + return -2; + }finally{ + stackRestore(stack); + } + }); + + /* + ** This function exists for (1) WASM testing purposes and (2) as a + ** hook to get Emscripten to export several EM_JS()-generated + ** functions (if we don't reference them from exported C functions + ** then they get stripped away at build time). It is not part of the + ** public API and its signature and semantics may change at any time. + ** It's not even part of the private API, for that matter - it's part + ** of the Emscripten C/JS/WASM glue. + */ + WASM_KEEP + int sqlite3__wasm_emjs_kvvfs(int whichOp){ + int rc = 0; + const char * zClass = + "sezzion" /*don't collide with "session" records!*/; + const char * zKey = "hello"; + switch( whichOp ){ + case 0: break; + case 1: + kvstorageWrite(zClass, zKey, "world"); + break; + case 2: { + char buffer[128] = {0}; + char * zBuf = &buffer[0]; + rc = kvstorageRead(zClass, zKey, zBuf, (int)sizeof(buffer)); + emscripten_console_logf("kvstorageRead()=%d %s\n", rc, zBuf); + break; + } + case 3: + kvstorageDelete(zClass, zKey); + break; + case 4: + kvstorageMakeKeyOnJSStack(0,0); + break; + default: break; + } + return rc; + } + + #undef WASM_KEEP + #else /* end ifdef __EMSCRIPTEN__ */ + /* Forward declarations for the low-level storage engine + */ + static int kvstorageWrite(const char*, const char *zKey, const char *zData); + static int kvstorageDelete(const char*, const char *zKey); + static int kvstorageRead(const char*, const char *zKey, char *zBuf, int nBuf); + + /* Write content into a key. zClass is the particular namespace of the + ** underlying key/value store to use - either "local" or "session". + ** + ** Both zKey and zData are zero-terminated pure text strings. + ** + ** Return the number of errors. + */ + static int kvstorageWrite( + const char *zClass, + const char *zKey, + const char *zData + ){ + FILE *fd; + char zXKey[KVSTORAGE_KEY_SZ]; + kvstorageMakeKey(zClass, zKey, zXKey); + fd = fopen(zXKey, "wb"); + if( fd ){ + SQLITE_KV_TRACE(("KVVFS-WRITE %-15s (%d) %.50s%s\n", zXKey, + (int)strlen(zData), zData, + strlen(zData)>50 ? "..." : "")); + fputs(zData, fd); + fclose(fd); + return 0; + }else{ + return 1; + } + } + + /* Delete a key (with its corresponding data) from the key/value + ** namespace given by zClass. If the key does not previously exist, + ** this routine is a no-op. + */ + static int kvstorageDelete(const char *zClass, const char *zKey){ + char zXKey[KVSTORAGE_KEY_SZ]; + kvstorageMakeKey(zClass, zKey, zXKey); + unlink(zXKey); + SQLITE_KV_TRACE(("KVVFS-DELETE %-15s\n", zXKey)); + return 0; + } + + /* Read the value associated with a zKey from the key/value namespace given + ** by zClass and put the text data associated with that key in the first + ** nBuf bytes of zBuf[]. The value might be truncated if zBuf is not large + ** enough to hold it all. The value put into zBuf must always be zero + ** terminated, even if it gets truncated because nBuf is not large enough. + ** + ** Return the total number of bytes in the data, without truncation, and + ** not counting the final zero terminator. Return -1 if the key does + ** not exist. + ** + ** If nBuf<=0 then this routine simply returns the size of the data without + ** actually reading it. + */ + static int kvstorageRead( + const char *zClass, + const char *zKey, + char *zBuf, + int nBuf + ){ + FILE *fd; + struct stat buf; + char zXKey[KVSTORAGE_KEY_SZ]; + kvstorageMakeKey(zClass, zKey, zXKey); + if( access(zXKey, R_OK)!=0 + || stat(zXKey, &buf)!=0 + || !S_ISREG(buf.st_mode) + ){ + SQLITE_KV_TRACE(("KVVFS-READ %-15s (-1)\n", zXKey)); + return -1; + } + if( nBuf<=0 ){ + return (int)buf.st_size; + }else if( nBuf==1 ){ + zBuf[0] = 0; + SQLITE_KV_TRACE(("KVVFS-READ %-15s (%d)\n", zXKey, + (int)buf.st_size)); + return (int)buf.st_size; + } + if( nBuf > buf.st_size + 1 ){ + nBuf = buf.st_size + 1; + } + fd = fopen(zXKey, "rb"); + if( fd==0 ){ + SQLITE_KV_TRACE(("KVVFS-READ %-15s (-1)\n", zXKey)); + return -1; + }else{ + sqlite3_int64 n = fread(zBuf, 1, nBuf-1, fd); + fclose(fd); + zBuf[n] = 0; + SQLITE_KV_TRACE(("KVVFS-READ %-15s (%lld) %.50s%s\n", zXKey, + n, zBuf, n>50 ? "..." : "")); + return (int)n; + } + } + #endif /* ifdef __EMSCRIPTEN__ */ + + /****** Utility subroutines ************************************************/ + + /* + ** Encode binary into the text encoded used to persist on disk. + ** The output text is stored in aOut[], which must be at least + ** nData+1 bytes in length. + ** + ** Return the actual length of the encoded text, not counting the + ** zero terminator at the end. + ** + ** Encoding format + ** --------------- + ** + ** * Non-zero bytes are encoded as upper-case hexadecimal + ** + ** * A sequence of one or more zero-bytes that are not at the + ** beginning of the buffer are encoded as a little-endian + ** base-26 number using a..z. "a" means 0. "b" means 1, + ** "z" means 25. "ab" means 26. "ac" means 52. And so forth. + ** + ** * Because there is no overlap between the encoding characters + ** of hexadecimal and base-26 numbers, it is always clear where + ** one stops and the next begins. + */ + static int kvvfsEncode(const char *aData, int nData, char *aOut){ + int i, j; + const unsigned char *a = (const unsigned char*)aData; + for(i=j=0; i>4]; + aOut[j++] = "0123456789ABCDEF"[c&0xf]; + }else{ + /* A sequence of 1 or more zeros is stored as a little-endian + ** base-26 number using a..z as the digits. So one zero is "b". + ** Two zeros is "c". 25 zeros is "z", 26 zeros is "ab", 27 is "bb", + ** and so forth. + */ + int k; + for(k=1; i+k0 ){ + aOut[j++] = 'a'+(k%26); + k /= 26; + } + } + } + aOut[j] = 0; + return j; + } + + /* Convert hex to binary */ + static char kvvfsHexToBinary(char c){ + if( c>='0' && c<='9' ) return c - '0'; + if( c>='A' && c<='F' ) return c - 'A' + 10; + return 0; + } + + /* + ** Decode the text encoding back to binary. The binary content is + ** written into pOut, which must be at least nOut bytes in length. + ** + ** The return value is the number of bytes actually written into aOut[]. + */ + static int kvvfsDecode(const char *aIn, char *aOut, int nOut){ + int i, j; + int c; + i = 0; + j = 0; + while( (c = aIn[i])!=0 ){ + if( c>='a' ){ + int n = 0; + int mult = 1; + while( c>='a' && c<='z' ){ + n += (c - 'a')*mult; + mult *= 26; + c = aIn[++i]; + } + if( j+n>nOut ) return -1; + while( n-->0 ){ + aOut[j++] = 0; + } + }else{ + if( j>nOut ) return -1; + aOut[j] = kvvfsHexToBinary(aIn[i++])<<4; + aOut[j++] += kvvfsHexToBinary(aIn[i++]); + } + } + return j; + } + + /* + ** Decode a complete journal file. Allocate space in pFile->aJrnl + ** and store the decoding there. Or leave pFile->aJrnl set to NULL + ** if an error is encountered. + ** + ** The first few characters of the text encoding will be a little-endian + ** base-26 number (digits a..z) that is the total number of bytes + ** in the decoded journal file image. This base-26 number is followed + ** by a single space, then the encoding of the journal. The space + ** separator is required to act as a terminator for the base-26 number. + */ + static void kvvfsDecodeJournal( + KVVfsFile *pFile, /* Store decoding in pFile->aJrnl */ + const char *zTxt, /* Text encoding. Zero-terminated */ + int nTxt /* Bytes in zTxt, excluding zero terminator */ + ){ - unsigned int n = 0; ++ unsigned int n; + int c, i, mult; + i = 0; + mult = 1; + while( (c = zTxt[i++])>='a' && c<='z' ){ + n += (zTxt[i] - 'a')*mult; + mult *= 26; + } + sqlite3_free(pFile->aJrnl); + pFile->aJrnl = sqlite3_malloc64( n ); + if( pFile->aJrnl==0 ){ + pFile->nJrnl = 0; + return; + } + pFile->nJrnl = n; + n = kvvfsDecode(zTxt+i, pFile->aJrnl, pFile->nJrnl); + if( nnJrnl ){ + sqlite3_free(pFile->aJrnl); + pFile->aJrnl = 0; + pFile->nJrnl = 0; + } + } + + /* + ** Read or write the "sz" element, containing the database file size. + */ + static sqlite3_int64 kvvfsReadFileSize(KVVfsFile *pFile){ + char zData[50]; + zData[0] = 0; + kvstorageRead(pFile->zClass, "sz", zData, sizeof(zData)-1); + return strtoll(zData, 0, 0); + } + static void kvvfsWriteFileSize(KVVfsFile *pFile, sqlite3_int64 sz){ + char zData[50]; + sqlite3_snprintf(sizeof(zData), zData, "%lld", sz); + kvstorageWrite(pFile->zClass, "sz", zData); + } + + /****** sqlite3_io_methods methods ******************************************/ + + /* + ** Close an kvvfs-file. + */ + static int kvvfsClose(sqlite3_file *pProtoFile){ + KVVfsFile *pFile = (KVVfsFile *)pProtoFile; + + SQLITE_KV_LOG(("xClose %s %s\n", pFile->zClass, + pFile->isJournal ? "journal" : "db")); + sqlite3_free(pFile->aJrnl); + return SQLITE_OK; + } + + /* + ** Read from the -journal file. + */ + static int kvvfsReadJrnl( + sqlite3_file *pProtoFile, + void *zBuf, + int iAmt, + sqlite_int64 iOfst + ){ + KVVfsFile *pFile = (KVVfsFile*)pProtoFile; + assert( pFile->isJournal ); + SQLITE_KV_LOG(("xRead('%s-journal',%d,%lld)\n", pFile->zClass, iAmt, iOfst)); + if( pFile->aJrnl==0 ){ + int szTxt = kvstorageRead(pFile->zClass, "jrnl", 0, 0); + char *aTxt; + if( szTxt<=4 ){ + return SQLITE_IOERR; + } + aTxt = sqlite3_malloc64( szTxt+1 ); + if( aTxt==0 ) return SQLITE_NOMEM; + kvstorageRead(pFile->zClass, "jrnl", aTxt, szTxt+1); + kvvfsDecodeJournal(pFile, aTxt, szTxt); + sqlite3_free(aTxt); + if( pFile->aJrnl==0 ) return SQLITE_IOERR; + } + if( iOfst+iAmt>pFile->nJrnl ){ + return SQLITE_IOERR_SHORT_READ; + } + memcpy(zBuf, pFile->aJrnl+iOfst, iAmt); + return SQLITE_OK; + } + + /* + ** Read from the database file. + */ + static int kvvfsReadDb( + sqlite3_file *pProtoFile, + void *zBuf, + int iAmt, + sqlite_int64 iOfst + ){ + KVVfsFile *pFile = (KVVfsFile*)pProtoFile; + unsigned int pgno; + int got, n; + char zKey[30]; + char aData[131073]; + assert( iOfst>=0 ); + assert( iAmt>=0 ); + SQLITE_KV_LOG(("xRead('%s-db',%d,%lld)\n", pFile->zClass, iAmt, iOfst)); + if( iOfst+iAmt>=512 ){ + if( (iOfst % iAmt)!=0 ){ + return SQLITE_IOERR_READ; + } + if( (iAmt & (iAmt-1))!=0 || iAmt<512 || iAmt>65536 ){ + return SQLITE_IOERR_READ; + } + pFile->szPage = iAmt; + pgno = 1 + iOfst/iAmt; + }else{ + pgno = 1; + } + sqlite3_snprintf(sizeof(zKey), zKey, "%u", pgno); + got = kvstorageRead(pFile->zClass, zKey, aData, sizeof(aData)-1); + if( got<0 ){ + n = 0; + }else{ + aData[got] = 0; + if( iOfst+iAmt<512 ){ + n = kvvfsDecode(aData, &aData[1000], 1000); + if( n>=iOfst+iAmt ){ + memcpy(zBuf, &aData[1000+iOfst], iAmt); + n = iAmt; + }else{ + n = 0; + } + }else{ + n = kvvfsDecode(aData, zBuf, iAmt); + } + } + if( nzClass, iAmt, iOfst)); + if( iEnd>=0x10000000 ) return SQLITE_FULL; + if( pFile->aJrnl==0 || pFile->nJrnlaJrnl, iEnd); + if( aNew==0 ){ + return SQLITE_IOERR_NOMEM; + } + pFile->aJrnl = aNew; + if( pFile->nJrnlaJrnl+pFile->nJrnl, 0, iOfst-pFile->nJrnl); + } + pFile->nJrnl = iEnd; + } + memcpy(pFile->aJrnl+iOfst, zBuf, iAmt); + return SQLITE_OK; + } + + /* + ** Write into the database file. + */ + static int kvvfsWriteDb( + sqlite3_file *pProtoFile, + const void *zBuf, + int iAmt, + sqlite_int64 iOfst + ){ + KVVfsFile *pFile = (KVVfsFile*)pProtoFile; + unsigned int pgno; + char zKey[30]; + char aData[131073]; + SQLITE_KV_LOG(("xWrite('%s-db',%d,%lld)\n", pFile->zClass, iAmt, iOfst)); + assert( iAmt>=512 && iAmt<=65536 ); + assert( (iAmt & (iAmt-1))==0 ); + pgno = 1 + iOfst/iAmt; + sqlite3_snprintf(sizeof(zKey), zKey, "%u", pgno); + kvvfsEncode(zBuf, iAmt, aData); + kvstorageWrite(pFile->zClass, zKey, aData); + if( iOfst+iAmt > pFile->szDb ){ + pFile->szDb = iOfst + iAmt; + } + return SQLITE_OK; + } + + /* + ** Truncate an kvvfs-file. + */ + static int kvvfsTruncateJrnl(sqlite3_file *pProtoFile, sqlite_int64 size){ + KVVfsFile *pFile = (KVVfsFile *)pProtoFile; + SQLITE_KV_LOG(("xTruncate('%s-journal',%lld)\n", pFile->zClass, size)); + assert( size==0 ); + kvstorageDelete(pFile->zClass, "jrnl"); + sqlite3_free(pFile->aJrnl); + pFile->aJrnl = 0; + pFile->nJrnl = 0; + return SQLITE_OK; + } + static int kvvfsTruncateDb(sqlite3_file *pProtoFile, sqlite_int64 size){ + KVVfsFile *pFile = (KVVfsFile *)pProtoFile; + if( pFile->szDb>size + && pFile->szPage>0 + && (size % pFile->szPage)==0 + ){ + char zKey[50]; + unsigned int pgno, pgnoMax; + SQLITE_KV_LOG(("xTruncate('%s-db',%lld)\n", pFile->zClass, size)); + pgno = 1 + size/pFile->szPage; + pgnoMax = 2 + pFile->szDb/pFile->szPage; + while( pgno<=pgnoMax ){ + sqlite3_snprintf(sizeof(zKey), zKey, "%u", pgno); + kvstorageDelete(pFile->zClass, zKey); + pgno++; + } + pFile->szDb = size; + kvvfsWriteFileSize(pFile, size); + return SQLITE_OK; + } + return SQLITE_IOERR; + } + + /* + ** Sync an kvvfs-file. + */ + static int kvvfsSyncJrnl(sqlite3_file *pProtoFile, int flags){ + int i, n; + KVVfsFile *pFile = (KVVfsFile *)pProtoFile; + char *zOut; + SQLITE_KV_LOG(("xSync('%s-journal')\n", pFile->zClass)); + if( pFile->nJrnl<=0 ){ + return kvvfsTruncateJrnl(pProtoFile, 0); + } + zOut = sqlite3_malloc64( pFile->nJrnl*2 + 50 ); + if( zOut==0 ){ + return SQLITE_IOERR_NOMEM; + } + n = pFile->nJrnl; + i = 0; + do{ + zOut[i++] = 'a' + (n%26); + n /= 26; + }while( n>0 ); + zOut[i++] = ' '; + kvvfsEncode(pFile->aJrnl, pFile->nJrnl, &zOut[i]); + kvstorageWrite(pFile->zClass, "jrnl", zOut); + sqlite3_free(zOut); + return SQLITE_OK; + } + static int kvvfsSyncDb(sqlite3_file *pProtoFile, int flags){ + KVVfsFile *pFile = (KVVfsFile *)pProtoFile; + SQLITE_KV_LOG(("xSync('%s-db')\n", pFile->zClass)); + if( pFile->szDb>0 ){ + kvvfsWriteFileSize(pFile, pFile->szDb); + } + return SQLITE_OK; + } + + /* + ** Return the current file-size of an kvvfs-file. + */ + static int kvvfsFileSizeJrnl(sqlite3_file *pProtoFile, sqlite_int64 *pSize){ + KVVfsFile *pFile = (KVVfsFile *)pProtoFile; + SQLITE_KV_LOG(("xFileSize('%s-journal')\n", pFile->zClass)); + *pSize = pFile->nJrnl; + return SQLITE_OK; + } + static int kvvfsFileSizeDb(sqlite3_file *pProtoFile, sqlite_int64 *pSize){ + KVVfsFile *pFile = (KVVfsFile *)pProtoFile; + SQLITE_KV_LOG(("xFileSize('%s-db')\n", pFile->zClass)); + if( pFile->szDb>=0 ){ + *pSize = pFile->szDb; + }else{ + *pSize = kvvfsReadFileSize(pFile); + } + return SQLITE_OK; + } + + /* + ** Lock an kvvfs-file. + */ + static int kvvfsLock(sqlite3_file *pProtoFile, int eLock){ + KVVfsFile *pFile = (KVVfsFile *)pProtoFile; + assert( !pFile->isJournal ); + SQLITE_KV_LOG(("xLock(%s,%d)\n", pFile->zClass, eLock)); + + if( eLock!=SQLITE_LOCK_NONE ){ + pFile->szDb = kvvfsReadFileSize(pFile); + } + return SQLITE_OK; + } + + /* + ** Unlock an kvvfs-file. + */ + static int kvvfsUnlock(sqlite3_file *pProtoFile, int eLock){ + KVVfsFile *pFile = (KVVfsFile *)pProtoFile; + assert( !pFile->isJournal ); + SQLITE_KV_LOG(("xUnlock(%s,%d)\n", pFile->zClass, eLock)); + if( eLock==SQLITE_LOCK_NONE ){ + pFile->szDb = -1; + } + return SQLITE_OK; + } + + /* + ** Check if another file-handle holds a RESERVED lock on an kvvfs-file. + */ + static int kvvfsCheckReservedLock(sqlite3_file *pProtoFile, int *pResOut){ + SQLITE_KV_LOG(("xCheckReservedLock\n")); + *pResOut = 0; + return SQLITE_OK; + } + + /* + ** File control method. For custom operations on an kvvfs-file. + */ + static int kvvfsFileControl(sqlite3_file *pProtoFile, int op, void *pArg){ + return SQLITE_NOTFOUND; + } + + /* + ** Return the sector-size in bytes for an kvvfs-file. + */ + static int kvvfsSectorSize(sqlite3_file *pFile){ + return 512; + } + + /* + ** Return the device characteristic flags supported by an kvvfs-file. + */ + static int kvvfsDeviceCharacteristics(sqlite3_file *pProtoFile){ + return 0; + } + + /****** sqlite3_vfs methods *************************************************/ + + /* + ** Open an kvvfs file handle. + */ + static int kvvfsOpen( + sqlite3_vfs *pProtoVfs, + const char *zName, + sqlite3_file *pProtoFile, + int flags, + int *pOutFlags + ){ + KVVfsFile *pFile = (KVVfsFile*)pProtoFile; + SQLITE_KV_LOG(("xOpen(\"%s\")\n", zName)); + if( strcmp(zName, "local")==0 + || strcmp(zName, "session")==0 + ){ + pFile->isJournal = 0; + pFile->base.pMethods = &kvvfs_db_io_methods; + }else + if( strcmp(zName, "local-journal")==0 + || strcmp(zName, "session-journal")==0 + ){ + pFile->isJournal = 1; + pFile->base.pMethods = &kvvfs_jrnl_io_methods; + }else{ + return SQLITE_CANTOPEN; + } + if( zName[0]=='s' ){ + pFile->zClass = "session"; + }else{ + pFile->zClass = "local"; + } + pFile->aJrnl = 0; + pFile->nJrnl = 0; + pFile->szPage = -1; + pFile->szDb = -1; + return SQLITE_OK; + } + + /* + ** Delete the file located at zPath. If the dirSync argument is true, + ** ensure the file-system modifications are synced to disk before + ** returning. + */ + static int kvvfsDelete(sqlite3_vfs *pVfs, const char *zPath, int dirSync){ + if( strcmp(zPath, "local-journal")==0 ){ + kvstorageDelete("local", "jrnl"); + }else + if( strcmp(zPath, "session-journal")==0 ){ + kvstorageDelete("session", "jrnl"); + } + return SQLITE_OK; + } + + /* + ** Test for access permissions. Return true if the requested permission + ** is available, or false otherwise. + */ + static int kvvfsAccess( + sqlite3_vfs *pProtoVfs, + const char *zPath, + int flags, + int *pResOut + ){ + SQLITE_KV_LOG(("xAccess(\"%s\")\n", zPath)); + if( strcmp(zPath, "local-journal")==0 ){ + *pResOut = kvstorageRead("local", "jrnl", 0, 0)>0; + }else + if( strcmp(zPath, "session-journal")==0 ){ + *pResOut = kvstorageRead("session", "jrnl", 0, 0)>0; + }else + if( strcmp(zPath, "local")==0 ){ + *pResOut = kvstorageRead("local", "sz", 0, 0)>0; + }else + if( strcmp(zPath, "session")==0 ){ + *pResOut = kvstorageRead("session", "sz", 0, 0)>0; + }else + { + *pResOut = 0; + } + SQLITE_KV_LOG(("xAccess returns %d\n",*pResOut)); + return SQLITE_OK; + } + + /* + ** Populate buffer zOut with the full canonical pathname corresponding + ** to the pathname in zPath. zOut is guaranteed to point to a buffer + ** of at least (INST_MAX_PATHNAME+1) bytes. + */ + static int kvvfsFullPathname( + sqlite3_vfs *pVfs, + const char *zPath, + int nOut, + char *zOut + ){ + size_t nPath = strlen(zPath); + SQLITE_KV_LOG(("xFullPathname(\"%s\")\n", zPath)); + if( nOut