From 24b13fb352594c9d259d75bb6822ef505bc9f30b Mon Sep 17 00:00:00 2001 From: stephan Date: Sun, 9 Oct 2022 13:33:34 +0000 Subject: [PATCH] Cherrypick [ea370b9b05f7ed7eaa] and part of [92b500da] into kv-vfs branch. FossilOrigin-Name: 812638039e1b8621a00dcf583ae3fb8a751f9aedbb18223b1fbffaee38cc0e15 --- Makefile.in | 3 +- manifest | 18 ++-- manifest.uuid | 2 +- src/os_kv.c | 273 +++++++++++--------------------------------------- 4 files changed, 70 insertions(+), 226 deletions(-) diff --git a/Makefile.in b/Makefile.in index d4d42c6b18..231c379a18 100644 --- a/Makefile.in +++ b/Makefile.in @@ -635,6 +635,7 @@ FUZZCHECK_OPT += -DSQLITE_ENABLE_DBSTAT_VTAB FUZZCHECK_OPT += -DSQLITE_ENABLE_BYTECODE_VTAB FUZZCHECK_SRC = $(TOP)/test/fuzzcheck.c $(TOP)/test/ossfuzz.c $(TOP)/test/fuzzinvariants.c DBFUZZ_OPT = +ST_OPT = -DSQLITE_OS_KV_OPTIONAL # This is the default Makefile target. The objects listed here # are what get build when you type just "make" with no arguments. @@ -1392,7 +1393,7 @@ LogEst$(TEXE): $(TOP)/tool/logest.c sqlite3.h wordcount$(TEXE): $(TOP)/test/wordcount.c sqlite3.lo $(LTLINK) -o $@ $(TOP)/test/wordcount.c sqlite3.lo $(TLIBS) -speedtest1$(TEXE): $(TOP)/test/speedtest1.c sqlite3.c +speedtest1$(TEXE): $(TOP)/test/speedtest1.c sqlite3.c Makefile $(LTLINK) $(ST_OPT) -o $@ $(TOP)/test/speedtest1.c sqlite3.c $(TLIBS) startup$(TEXE): $(TOP)/test/startup.c sqlite3.c diff --git a/manifest b/manifest index 75b349f4cb..6a691be3e3 100644 --- a/manifest +++ b/manifest @@ -1,9 +1,9 @@ -C In\sos_kv.c,\severy\sxWrite\smethod\scall\son\sa\sdatabase\sshould\srecord\sthe\spage\ssize. -D 2022-10-03T18:23:54.243 +C Cherrypick\s[ea370b9b05f7ed7eaa]\sand\spart\sof\s[92b500da]\sinto\skv-vfs\sbranch. +D 2022-10-09T13:33:34.070 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 -F Makefile.in ee179f405fd5f8845473f888517c4ada46099306c33ae1f27dd1aef53fe8e867 +F Makefile.in c80d05a84c16fce844db1dafdfcd2cdd116177027202e09c4a54681ecd72c6ec F Makefile.linux-gcc f609543700659711fbd230eced1f01353117621dccae7b9fb70daa64236c5241 F Makefile.msc d547a2fdba38a1c6cd1954977d0b0cc017f5f8fbfbc65287bf8d335808938016 F README.md 8b8df9ca852aeac4864eb1e400002633ee6db84065bd01b78c33817f97d31f5e @@ -574,7 +574,7 @@ F src/notify.c 89a97dc854c3aa62ad5f384ef50c5a4a11d70fcc69f86de3e991573421130ed6 F src/os.c 0eb831ba3575af5277e47f4edd14fdfc90025c67eb25ce5cda634518d308d4e9 F src/os.h 1ff5ae51d339d0e30d8a9d814f4b8f8e448169304d83a7ed9db66a65732f3e63 F src/os_common.h b2f4707a603e36811d9b1a13278bffd757857b85 -F src/os_kv.c f9e56534d8825b6a7c122c9cf6c52c27f920cc6d0c5ce8cf7eeb1c1cf85a2560 +F src/os_kv.c fe6d52ab2bcb00b855a4237f0320b6f1789bfb1ac805bd5b52559451e8017041 F src/os_setup.h 0711dbc4678f3ac52d7fe736951b6384a0615387c4ba5135a4764e4e31f4b6a6 F src/os_unix.c 287aa5f5691a2b356780c63e83abaa33549add84227b8313395f04088486d79c F src/os_win.c 8d129ae3e59e0fa900e20d0ad789e96f2e08177f0b00b53cdda65c40331e0902 @@ -2004,8 +2004,10 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 3d350aa0c04d68ddbfa3ee4936bd7f700e400ce9e83361e908b1442feff4cbff -R 1f3ba01fb41faf792f0840d36953234e -U drh -Z fcee7b37cc91592220fd4b4fd4c975a1 +P e9411c74fcce5e109fd02e7db55c0bf113ae8e02f46dfccc0b950dd06e5bd39b +Q +92b500da70a3dc64e910c232a2cac7620b6609162c2a5058b269d3b83d763c02 +Q +ea370b9b05f7ed7eaa154ba58019f6642217eabc18517e721567adf948d93980 +R cb72d36beff7f7f7a1bade2b759b933e +U stephan +Z 6daf03a792b7f21d2196b1f5d1530641 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index b18d9ac5b6..ded9e9bb85 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e9411c74fcce5e109fd02e7db55c0bf113ae8e02f46dfccc0b950dd06e5bd39b \ No newline at end of file +812638039e1b8621a00dcf583ae3fb8a751f9aedbb18223b1fbffaee38cc0e15 \ No newline at end of file diff --git a/src/os_kv.c b/src/os_kv.c index e5b40f9782..f9671bbb71 100644 --- a/src/os_kv.c +++ b/src/os_kv.c @@ -23,14 +23,14 @@ /* SQLITE_KV_TRACE() is used for tracing calls to kvstorage routines. */ #if 0 -#define SQLITE_KV_TRACE(X) printf X; +#define SQLITE_KV_TRACE(X) printf X #else #define SQLITE_KV_TRACE(X) #endif /* SQLITE_KV_LOG() is used for tracing calls to the VFS interface */ #if 0 -#define SQLITE_KV_LOG(X) printf X; +#define SQLITE_KV_LOG(X) printf X #else #define SQLITE_KV_LOG(X) #endif @@ -166,6 +166,9 @@ static sqlite3_io_methods kvvfs_jrnl_io_methods = { /* Forward declarations for the low-level storage engine */ +static int kvstorageWrite(const char*, const char *zKey, const char *zData); +static int kvstorageDelete(const char*, const char *zKey); +static int kvstorageRead(const char*, const char *zKey, char *zBuf, int nBuf); #define KVSTORAGE_KEY_SZ 32 /* Expand the key name with an appropriate prefix and put the result @@ -180,206 +183,6 @@ static void kvstorageMakeKey( sqlite3_snprintf(KVSTORAGE_KEY_SZ, zKeyOut, "kvvfs-%s-%s", zClass, zKeyIn); } -#ifdef __EMSCRIPTEN__ -/* Provide Emscripten-based impls of kvstorageWrite/Read/Delete()... */ -#include -#include - -/* -** WASM_KEEP is identical to EMSCRIPTEN_KEEPALIVE but is not -** Emscripten-specific. It explicitly includes marked functions for -** export into the target wasm file without requiring explicit listing -** of those functions in Emscripten's -sEXPORTED_FUNCTIONS=... list -** (or equivalent in other build platforms). Any function with neither -** this attribute nor which is listed as an explicit export will not -** be exported from the wasm file (but may still be used internally -** within the wasm file). -** -** The functions in this file (sqlite3-wasm.c) which require exporting -** are marked with this flag. They may also be added to any explicit -** build-time export list but need not be. All of these APIs are -** intended for use only within the project's own JS/WASM code, and -** not by client code, so an argument can be made for reducing their -** visibility by not including them in any build-time export lists. -** -** 2022-09-11: it's not yet _proven_ that this approach works in -** non-Emscripten builds. If not, such builds will need to export -** those using the --export=... wasm-ld flag (or equivalent). As of -** this writing we are tied to Emscripten for various reasons -** and cannot test the library with other build environments. -*/ -#define WASM_KEEP __attribute__((used,visibility("default"))) -/* -** An internal level of indirection for accessing the static -** kvstorageMakeKey() from EM_JS()-generated functions. This must be -** made available for export via Emscripten but is not intended to be -** used from client code. If called with a NULL zKeyOut it is a no-op. -** It returns KVSTORAGE_KEY_SZ, so JS code (which cannot see that -** constant) may call it with NULL arguments to get the size of the -** allocation they'll need for a kvvfs key. -** -** Maintenance reminder: Emscripten will install this in the Module -** init scope and will prefix its name with "_". -*/ -WASM_KEEP -int sqlite3_wasm__kvvfsMakeKey(const char *zClass, const char *zKeyIn, - char *zKeyOut){ - if( 0!=zKeyOut ) kvstorageMakeKey(zClass, zKeyIn, zKeyOut); - return KVSTORAGE_KEY_SZ; -} -/* -** Internal helper for kvstorageWrite/Read/Delete() which creates a -** storage key for the given zClass/zKeyIn combination. Returns a -** pointer to the key: a C string allocated on the WASM stack, or 0 if -** allocation fails. It is up to the caller to save/restore the stack -** before/after this operation. -*/ -EM_JS(const char *, kvstorageMakeKeyOnJSStack, - (const char *zClass, const char *zKeyIn),{ - if( 0==zClass || 0==zKeyIn) return 0; - const zXKey = stackAlloc(_sqlite3_wasm__kvvfsMakeKey(0,0,0)); - if(zXKey) _sqlite3_wasm__kvvfsMakeKey(zClass, zKeyIn, zXKey); - return zXKey; -}); - -/* -** JS impl of kvstorageWrite(). Main docs are in the C impl. This impl -** writes zData to the global sessionStorage (if zClass starts with -** 's') or localStorage, using a storage key derived from zClass and -** zKey. -*/ -EM_JS(int, kvstorageWrite, - (const char *zClass, const char *zKey, const char *zData),{ - const stack = stackSave(); - try { - const zXKey = kvstorageMakeKeyOnJSStack(zClass,zKey); - if(!zXKey) return 1/*OOM*/; - const jKey = UTF8ToString(zXKey); - /** - We could simplify this function and eliminate the - kvstorageMakeKey() symbol acrobatics if we'd simply hard-code - the key algo into the 3 functions which need it: - - const jKey = "kvvfs-"+UTF8ToString(zClass)+"-"+UTF8ToString(zKey); - */ - ((115/*=='s'*/===getValue(zClass)) - ? sessionStorage : localStorage).setItem(jKey, UTF8ToString(zData)); - }catch(e){ - console.error("kvstorageWrite()",e); - return 1; // Can't access SQLITE_xxx from here - }finally{ - stackRestore(stack); - } - return 0; -}); - -/* -** JS impl of kvstorageDelete(). Main docs are in the C impl. This -** impl generates a key derived from zClass and zKey, and removes the -** matching entry (if any) from global sessionStorage (if zClass -** starts with 's') or localStorage. -*/ -EM_JS(int, kvstorageDelete, - (const char *zClass, const char *zKey),{ - const stack = stackSave(); - try { - const zXKey = kvstorageMakeKeyOnJSStack(zClass,zKey); - if(!zXKey) return 1/*OOM*/; - const jKey = UTF8ToString(zXKey); - ((115/*=='s'*/===getValue(zClass)) - ? sessionStorage : localStorage).removeItem(jKey); - }catch(e){ - console.error("kvstorageDelete()",e); - return 1; - }finally{ - stackRestore(stack); - } - return 0; -}); - -/* -** JS impl of kvstorageRead(). Main docs are in the C impl. This impl -** reads its data from the global sessionStorage (if zClass starts -** with 's') or localStorage, using a storage key derived from zClass -** and zKey. -*/ -EM_JS(int, kvstorageRead, - (const char *zClass, const char *zKey, char *zBuf, int nBuf),{ - const stack = stackSave(); - try { - const zXKey = kvstorageMakeKeyOnJSStack(zClass,zKey); - if(!zXKey) return -3/*OOM*/; - const jKey = UTF8ToString(zXKey); - const jV = ((115/*=='s'*/===getValue(zClass)) - ? sessionStorage : localStorage).getItem(jKey); - if(!jV) return -1; - const nV = jV.length /* Note that we are relying 100% on v being - ASCII so that jV.length is equal to the - C-string's byte length. */; - if(nBuf<=0) return nV; - else if(1===nBuf){ - setValue(zBuf, 0); - return nV; - } - const zV = allocateUTF8OnStack(jV); - if(nBuf > nV + 1) nBuf = nV + 1; - HEAPU8.copyWithin(zBuf, zV, zV + nBuf - 1); - setValue( zBuf + nBuf - 1, 0 ); - return nBuf - 1; - }catch(e){ - console.error("kvstorageRead()",e); - return -2; - }finally{ - stackRestore(stack); - } -}); - -/* -** This function exists for (1) WASM testing purposes and (2) as a -** hook to get Emscripten to export several EM_JS()-generated -** functions (if we don't reference them from exported C functions -** then they get stripped away at build time). It is not part of the -** public API and its signature and semantics may change at any time. -** It's not even part of the private API, for that matter - it's part -** of the Emscripten C/JS/WASM glue. -*/ -WASM_KEEP -int sqlite3__wasm_emjs_kvvfs(int whichOp){ - int rc = 0; - const char * zClass = - "sezzion" /*don't collide with "session" records!*/; - const char * zKey = "hello"; - switch( whichOp ){ - case 0: break; - case 1: - rc = kvstorageWrite(zClass, zKey, "world"); - break; - case 2: { - char buffer[128] = {0}; - char * zBuf = &buffer[0]; - rc = kvstorageRead(zClass, zKey, zBuf, (int)sizeof(buffer)); - emscripten_console_logf("kvstorageRead()=%d %s\n", rc, zBuf); - break; - } - case 3: - kvstorageDelete(zClass, zKey); - break; - case 4: - kvstorageMakeKeyOnJSStack(0,0); - break; - default: break; - } - return rc; -} - -#undef WASM_KEEP -#else /* end ifdef __EMSCRIPTEN__ */ -/* Forward declarations for the low-level storage engine -*/ -static int kvstorageWrite(const char*, const char *zKey, const char *zData); -static int kvstorageDelete(const char*, const char *zKey); -static int kvstorageRead(const char*, const char *zKey, char *zBuf, int nBuf); - /* Write content into a key. zClass is the particular namespace of the ** underlying key/value store to use - either "local" or "session". ** @@ -474,7 +277,45 @@ static int kvstorageRead( return (int)n; } } -#endif /* ifdef __EMSCRIPTEN__ */ + +/* +** An internal level of indirection which enables us to replace the +** kvvfs i/o methods with JavaScript implementations in WASM builds. +** Maintenance reminder: if this struct changes in any way, the JSON +** rendering of its structure must be updated in +** sqlite3_wasm_enum_json(). There are no binary compatibility +** concerns, so it does not need an iVersion member. This file is +** necessarily always compiled together with sqlite3_wasm_enum_json(), +** and JS code dynamically creates the mapping of members based on +** that JSON description. +*/ +typedef struct sqlite3_kvvfs_methods sqlite3_kvvfs_methods; +struct sqlite3_kvvfs_methods { + int (*xRead)(const char *zClass, const char *zKey, char *zBuf, int nBuf); + int (*xWrite)(const char *zClass, const char *zKey, const char *zData); + int (*xDelete)(const char *zClass, const char *zKey); + const int nKeySize; +}; + +/* +** This object holds the kvvfs I/O methods which may be swapped out +** for JavaScript-side implementations in WASM builds. In such builds +** it cannot be const, but in native builds it should be so that +** the compiler can hopefully optimize this level of indirection out. +** That said, kvvfs is intended primarily for use in WASM builds. +** +** Note that this is not explicitly flagged as static because the +** amalgamation build will tag it with SQLITE_PRIVATE. +*/ +#ifndef SQLITE_WASM +const +#endif +sqlite3_kvvfs_methods sqlite3KvvfsMethods = { +kvstorageRead, +kvstorageWrite, +kvstorageDelete, +KVSTORAGE_KEY_SZ +}; /****** Utility subroutines ************************************************/ @@ -632,13 +473,13 @@ static void kvvfsDecodeJournal( static sqlite3_int64 kvvfsReadFileSize(KVVfsFile *pFile){ char zData[50]; zData[0] = 0; - kvstorageRead(pFile->zClass, "sz", zData, sizeof(zData)-1); + sqlite3KvvfsMethods.xRead(pFile->zClass, "sz", zData, sizeof(zData)-1); return strtoll(zData, 0, 0); } static int kvvfsWriteFileSize(KVVfsFile *pFile, sqlite3_int64 sz){ char zData[50]; sqlite3_snprintf(sizeof(zData), zData, "%lld", sz); - return kvstorageWrite(pFile->zClass, "sz", zData); + return sqlite3KvvfsMethods.xWrite(pFile->zClass, "sz", zData); } /****** sqlite3_io_methods methods ******************************************/ @@ -717,7 +558,7 @@ static int kvvfsReadDb( pgno = 1; } sqlite3_snprintf(sizeof(zKey), zKey, "%u", pgno); - got = kvstorageRead(pFile->zClass, zKey, aData, sizeof(aData)-1); + got = sqlite3KvvfsMethods.xRead(pFile->zClass, zKey, aData, sizeof(aData)-1); if( got<0 ){ n = 0; }else{ @@ -791,7 +632,7 @@ static int kvvfsWriteDb( pgno = 1 + iOfst/iAmt; sqlite3_snprintf(sizeof(zKey), zKey, "%u", pgno); kvvfsEncode(zBuf, iAmt, aData); - if( kvstorageWrite(pFile->zClass, zKey, aData) ){ + if( sqlite3KvvfsMethods.xWrite(pFile->zClass, zKey, aData) ){ return SQLITE_IOERR; } if( iOfst+iAmt > pFile->szDb ){ @@ -810,7 +651,7 @@ static int kvvfsTruncateJrnl(sqlite3_file *pProtoFile, sqlite_int64 size){ KVVfsFile *pFile = (KVVfsFile *)pProtoFile; SQLITE_KV_LOG(("xTruncate('%s-journal',%lld)\n", pFile->zClass, size)); assert( size==0 ); - kvstorageDelete(pFile->zClass, "jrnl"); + sqlite3KvvfsMethods.xDelete(pFile->zClass, "jrnl"); sqlite3_free(pFile->aJrnl); pFile->aJrnl = 0; pFile->nJrnl = 0; @@ -829,7 +670,7 @@ static int kvvfsTruncateDb(sqlite3_file *pProtoFile, sqlite_int64 size){ pgnoMax = 2 + pFile->szDb/pFile->szPage; while( pgno<=pgnoMax ){ sqlite3_snprintf(sizeof(zKey), zKey, "%u", pgno); - kvstorageDelete(pFile->zClass, zKey); + sqlite3KvvfsMethods.xDelete(pFile->zClass, zKey); pgno++; } pFile->szDb = size; @@ -861,7 +702,7 @@ static int kvvfsSyncJrnl(sqlite3_file *pProtoFile, int flags){ }while( n>0 ); zOut[i++] = ' '; kvvfsEncode(pFile->aJrnl, pFile->nJrnl, &zOut[i]); - i = kvstorageWrite(pFile->zClass, "jrnl", zOut); + i = sqlite3KvvfsMethods.xWrite(pFile->zClass, "jrnl", zOut); sqlite3_free(zOut); return i ? SQLITE_IOERR : SQLITE_OK; } @@ -1007,10 +848,10 @@ static int kvvfsOpen( */ static int kvvfsDelete(sqlite3_vfs *pVfs, const char *zPath, int dirSync){ if( strcmp(zPath, "local-journal")==0 ){ - kvstorageDelete("local", "jrnl"); + sqlite3KvvfsMethods.xDelete("local", "jrnl"); }else if( strcmp(zPath, "session-journal")==0 ){ - kvstorageDelete("session", "jrnl"); + sqlite3KvvfsMethods.xDelete("session", "jrnl"); } return SQLITE_OK; } @@ -1027,16 +868,16 @@ static int kvvfsAccess( ){ SQLITE_KV_LOG(("xAccess(\"%s\")\n", zPath)); if( strcmp(zPath, "local-journal")==0 ){ - *pResOut = kvstorageRead("local", "jrnl", 0, 0)>0; + *pResOut = sqlite3KvvfsMethods.xRead("local", "jrnl", 0, 0)>0; }else if( strcmp(zPath, "session-journal")==0 ){ - *pResOut = kvstorageRead("session", "jrnl", 0, 0)>0; + *pResOut = sqlite3KvvfsMethods.xRead("session", "jrnl", 0, 0)>0; }else if( strcmp(zPath, "local")==0 ){ - *pResOut = kvstorageRead("local", "sz", 0, 0)>0; + *pResOut = sqlite3KvvfsMethods.xRead("local", "sz", 0, 0)>0; }else if( strcmp(zPath, "session")==0 ){ - *pResOut = kvstorageRead("session", "sz", 0, 0)>0; + *pResOut = sqlite3KvvfsMethods.xRead("session", "sz", 0, 0)>0; }else { *pResOut = 0; -- 2.47.2