isSharedTypedArray,
toss: function(...args){throw new Error(args.join(' '))},
toss3,
- typedArrayPart
- };
+ typedArrayPart,
+ /**
+ Given a byte array or ArrayBuffer, this function throws if the
+ lead bytes of that buffer do not hold a SQLite3 database header,
+ else it returns without side effects.
+
+ Added in 3.44.
+ */
+ affirmDbHeader: function(bytes){
+ if(bytes instanceof ArrayBuffer) bytes = new Uint8Array(bytes);
+ const header = "SQLite format 3";
+ if( header.length > bytes.byteLength ){
+ toss3("Input does not contain an SQLite3 database header.");
+ }
+ for(let i = 0; i < header.length; ++i){
+ if( header.charCodeAt(i) !== bytes[i] ){
+ toss3("Input does not contain an SQLite3 database header.");
+ }
+ }
+ },
+ /**
+ Given a byte array or ArrayBuffer, this function throws if the
+ database does not, at a cursory glance, appear to be an SQLite3
+ database. It only examines the size and header, but further
+ checks may be added in the future.
+
+ Added in 3.44.
+ */
+ affirmIsDb: function(bytes){
+ if(bytes instanceof ArrayBuffer) bytes = new Uint8Array(bytes);
+ const n = bytes.byteLength;
+ if(n<512 || n%512!==0) {
+ toss3("Byte array size",n,"is invalid for an SQLite3 db.");
+ }
+ util.affirmDbHeader(bytes);
+ }
+ }/*util*/;
Object.assign(wasm, {
/**
const toss3 = sqlite3.util.toss3;
const initPromises = Object.create(null);
const capi = sqlite3.capi;
+ const util = sqlite3.util;
const wasm = sqlite3.wasm;
// Config opts for the VFS...
const SECTOR_SIZE = 4096;
return b;
}
+ //! Impl for importDb() when its 2nd arg is a function.
+ async importDbChunked(name, callback){
+ const sah = this.#mapFilenameToSAH.get(name)
+ || this.nextAvailableSAH()
+ || toss("No available handles to import to.");
+ sah.truncate(0);
+ let nWrote = 0, chunk, checkedHeader = false, err = false;
+ try{
+ while( undefined !== (chunk = await callback()) ){
+ if(chunk instanceof ArrayBuffer) chunk = new Uint8Array(chunk);
+ if( 0===nWrote && chunk.byteLength>=15 ){
+ util.affirmDbHeader(chunk);
+ checkedHeader = true;
+ }
+ sah.write(chunk, {at: HEADER_OFFSET_DATA + nWrote});
+ nWrote += chunk.byteLength;
+ }
+ if( nWrote < 512 || 0!==nWrote % 512 ){
+ toss("Input size",nWrote,"is not correct for an SQLite database.");
+ }
+ if( !checkedHeader ){
+ const header = new Uint8Array(20);
+ sah.read( header, {at: 0} );
+ util.affirmDbHeader( header );
+ }
+ sah.write(new Uint8Array(2), {
+ at: HEADER_OFFSET_DATA + 18
+ }/*force db out of WAL mode*/);
+ }catch(e){
+ this.setAssociatedPath(sah, '', 0);
+ }
+ this.setAssociatedPath(sah, name, capi.SQLITE_OPEN_MAIN_DB);
+ return nWrote;
+ }
+
//! Documented elsewhere in this file.
importDb(name, bytes){
- if(bytes instanceof ArrayBuffer) bytes = new Uint8Array(bytes);
+ if( bytes instanceof ArrayBuffer ) bytes = new Uint8Array(bytes);
+ else if( bytes instanceof Function ) return this.importDbChunked(name, bytes);
+ const sah = this.#mapFilenameToSAH.get(name)
+ || this.nextAvailableSAH()
+ || toss("No available handles to import to.");
const n = bytes.byteLength;
if(n<512 || n%512!=0){
toss("Byte array size is invalid for an SQLite db.");
toss("Input does not contain an SQLite database header.");
}
}
- const sah = this.#mapFilenameToSAH.get(name)
- || this.nextAvailableSAH()
- || toss("No available handles to import to.");
const nWrote = sah.write(bytes, {at: HEADER_OFFSET_DATA});
if(nWrote != n){
this.setAssociatedPath(sah, '', 0);
toss("Expected to write "+n+" bytes but wrote "+nWrote+".");
}else{
+ sah.write(new Uint8Array([0,0]), {at: HEADER_OFFSET_DATA+18}
+ /* force db out of WAL mode */);
this.setAssociatedPath(sah, name, capi.SQLITE_OPEN_MAIN_DB);
}
+ return nWrote;
}
}/*class OpfsSAHPool*/;
automatically clean up any non-database files so importing them
is pointless.
+ If passed a function for its second argument, its behavior
+ changes to asynchronous and it imports its data in chunks fed to
+ it by the given callback function. It calls the callback (which
+ may be async) repeatedly, expecting either a Uint8Array or
+ ArrayBuffer (to denote new input) or undefined (to denote
+ EOF). For so long as the callback continues to return
+ non-undefined, it will append incoming data to the given
+ VFS-hosted database file. The result of the resolved Promise when
+ called this way is the size of the resulting database.
+
+ On succes this routine rewrites the database header bytes in the
+ output file (not the input array) to force disabling of WAL mode.
+
On a write error, the handle is removed from the pool and made
available for re-use.
const error = (...args)=>logImpl(0, ...args);
const toss = sqlite3.util.toss;
const capi = sqlite3.capi;
+ const util = sqlite3.util;
const wasm = sqlite3.wasm;
const sqlite3_vfs = capi.sqlite3_vfs;
const sqlite3_file = capi.sqlite3_file;
doDir(opt.directory, 0);
};
+ /**
+ impl of importDb() when it's given a function as its second
+ argument.
+ */
+ const importDbChunked = async function(filename, callback){
+ const [hDir, fnamePart] = await opfsUtil.getDirForFilename(filename, true);
+ const hFile = await hDir.getFileHandle(fnamePart, {create:true});
+ const sah = await hFile.createSyncAccessHandle();
+ sah.truncate(0);
+ let nWrote = 0, chunk, checkedHeader = false, err = false;
+ try{
+ while( undefined !== (chunk = await callback()) ){
+ if(chunk instanceof ArrayBuffer) chunk = new Uint8Array(chunk);
+ if( 0===nWrote && chunk.byteLength>=15 ){
+ util.affirmDbHeader(chunk);
+ checkedHeader = true;
+ }
+ sah.write(chunk, {at: nWrote});
+ nWrote += chunk.byteLength;
+ }
+ if( nWrote < 512 || 0!==nWrote % 512 ){
+ toss("Input size",nWrote,"is not correct for an SQLite database.");
+ }
+ if( !checkedHeader ){
+ const header = new Uint8Array(20);
+ sah.read( header, {at: 0} );
+ util.affirmDbHeader( header );
+ }
+ sah.write(new Uint8Array(2), {at: 18}/*force db out of WAL mode*/);
+ return nWrote;
+ }catch(e){
+ await hDir.removeEntry( fnamePart ).catch(()=>{});
+ throw e;
+ }finally {
+ await sah.close();
+ }
+ };
+
/**
Asynchronously imports the given bytes (a byte array or
ArrayBuffer) into the given database file.
+ If passed a function for its second argument, its behaviour
+ changes to async and it imports its data in chunks fed to it by
+ the given callback function. It calls the callback (which may
+ be async) repeatedly, expecting either a Uint8Array or
+ ArrayBuffer (to denote new input) or undefined (to denote
+ EOF). For so long as the callback continues to return
+ non-undefined, it will append incoming data to the given
+ VFS-hosted database file. When called this way, the resolved
+ value of the returned Promise is the number of bytes written to
+ the target file.
+
It very specifically requires the input to be an SQLite3
database and throws if that's not the case. It does so in
order to prevent this function from taking on a larger scope
than it is specifically intended to. i.e. we do not want it to
become a convenience for importing arbitrary files into OPFS.
- Throws on error. Resolves to the number of bytes written.
+ This routine rewrites the database header bytes in the output
+ file (not the input array) to force disabling of WAL mode.
+
+ On error this throws and the state of the input file is
+ undefined (it depends on where the exception was triggered).
+
+ On success, resolves to the number of bytes written.
*/
opfsUtil.importDb = async function(filename, bytes){
+ if( bytes instanceof Function ){
+ return importDbChunked(filename, bytes);
+ }
if(bytes instanceof ArrayBuffer) bytes = new Uint8Array(bytes);
+ util.affirmIsDb(bytes);
const n = bytes.byteLength;
- if(n<512 || n%512!=0){
- toss("Byte array size is invalid for an SQLite db.");
- }
- const header = "SQLite format 3";
- for(let i = 0; i < header.length; ++i){
- if( header.charCodeAt(i) !== bytes[i] ){
- toss("Input does not contain an SQLite database header.");
- }
- }
const [hDir, fnamePart] = await opfsUtil.getDirForFilename(filename, true);
- const hFile = await hDir.getFileHandle(fnamePart, {create:true});
- const sah = await hFile.createSyncAccessHandle();
- sah.truncate(0);
- const nWrote = sah.write(bytes, {at: 0});
- sah.close();
- if(nWrote != n){
- toss("Expected to write "+n+" bytes but wrote "+nWrote+".");
+ let sah, err, nWrote = 0;
+ try {
+ const hFile = await hDir.getFileHandle(fnamePart, {create:true});
+ sah = await hFile.createSyncAccessHandle();
+ sah.truncate(0);
+ nWrote = sah.write(bytes, {at: 0});
+ if(nWrote != n){
+ toss("Expected to write "+n+" bytes but wrote "+nWrote+".");
+ }
+ sah.write(new Uint8Array(2), {at: 18}) /* force db out of WAL mode */;
+ return nWrote;
+ }catch(e){
+ await hDir.removeEntry( fnamePart ).catch(()=>{});
+ throw e;
+ }finally{
+ if( sah ) await sah.close();
}
- return nWrote;
};
if(sqlite3.oo1){
let db;
try {
const exp = this.opfsDbExport;
+ const filename = this.opfsDbFile;
delete this.opfsDbExport;
- this.opfsImportSize = await sqlite3.oo1.OpfsDb.importDb(this.opfsDbFile, exp);
+ this.opfsImportSize = await sqlite3.oo1.OpfsDb.importDb(filename, exp);
+ db = new sqlite3.oo1.OpfsDb(this.opfsDbFile);
+ T.assert(6 === db.selectValue('select count(*) from p')).
+ assert( this.opfsImportSize == exp.byteLength );
+ db.close();
+ this.opfsUnlink(filename);
+ T.assert(!(await sqlite3.opfs.entryExists(filename)));
+ // Try again with a function as an input source:
+ let cursor = 0;
+ const blockSize = 512, end = exp.byteLength;
+ const reader = async function(){
+ if(cursor >= exp.byteLength){
+ return undefined;
+ }
+ const rv = exp.subarray(cursor, cursor+blockSize>end ? end : cursor+blockSize);
+ cursor += blockSize;
+ return rv;
+ };
+ this.opfsImportSize = await sqlite3.oo1.OpfsDb.importDb(filename, reader);
db = new sqlite3.oo1.OpfsDb(this.opfsDbFile);
T.assert(6 === db.selectValue('select count(*) from p')).
assert( this.opfsImportSize == exp.byteLength );
const dbytes = u1.exportFile(dbName);
T.assert(dbytes.length >= 4096);
const dbName2 = '/exported.db';
- u1.importDb(dbName2, dbytes);
- T.assert( 2 == u1.getFileCount() );
+ let nWrote = u1.importDb(dbName2, dbytes);
+ T.assert( 2 == u1.getFileCount() )
+ .assert( dbytes.byteLength == nWrote );
let db2 = new u1.OpfsSAHPoolDb(dbName2);
T.assert(db2 instanceof sqlite3.oo1.DB)
.assert(3 === db2.selectValue('select count(*) from t'));
.assert(false === u1.unlink(dbName2))
.assert(1 === u1.getFileCount())
.assert(1 === u1.getFileNames().length);
+ // Try again with a function as an input source:
+ let cursor = 0;
+ const blockSize = 1024, end = dbytes.byteLength;
+ const reader = async function(){
+ if(cursor >= dbytes.byteLength){
+ return undefined;
+ }
+ const rv = dbytes.subarray(cursor, cursor+blockSize>end ? end : cursor+blockSize);
+ cursor += blockSize;
+ return rv;
+ };
+ nWrote = await u1.importDb(dbName2, reader);
+ T.assert( 2 == u1.getFileCount() );
+ db2 = new u1.OpfsSAHPoolDb(dbName2);
+ T.assert(db2 instanceof sqlite3.oo1.DB)
+ .assert(3 === db2.selectValue('select count(*) from t'));
+ db2.close();
+ T.assert(true === u1.unlink(dbName2))
+ .assert(dbytes.byteLength == nWrote);
}
T.assert(true === u1.unlink(dbName))
-C Add\snote\sabout\sthe\scurrent\sthreading\slimitation\sto\sext/jni/README.md.\sNo\scode\schanges.
-D 2023-08-15T13:01:20.690
+C Extend\sthe\simportDb()\smethod\sof\sboth\sOPFS\sVFSes\sto\s(A)\ssupport\sreading\sin\san\sasync\sstreaming\sfashion\svia\sa\scallback\sand\s(B)\sautomatically\sdisable\sWAL\smode\sin\sthe\simported\sdb.
+D 2023-08-18T14:16:26.669
F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
F ext/wasm/api/sqlite3-api-cleanup.js d235ad237df6954145404305040991c72ef8b1881715d2a650dda7b3c2576d0e
F ext/wasm/api/sqlite3-api-glue.js b65e546568f1dfb35205b9792feb5146a6323d71b55cda58e2ed30def6dd52f3
F ext/wasm/api/sqlite3-api-oo1.js 9678dc4d9a5d39632b6ffe6ea94a023119260815bf32f265bf5f6c36c9516db8
-F ext/wasm/api/sqlite3-api-prologue.js 5f283b096b98bfb1ee2f2201e7ff0489dff00e29e1030c30953bdb4f5b87f4bd
+F ext/wasm/api/sqlite3-api-prologue.js ef6f67c5ea718490806e5e17d2644b8b2f6e6ba5284d23dc1fbfd14d401c1ab5
F ext/wasm/api/sqlite3-api-worker1.js 9f32af64df1a031071912eea7a201557fe39b1738645c0134562bb84e88e2fec
F ext/wasm/api/sqlite3-license-version-header.js 0c807a421f0187e778dc1078f10d2994b915123c1223fe752b60afdcd1263f89
F ext/wasm/api/sqlite3-opfs-async-proxy.js 8cf8a897726f14071fae6be6648125162b256dfb4f96555b865dbb7a6b65e379
F ext/wasm/api/sqlite3-v-helper.js 7daa0eab0a513a25b05e9abae7b5beaaa39209b3ed12f86aeae9ef8d2719ed25
-F ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js abb69b5e008961026bf5ff433d7116cb046359af92a5daf73208af2e7ac80ae7
-F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js e04fc2fda6a0200ef80efdbb4ddfa0254453558adb17ec3a230f93d2bf1d711c
+F ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js 561463ac5380e4ccf1839a1922e6d7a5585660f32e3b9701a270b78cd35566cf
+F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js 656952a75c36d96e3248b03ae26d6a7f8d6ff31e66432c63e1c0bb021f1234ab
F ext/wasm/api/sqlite3-wasm.c d4d4c2b349b43b7b861e6d2994299630fb79e07573ea6b61e28e8071b7d16b61
F ext/wasm/api/sqlite3-worker1-promiser.c-pp.js bc06df0d599e625bde6a10a394e326dc68da9ff07fa5404354580f81566e591f
F ext/wasm/api/sqlite3-worker1.c-pp.js da509469755035e919c015deea41b4514b5e84c12a1332e6cc8d42cb2cc1fb75
F ext/wasm/test-opfs-vfs.js f09266873e1a34d9bdb6d3981ec8c9e382f31f215c9fd2f9016d2394b8ae9b7b
F ext/wasm/tester1-worker.html ebc4b820a128963afce328ecf63ab200bd923309eb939f4110510ab449e9814c
F ext/wasm/tester1.c-pp.html 1c1bc78b858af2019e663b1a31e76657b73dc24bede28ca92fbe917c3a972af2
-F ext/wasm/tester1.c-pp.js 64eb0ee6e695d5638d0f758f31a0ca2231e627ca5d768de3d8b44f9f494de8d4
+F ext/wasm/tester1.c-pp.js 9e0f4da49f02753a73a5f931bfb9b1458175518daa3fec40b5ebdc06c285539c
F ext/wasm/tests/opfs/concurrency/index.html 0802373d57034d51835ff6041cda438c7a982deea6079efd98098d3e42fbcbc1
F ext/wasm/tests/opfs/concurrency/test.js a98016113eaf71e81ddbf71655aa29b0fed9a8b79a3cdd3620d1658eb1cc9a5d
F ext/wasm/tests/opfs/concurrency/worker.js 0a8c1a3e6ebb38aabbee24f122693f1fb29d599948915c76906681bb7da1d3d2
F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 00ac653562a66aad3112ea322d08be68e05e6bf7413c814dd3f81bf850fcf43b
-R 68ba6514d93093aa325f9f497a3147a9
+P 653ed92dc39185cdedfab3ea518bc7ec2d2826120e5fa4cbdee3343301396184
+R 6b40a7c1cd4b822a33b7a897821b1648
U stephan
-Z 5cbd8ac0238d9d299017246e54070fc0
+Z b0064c5da7f02d0dcda913e568fa1924
# Remove this line to create a well-formed Fossil manifest.
-653ed92dc39185cdedfab3ea518bc7ec2d2826120e5fa4cbdee3343301396184
\ No newline at end of file
+9b1398c96a4fd0b59e65faa8d5c98de4129f0f0357732f12cb2f5c53a08acdc2
\ No newline at end of file
projects / thirdparty / sqlite.git / commitdiff
