From: larrybr Date: Fri, 5 Mar 2021 20:46:53 +0000 (+0000) Subject: Fix appendvfs bug exposed by docsapp build (in docsrc), and tighten/revise published... X-Git-Tag: version-3.35.0~32^2~8 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=feecc9f54715047919088a209c7dddba76299bdd;p=thirdparty%2Fsqlite.git Fix appendvfs bug exposed by docsapp build (in docsrc), and tighten/revise published behavior. FossilOrigin-Name: 7f0f2eacdb496f7f36865ba62164a48edb324a3c6baede736b4e7edf5e8b0fdb --- diff --git a/ext/misc/appendvfs.c b/ext/misc/appendvfs.c index 14260efb52..6d35788004 100644 --- a/ext/misc/appendvfs.c +++ b/ext/misc/appendvfs.c @@ -14,24 +14,25 @@ ** appended onto the end of some other file, such as an executable. ** ** A special record must appear at the end of the file that identifies the -** file as an appended database and provides an offset to page 1. For -** best performance page 1 should be located at a disk page boundary, though -** that is not required. +** file as an appended database and provides the offset to the first page +** of the exposed content. (Or, it is the length of the content prefix.) +** For best performance page 1 should be located at a disk page boundary, +** though that is not required. ** ** When opening a database using this VFS, the connection might treat -** the file as an ordinary SQLite database, or it might treat is as a +** the file as an ordinary SQLite database, or it might treat it as a ** database appended onto some other file. Here are the rules: ** ** (1) When opening a new empty file, that file is treated as an ordinary ** database. ** -** (2) When opening a file that begins with the standard SQLite prefix -** string "SQLite format 3", that file is treated as an ordinary -** database. -** -** (3) When opening a file that ends with the appendvfs trailer string +** (2) When opening a file that ends with the appendvfs trailer string ** "Start-Of-SQLite3-NNNNNNNN" that file is treated as an appended -** database. +** database, even if rule 3 otherwise applies. +** +** (3) When opening a file that begins with the standard SQLite prefix +** string "SQLite format 3", that file is treated as an ordinary +** database, unless rule 2 applies. ** ** (4) If none of the above apply and the SQLITE_OPEN_CREATE flag is ** set, then a new database is appended to the already existing file. @@ -39,13 +40,13 @@ ** (5) Otherwise, SQLITE_CANTOPEN is returned. ** ** To avoid unnecessary complications with the PENDING_BYTE, the size of -** the file containing the database is limited to 1GB. This VFS will refuse -** to read or write past the 1GB mark. This restriction might be lifted in -** future versions. For now, if you need a large database, then keep the -** database in a separate file. +** the file containing the database is limited to 1GB. (1000013824 bytes) +** This VFS will not read or write past the 1GB mark. This restriction +** might be lifted in future versions. For now, if you need a larger +** database, then keep it in a separate file. ** -** If the file being opened is not an appended database, then this shim is -** a pass-through into the default underlying VFS. +** If the file being opened is a plain database (not an appended one), then +** this shim is a pass-through into the default underlying VFS. (rule 3) **/ #include "sqlite3ext.h" SQLITE_EXTENSION_INIT1 @@ -58,11 +59,12 @@ SQLITE_EXTENSION_INIT1 ** 123456789 123456789 12345 ** ** The NNNNNNNN represents a 64-bit big-endian unsigned integer which is -** the offset to page 1. +** the offset to page 1, and also the length of the prefix content. */ #define APND_MARK_PREFIX "Start-Of-SQLite3-" #define APND_MARK_PREFIX_SZ 17 -#define APND_MARK_SIZE 25 +#define APND_MARK_FOS_SZ 8 +#define APND_MARK_SIZE (APND_MARK_PREFIX_SZ+APND_MARK_FOS_SZ) /* ** Maximum size of the combined prefix + database + append-mark. This @@ -70,6 +72,11 @@ SQLITE_EXTENSION_INIT1 */ #define APND_MAX_SIZE (65536*15259) +/* +** Size of storage page upon which to align appendvfs portion. +*/ +#define APND_ROUNDUP_BITS 12 + /* ** Forward declaration of objects used by this utility */ @@ -82,11 +89,39 @@ typedef struct ApndFile ApndFile; #define ORIGVFS(p) ((sqlite3_vfs*)((p)->pAppData)) #define ORIGFILE(p) ((sqlite3_file*)(((ApndFile*)(p))+1)) -/* An open file */ +/* Invariants for an open appendvfs file: + * Once an appendvfs file is opened, it will be in one of three states: + * State 0: Never written. Underlying file (if any) is unaltered. + * State 1: Append mark is persisted, content write is in progress. + * State 2: Append mark is persisted, content writes are complete. + * + * State 0 is persistent in the sense that nothing will have been done + * to the underlying file, including any attempt to convert it to an + * appendvfs file. + * + * State 1 is normally transitory. However, if a write operation ends + * abnormally (disk full, power loss, process kill, etc.), then State 1 + * may be persistent on disk with an incomplete content write-out. This + * is logically equivalent to an interrupted write to an ordinary file, + * where some unknown portion of to-be-written data is persisted while + * the remainder is not. Database integrity in such cases is maintained + * (or not) by the same measures available for ordinary file access. + * + * State 2 is persistent under normal circumstances (when there is no + * abnormal termination of a write operation such that data provided + * to the underlying VFS write method has not yet reached storage.) + * + * In order to maintain the state invariant, the append mark is written + * in advance of content writes where any part of such content would + * overwrite an existing (or yet to be written) append mark. + */ struct ApndFile { - sqlite3_file base; /* IO methods */ - sqlite3_int64 iPgOne; /* File offset to page 1 */ - sqlite3_int64 iMark; /* Start of the append-mark */ + /* IO methods of the underlying file */ + sqlite3_file base; + /* File offset to beginning of appended content (unchanging) */ + sqlite3_int64 iPgOne; + /* File offset of written append-mark, or -1 if unwritten */ + sqlite3_int64 iMark; }; /* @@ -178,8 +213,6 @@ static const sqlite3_io_methods apnd_io_methods = { apndUnfetch /* xUnfetch */ }; - - /* ** Close an apnd-file. */ @@ -203,16 +236,29 @@ static int apndRead( } /* -** Add the append-mark onto the end of the file. +** Add the append-mark onto what should become the end of the file. +* If and only if this succeeds, the ApndFile.iMark is updated. +* If it fails, there is little reason to proceed with content writes. +* Parameter imoNext is the appendvfs-relative offset of the new mark. */ -static int apndWriteMark(ApndFile *p, sqlite3_file *pFile){ - int i; +static int apndWriteMark( + ApndFile *p, + sqlite3_file *pFile, + sqlite_int64 imoNext +){ unsigned char a[APND_MARK_SIZE]; + int ibs = (APND_MARK_FOS_SZ - 1) * 8; + int i, rc; memcpy(a, APND_MARK_PREFIX, APND_MARK_PREFIX_SZ); - for(i=0; i<8; i++){ - a[APND_MARK_PREFIX_SZ+i] = (p->iPgOne >> (56 - i*8)) & 0xff; + for(i=0; iiPgOne >> ibs) & 0xff; + } + imoNext += p->iPgOne; + if( SQLITE_OK==(rc = pFile->pMethods->xWrite + (pFile, a, APND_MARK_SIZE, imoNext)) ){ + p->iMark = imoNext; } - return pFile->pMethods->xWrite(pFile, a, APND_MARK_SIZE, p->iMark); + return rc; } /* @@ -224,23 +270,16 @@ static int apndWrite( int iAmt, sqlite_int64 iOfst ){ - int rc; ApndFile *p = (ApndFile *)pFile; + sqlite_int64 imoNext = iOfst + iAmt; + if( imoNext>=APND_MAX_SIZE ) return SQLITE_FULL; pFile = ORIGFILE(pFile); - if( iOfst+iAmt>=APND_MAX_SIZE ) return SQLITE_FULL; - rc = pFile->pMethods->xWrite(pFile, zBuf, iAmt, iOfst+p->iPgOne); - if( rc==SQLITE_OK && iOfst + iAmt + p->iPgOne > p->iMark ){ - sqlite3_int64 sz = 0; - rc = pFile->pMethods->xFileSize(pFile, &sz); - if( rc==SQLITE_OK ){ - p->iMark = sz - APND_MARK_SIZE; - if( iOfst + iAmt + p->iPgOne > p->iMark ){ - p->iMark = p->iPgOne + iOfst + iAmt; - rc = apndWriteMark(p, pFile); - } - } + if( p->iMark < 0 || imoNext + p->iPgOne > p->iMark ){ + int rc = apndWriteMark(p, pFile, imoNext); + if( SQLITE_OK!=rc ) + return rc; } - return rc; + return pFile->pMethods->xWrite(pFile, zBuf, iAmt, iOfst+p->iPgOne); } /* @@ -249,11 +288,13 @@ static int apndWrite( static int apndTruncate(sqlite3_file *pFile, sqlite_int64 size){ int rc; ApndFile *p = (ApndFile *)pFile; + sqlite_int64 iomNext = size+p->iPgOne; pFile = ORIGFILE(pFile); - rc = pFile->pMethods->xTruncate(pFile, size+p->iPgOne+APND_MARK_SIZE); + rc = apndWriteMark(p, pFile, iomNext); if( rc==SQLITE_OK ){ - p->iMark = p->iPgOne+size; - rc = apndWriteMark(p, pFile); + rc = pFile->pMethods->xTruncate(pFile, size+p->iPgOne+APND_MARK_SIZE); + if( rc==SQLITE_OK ) + p->iMark = iomNext; } return rc; } @@ -268,16 +309,12 @@ static int apndSync(sqlite3_file *pFile, int flags){ /* ** Return the current file-size of an apnd-file. +** If the append mark is not yet there, the file-size is 0. */ static int apndFileSize(sqlite3_file *pFile, sqlite_int64 *pSize){ - ApndFile *p = (ApndFile *)pFile; - int rc; - pFile = ORIGFILE(p); - rc = pFile->pMethods->xFileSize(pFile, pSize); - if( rc==SQLITE_OK && p->iPgOne ){ - *pSize -= p->iPgOne + APND_MARK_SIZE; - } - return rc; + ApndFile *paf = (ApndFile *)pFile; + *pSize = ( paf->iMark >= 0 )? (paf->iMark - paf->iPgOne) : 0; + return SQLITE_OK; } /* @@ -372,6 +409,8 @@ static int apndFetch( void **pp ){ ApndFile *p = (ApndFile *)pFile; + if( p->iMark < 0 || iOfst+iAmt > p->iMark) + return SQLITE_IOERR; /* Cannot read what is not yet there. */ pFile = ORIGFILE(pFile); return pFile->pMethods->xFetch(pFile, iOfst+p->iPgOne, iAmt, pp); } @@ -383,40 +422,69 @@ static int apndUnfetch(sqlite3_file *pFile, sqlite3_int64 iOfst, void *pPage){ return pFile->pMethods->xUnfetch(pFile, iOfst+p->iPgOne, pPage); } -/* -** Check to see if the file is an ordinary SQLite database file. -*/ -static int apndIsOrdinaryDatabaseFile(sqlite3_int64 sz, sqlite3_file *pFile){ - int rc; - char zHdr[16]; - static const char aSqliteHdr[] = "SQLite format 3"; - if( sz<512 ) return 0; - rc = pFile->pMethods->xRead(pFile, zHdr, sizeof(zHdr), 0); - if( rc ) return 0; - return memcmp(zHdr, aSqliteHdr, sizeof(zHdr))==0; -} - /* ** Try to read the append-mark off the end of a file. Return the -** start of the appended database if the append-mark is present. If -** there is no append-mark, return -1; +** start of the appended database if the append-mark is present. +** If there is no valid append-mark, return -1; */ static sqlite3_int64 apndReadMark(sqlite3_int64 sz, sqlite3_file *pFile){ int rc, i; sqlite3_int64 iMark; + int msbs = 8 * (APND_MARK_FOS_SZ-1); unsigned char a[APND_MARK_SIZE]; - if( sz<=APND_MARK_SIZE ) return -1; + if( APND_MARK_SIZE!=(sz & 0x1ff) ) return -1; rc = pFile->pMethods->xRead(pFile, a, APND_MARK_SIZE, sz-APND_MARK_SIZE); if( rc ) return -1; if( memcmp(a, APND_MARK_PREFIX, APND_MARK_PREFIX_SZ)!=0 ) return -1; - iMark = ((sqlite3_int64)(a[APND_MARK_PREFIX_SZ]&0x7f))<<56; - for(i=1; i<8; i++){ - iMark += (sqlite3_int64)a[APND_MARK_PREFIX_SZ+i]<<(56-8*i); + iMark = ((sqlite3_int64)(a[APND_MARK_PREFIX_SZ] & 0x7f)) << msbs; + for(i=1; i<8; i++){ + msbs -= 8; + iMark |= (sqlite3_int64)a[APND_MARK_PREFIX_SZ+i]<=0 ){ + /* If file has right end-marker, the expected odd size, and the + * SQLite DB type marker where the end-marker puts it, then it + * is an appendvfs database (to be treated as such.) + */ + rc = pFile->pMethods->xRead(pFile, zHdr, sizeof(zHdr), iMark); + if( SQLITE_OK==rc && memcmp(zHdr, apvfsSqliteHdr, sizeof(zHdr))==0 + && (sz & 0x1ff)== APND_MARK_SIZE && sz>=512+APND_MARK_SIZE ) + return 1; /* It's an appendvfs database */ + } + return 0; +} + +/* +** Check to see if the file is an ordinary SQLite database file. +** Return true iff so. Parameter sz is the file's size. +*/ +static int apndIsOrdinaryDatabaseFile(sqlite3_int64 sz, sqlite3_file *pFile){ + char zHdr[16]; + if( apndIsAppendvfsDatabase(sz, pFile) /* rule 2 */ + || (sz & 0x1ff) != 0 + || SQLITE_OK!=pFile->pMethods->xRead(pFile, zHdr, sizeof(zHdr), 0) + || memcmp(zHdr, apvfsSqliteHdr, sizeof(zHdr))!=0 ) + return 0; + return 1; +} + +/* Round-up used to get appendvfs portion to begin at a page boundary. */ +#define APND_START_ROUNDUP(fsz, nPageBits) \ + ((fsz) + ((1<xOpen(pSubVfs, zName, pFile, flags, pOutFlags); } @@ -451,27 +523,42 @@ static int apndOpen( memmove(pFile, pSubFile, pSubVfs->szOsFile); return SQLITE_OK; } - p->iMark = 0; + /* Record that append mark has not been written until seen otherwise. */ + p->iMark = -1; p->iPgOne = apndReadMark(sz, pFile); - if( p->iPgOne>0 ){ + if( p->iPgOne>=0 ){ + /* Append mark was found, infer its offset */ + p->iMark = sz - p->iPgOne - APND_MARK_SIZE; return SQLITE_OK; } if( (flags & SQLITE_OPEN_CREATE)==0 ){ pSubFile->pMethods->xClose(pSubFile); rc = SQLITE_CANTOPEN; } - p->iPgOne = (sz+0xfff) & ~(sqlite3_int64)0xfff; + /* Round newly added appendvfs location to #define'd page boundary. + * Note that nothing has yet been written to the underlying file. + * The append mark will be written along with first content write. + * Until then, the p->iMark value indicates it is not yet written. + */ + p->iPgOne = APND_START_ROUNDUP(sz, APND_ROUNDUP_BITS); apnd_open_done: if( rc ) pFile->pMethods = 0; return rc; } /* -** All other VFS methods are pass-thrus. +** Delete an apnd file. +** For an appendvfs, this could mean delete the appendvfs portion, +** leaving the appendee as it was before it gained an appendvfs. +** For now, this code deletes the underlying file too. */ static int apndDelete(sqlite3_vfs *pVfs, const char *zPath, int dirSync){ return ORIGVFS(pVfs)->xDelete(ORIGVFS(pVfs), zPath, dirSync); } + +/* +** All other VFS methods are pass-thrus. +*/ static int apndAccess( sqlite3_vfs *pVfs, const char *zPath, diff --git a/manifest b/manifest index f65ad9cbeb..3a5ee2ff3d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Create\snew\sbranch\snamed\s"appendvfs_tighten" -D 2021-03-05T09:42:58.132 +C Fix\sappendvfs\sbug\sexposed\sby\sdocsapp\sbuild\s(in\sdocsrc),\sand\stighten/revise\spublished\sbehavior. +D 2021-03-05T20:46:53.770 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -286,7 +286,7 @@ F ext/lsm1/tool/mklsm1c.tcl f31561bbee5349f0a554d1ad7236ac1991fc09176626f529f607 F ext/misc/README.md d6dd0fe1d8af77040216798a6a2b0c46c73054d2f0ea544fbbcdccf6f238c240 F ext/misc/amatch.c e3ad5532799cee9a97647f483f67f43b38796b84b5a8c60594fe782a4338f358 F ext/misc/anycollseq.c 5ffdfde9829eeac52219136ad6aa7cd9a4edb3b15f4f2532de52f4a22525eddb -F ext/misc/appendvfs.c 55121d311d408ba9c62c3cfa367408887638f02f9522dd9859891d0ee69a7eba +F ext/misc/appendvfs.c 82cd0dc37b5645b079365c2ebd5b361548896c401db9280da2f7f64d4f02f618 F ext/misc/blobio.c a867c4c4617f6ec223a307ebfe0eabb45e0992f74dd47722b96f3e631c0edb2a F ext/misc/btreeinfo.c d28ce349b40054eaa9473e835837bad7a71deec33ba13e39f963d50933bfa0f9 F ext/misc/carray.c b75a0f207391038bf1540d3372f482a95c3613511c7c474db51ede1196321c7c @@ -1909,10 +1909,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 23459071091248e94202e609bb8031c3e34722b8ae8ff5a66851dcc528a2d2c2 -R 3f829008faf6bb2f8b2a8086ffa5bb99 -T *branch * appendvfs_tighten -T *sym-appendvfs_tighten * -T -sym-trunk * +P 48c968bf4c8e1517864bec68d3072441994ea511a9cb446d2fdf031d5f43b077 +R d9b9b7ebcf10ff9c889756135ed9e77d U larrybr -Z 63ac0fc408a0a285700284c9e62510e0 +Z c2502dcdc1270b481b2f881df0282e61 diff --git a/manifest.uuid b/manifest.uuid index 90afab8359..6b7bb0e60d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -48c968bf4c8e1517864bec68d3072441994ea511a9cb446d2fdf031d5f43b077 \ No newline at end of file +7f0f2eacdb496f7f36865ba62164a48edb324a3c6baede736b4e7edf5e8b0fdb \ No newline at end of file