From: larrybr Date: Sun, 7 Mar 2021 18:55:25 +0000 (+0000) Subject: Add grow/shrink tests. Fix truncate bug. Make appendvfs.c easier to inspect. X-Git-Tag: version-3.35.0~32^2~4 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=7e398c04fed96d4b00f167725c5f7f553bfdc6c1;p=thirdparty%2Fsqlite.git Add grow/shrink tests. Fix truncate bug. Make appendvfs.c easier to inspect. FossilOrigin-Name: c9521fa55620d0cb3cec20db7b42bc5f18876b42dca07811d1cd0f4b2907eab2 --- diff --git a/ext/misc/appendvfs.c b/ext/misc/appendvfs.c index 6d35788004..b7641c4853 100644 --- a/ext/misc/appendvfs.c +++ b/ext/misc/appendvfs.c @@ -75,7 +75,9 @@ SQLITE_EXTENSION_INIT1 /* ** Size of storage page upon which to align appendvfs portion. */ +#ifndef APND_ROUNDUP_BITS #define APND_ROUNDUP_BITS 12 +#endif /* ** Forward declaration of objects used by this utility @@ -116,7 +118,7 @@ typedef struct ApndFile ApndFile; * overwrite an existing (or yet to be written) append mark. */ struct ApndFile { - /* IO methods of the underlying file */ + /* Access to IO methods of the underlying file */ sqlite3_file base; /* File offset to beginning of appended content (unchanging) */ sqlite3_int64 iPgOne; @@ -230,33 +232,35 @@ static int apndRead( int iAmt, sqlite_int64 iOfst ){ - ApndFile *p = (ApndFile *)pFile; + ApndFile *paf = (ApndFile *)pFile; pFile = ORIGFILE(pFile); - return pFile->pMethods->xRead(pFile, zBuf, iAmt, iOfst+p->iPgOne); + return pFile->pMethods->xRead(pFile, zBuf, iAmt, paf->iPgOne+iOfst); } /* ** Add the append-mark onto what should become the end of the file. -* If and only if this succeeds, the ApndFile.iMark is updated. -* If it fails, there is little reason to proceed with content writes. -* Parameter imoNext is the appendvfs-relative offset of the new mark. +* If and only if this succeeds, internal ApndFile.iMark is updated. +* Parameter iWriteEnd is the appendvfs-relative offset of the new mark. */ static int apndWriteMark( - ApndFile *p, + ApndFile *paf, sqlite3_file *pFile, - sqlite_int64 imoNext + sqlite_int64 iWriteEnd ){ + sqlite_int64 iPgOne = paf->iPgOne; unsigned char a[APND_MARK_SIZE]; - int ibs = (APND_MARK_FOS_SZ - 1) * 8; - int i, rc; + int i = APND_MARK_FOS_SZ; + int rc; + assert(pFile == ORIGFILE(paf)); memcpy(a, APND_MARK_PREFIX, APND_MARK_PREFIX_SZ); - for(i=0; iiPgOne >> ibs) & 0xff; + while (--i >= 0) { + a[APND_MARK_PREFIX_SZ+i] = (unsigned char)(iPgOne & 0xff); + iPgOne >>= 8; } - imoNext += p->iPgOne; + iWriteEnd += paf->iPgOne; if( SQLITE_OK==(rc = pFile->pMethods->xWrite - (pFile, a, APND_MARK_SIZE, imoNext)) ){ - p->iMark = imoNext; + (pFile, a, APND_MARK_SIZE, iWriteEnd)) ){ + paf->iMark = iWriteEnd; } return rc; } @@ -270,33 +274,30 @@ static int apndWrite( int iAmt, sqlite_int64 iOfst ){ - ApndFile *p = (ApndFile *)pFile; - sqlite_int64 imoNext = iOfst + iAmt; - if( imoNext>=APND_MAX_SIZE ) return SQLITE_FULL; + ApndFile *paf = (ApndFile *)pFile; + sqlite_int64 iWriteEnd = iOfst + iAmt; + if( iWriteEnd>=APND_MAX_SIZE ) return SQLITE_FULL; pFile = ORIGFILE(pFile); - if( p->iMark < 0 || imoNext + p->iPgOne > p->iMark ){ - int rc = apndWriteMark(p, pFile, imoNext); + /* If append-mark is absent or will be overwritten, write it. */ + if( paf->iMark < 0 || paf->iPgOne + iWriteEnd > paf->iMark ){ + int rc = apndWriteMark(paf, pFile, iWriteEnd); if( SQLITE_OK!=rc ) return rc; } - return pFile->pMethods->xWrite(pFile, zBuf, iAmt, iOfst+p->iPgOne); + return pFile->pMethods->xWrite(pFile, zBuf, iAmt, paf->iPgOne+iOfst); } /* ** Truncate an apnd-file. */ static int apndTruncate(sqlite3_file *pFile, sqlite_int64 size){ - int rc; - ApndFile *p = (ApndFile *)pFile; - sqlite_int64 iomNext = size+p->iPgOne; + ApndFile *paf = (ApndFile *)pFile; pFile = ORIGFILE(pFile); - rc = apndWriteMark(p, pFile, iomNext); - if( rc==SQLITE_OK ){ - rc = pFile->pMethods->xTruncate(pFile, size+p->iPgOne+APND_MARK_SIZE); - if( rc==SQLITE_OK ) - p->iMark = iomNext; - } - return rc; + /* The append mark goes out first so truncate failure does not lose it. */ + if( SQLITE_OK!=apndWriteMark(paf, pFile, size) ) + return SQLITE_IOERR; + /* Truncate underlying file just past append mark */ + return pFile->pMethods->xTruncate(pFile, paf->iMark+APND_MARK_SIZE); } /* @@ -345,12 +346,12 @@ static int apndCheckReservedLock(sqlite3_file *pFile, int *pResOut){ ** File control method. For custom operations on an apnd-file. */ static int apndFileControl(sqlite3_file *pFile, int op, void *pArg){ - ApndFile *p = (ApndFile *)pFile; + ApndFile *paf = (ApndFile *)pFile; int rc; pFile = ORIGFILE(pFile); rc = pFile->pMethods->xFileControl(pFile, op, pArg); if( rc==SQLITE_OK && op==SQLITE_FCNTL_VFSNAME ){ - *(char**)pArg = sqlite3_mprintf("apnd(%lld)/%z", p->iPgOne, *(char**)pArg); + *(char**)pArg = sqlite3_mprintf("apnd(%lld)/%z", paf->iPgOne, *(char**)pArg); } return rc; } diff --git a/main.mk b/main.mk index 98dabdf446..733d51372a 100644 --- a/main.mk +++ b/main.mk @@ -360,8 +360,8 @@ TESTSRC = \ # TESTSRC += \ $(TOP)/ext/misc/amatch.c \ - $(TOP)/ext/misc/carray.c \ $(TOP)/ext/misc/appendvfs.c \ + $(TOP)/ext/misc/carray.c \ $(TOP)/ext/misc/cksumvfs.c \ $(TOP)/ext/misc/closure.c \ $(TOP)/ext/misc/csv.c \ diff --git a/manifest b/manifest index 981b78f9b7..6447ce8b9f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\sfrom\strunk. -D 2021-03-06T23:49:18.484 +C Add\sgrow/shrink\stests.\sFix\struncate\sbug.\sMake\sappendvfs.c\seasier\sto\sinspect. +D 2021-03-07T18:55:25.654 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -286,7 +286,7 @@ F ext/lsm1/tool/mklsm1c.tcl f31561bbee5349f0a554d1ad7236ac1991fc09176626f529f607 F ext/misc/README.md d6dd0fe1d8af77040216798a6a2b0c46c73054d2f0ea544fbbcdccf6f238c240 F ext/misc/amatch.c e3ad5532799cee9a97647f483f67f43b38796b84b5a8c60594fe782a4338f358 F ext/misc/anycollseq.c 5ffdfde9829eeac52219136ad6aa7cd9a4edb3b15f4f2532de52f4a22525eddb -F ext/misc/appendvfs.c 82cd0dc37b5645b079365c2ebd5b361548896c401db9280da2f7f64d4f02f618 +F ext/misc/appendvfs.c 747f1ee3d0cd708426f8569bc5a74f76a6c5c9a9effd9dbbdb1d72ebef978f25 F ext/misc/blobio.c a867c4c4617f6ec223a307ebfe0eabb45e0992f74dd47722b96f3e631c0edb2a F ext/misc/btreeinfo.c d28ce349b40054eaa9473e835837bad7a71deec33ba13e39f963d50933bfa0f9 F ext/misc/carray.c b75a0f207391038bf1540d3372f482a95c3613511c7c474db51ede1196321c7c @@ -464,7 +464,7 @@ F ext/userauth/userauth.c 7f00cded7dcaa5d47f54539b290a43d2e59f4b1eb5f447545fa865 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60 -F main.mk 2c2a5c91eefeacc400a3e19018250fa8d9d236b49395e8e15bf0045ebdb71510 +F main.mk c6afb1c8bcacc4c3e320ab10259879bc927f0747401e184b64e8f4b78f057ab2 F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83 F mptest/config01.test 3c6adcbc50b991866855f1977ff172eb6d901271 F mptest/config02.test 4415dfe36c48785f751e16e32c20b077c28ae504 @@ -699,7 +699,7 @@ F test/autoindex4.test 49d3cd791a9baa16fb461d7ea3de80d019a819cf F test/autoindex5.test a5d72fe8c217cc0ea356dc6fa06a282a8a3fc53aa807709d79dba07a8f248102 F test/autovacuum.test 0831cd34e14695d297187f7f6519265e3121c5b0a1720e548e86829e796129e9 F test/autovacuum_ioerr2.test 8a367b224183ad801e0e24dcb7d1501f45f244b4 -F test/avfs.test 810ecf3b0ad344e917ea6bcfb599a1ac5aa6913b6eefe57f2c72c1414fa6cf24 +F test/avfs.test 243f1d2794b1c411e203cb5ec6f57875c405680aee393fe653dfb7e4a5e29703 F test/avtrans.test b7dc25459ecbd86c6fa9c606ee3068f59d81e225118617dcf2bbb6ded2ade89e F test/backcompat.test 3e64cedda754c778ef6bbe417b6e7a295e662a4d F test/backup.test dd4a5ff756e3df3931dacb1791db0584d4bad989 @@ -1910,7 +1910,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 9f33d45179f5985bd4c4f7582e94f5833a63f853c8cdc19f39cf7e3ee70177d0 5ddd8032ef9ecd5b53909d304c8f1375a72f270fd7810964cb4e385e25bedd4f -R cbc59f10ccb84702beac2fcfd70d381a +P 729117cdfa2d6ff24b66656e9ccc380199606b830fc338286e4dd10e8e787597 +R 4ca236c48800b688829480b8bceb70c7 U larrybr -Z b57dd333b8789a25a8927f035a04d7d3 +Z e10dee79bfa99113490b8465597823aa diff --git a/manifest.uuid b/manifest.uuid index 6d705275ad..65e0ebe948 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -729117cdfa2d6ff24b66656e9ccc380199606b830fc338286e4dd10e8e787597 \ No newline at end of file +c9521fa55620d0cb3cec20db7b42bc5f18876b42dca07811d1cd0f4b2907eab2 \ No newline at end of file diff --git a/test/avfs.test b/test/avfs.test index 9e4b68a727..1f292ee9c3 100644 --- a/test/avfs.test +++ b/test/avfs.test @@ -12,9 +12,13 @@ # This file implements tests for the appendvfs extension. # # Tests performed: -# avfs-1. Test that an appendvfs DB can be added to a simple text file. -# avfs-2. Test that the DB can be read with correct content upon reopen. -# avfs-3. Test that the simple text file retains its initial text. +# avfs-1.0. Test that an appendvfs DB can be added to an empty file. +# avfs-1.1. Test that the DB can be read with correct content upon reopen. +# avfs-1.2. Test that an appendvfs DB can be added to a simple text file. +# avfs-1.3. Test that the DB can be read with correct content upon reopen. +# avfs-2.1. Test that the simple text file retains its initial text. +# avfs-3.1. Test that the appendvfs can grow and shrink, remaining intact. +# avfs-3.2. Test that appendvfs is intact after grow/shrink/close/reopen. # ... # (more to come) @@ -24,43 +28,97 @@ set ::testprefix avfs load_static_extension db appendvfs -set ::fa file1.adb +set ::fa avfs.adb +set ::fza avfs.sdb +forcedelete $::fa $::fza set ::result {} +set ::vf "&vfs=apndvfs" + +# Return file offset of appendvfs portion of a file, or {} if none such. +proc fosAvfs {fname} { + if {[file size $fname] < 25} { + return {} + } + if {[catch {set fd [open $fname rb]}]} { + return {} + } + seek $fd -25 end + set am [read $fd 17] + set ao [read $fd 8] + close $fd + if {$am ne "Start-Of-SQLite3-"} { + return {} + } + binary scan $ao "W" rvo + return $rvo +} + do_test 1.0 { set results {} - forcedelete $::fa + set out [open $::fza wb] + close $out + sqlite3 adb "file:$::fza?mode=rwc$::vf" -uri 1 + adb eval { + PRAGMA page_size=1024; + PRAGMA cache_size=10; + CREATE TABLE t1(a TEXT); + INSERT INTO t1 VALUES ('dog'),('cat'); + SELECT group_concat(a) as pets FROM (SELECT a FROM t1 ORDER BY a); + } { lappend results $pets } + adb close + lappend results [fosAvfs $fza] + set ::result [join $results " | "] +} {cat,dog | 0} + +do_test 1.1 { + set results {} + sqlite3 adb "file:$::fza?mode=rw$::vf" -uri 1 + adb eval { + SELECT group_concat(a) as pets FROM (SELECT a FROM t1 ORDER BY a DESC); + } { lappend results $pets } + adb close + set ::result [join $results " | "] +} {dog,cat} + +do_test 1.2 { + set results {} set out [open $::fa wb] set ::tlo { "Just some text," "and more text," "ending at 3 lines." } puts $out [join $::tlo "\n"] close $out set adbSz [file size $::fa] - sqlite3 adb "file:$::fa?mode=rwc&vfs=apndvfs" -uri 1 + sqlite3 adb "file:$::fa?mode=rwc$::vf" -uri 1 adb eval { PRAGMA page_size=512; + PRAGMA cache_size=0; CREATE TABLE t1(a TEXT); - INSERT INTO t1 VALUES ('dog'),('cat'); - SELECT group_concat(a) as pets FROM t1; + INSERT INTO t1 VALUES ('dog'),('cat'),('pig'); + SELECT group_concat(a) as pets FROM (SELECT a FROM t1 ORDER BY a); } { lappend results $pets } adb close set adaSz [file size $::fa] lappend results "Bytes before/after $adbSz/$adaSz" set ::result [join $results " | "] -} {dog,cat | Bytes before/after 50/5145} +} {cat,dog,pig | Bytes before/after 50/5145} -do_test 2.0 { +do_test 1.3 { set results {} - sqlite3 adb "file:$::fa?mode=rw&vfs=apndvfs" -uri 1 + sqlite3 adb "file:$::fa?mode=rw$::vf" -uri 1 adb eval { - SELECT group_concat(a) as pets FROM t1; + SELECT group_concat(a) as pets FROM (SELECT a FROM t1 ORDER BY a DESC); } { lappend results $pets } adb close + lappend results [fosAvfs $fa] set ::result [join $results " | "] -} {dog,cat} +} {pig,dog,cat | 4096} -do_test 3.0 { +do_test 2.1 { set in [open $::fa r] - set tli [list [gets $in] [gets $in] [gets $in]] + set tli {} + for {set i [llength $::tlo]} {$i > 0} {incr i -1} { + lappend tli [gets $in] + } close $in if { [join $tli ":"] ne [join $::tlo ":"] } { set ::result "Appendee changed." @@ -69,8 +127,69 @@ do_test 3.0 { } } {Appendee intact.} -unset ::fa ::tlo ::result +# Set of repeatable random integers for a couple tests. +proc rint {v} { + return [::tcl::mathfunc::int [expr $v * 100000]] +} +array set ::randints [list 0 [rint [::tcl::mathfunc::srand 0]]] +for {set i 1} {$i < 10000} {incr i} { + set ::randints($i) [rint [::tcl::mathfunc::rand]] +} + +do_test 3.1 { + set results {} + sqlite3 adb "file:$::fa?mode=rw$::vf" -uri 1 + adb eval { + DROP TABLE t1; + PRAGMA cache_size=10; + CREATE TABLE ri (i INTEGER); + BEGIN; + } + for {set i 0} {$i < 10000} {incr i} { + set r $::randints($i) + set s $::randints([incr i]) + set t $::randints([incr i]) + set u $::randints([incr i]) + set v $::randints([incr i]) + adb eval { + INSERT INTO ri VALUES ($r),($s),($t),($u),($v) + } + } + adb eval { + COMMIT; + SELECT integrity_check as ic FROM pragma_integrity_check(); + } { lappend results $ic } + set adbSz [file size $::fa] + set qr {} + adb eval { + SELECT count(*) as ic FROM ri; + DELETE FROM ri WHERE (i % 50) <> 25; + SELECT integrity_check as ic FROM pragma_integrity_check(); + VACUUM; + SELECT integrity_check as ic FROM pragma_integrity_check(); + SELECT count(*) as ic FROM ri; + } { lappend qr $ic } + adb close + set adaSz [file size $::fa] + set adba [expr ($adbSz + 0.1)/$adaSz] + # lappend results $adbSz $adaSz + set results [concat $results [lrange $qr 0 2]] + lappend results [expr {$adba > 10.0 && $adba < 20.0}] + set ::result [join $results " | "] +} {ok | 10000 | ok | ok | 1} + +do_test 3.2 { + set results {} + sqlite3 adb "file:$::fa?mode=rw$::vf" -uri 1 + adb eval { + SELECT integrity_check as ic FROM pragma_integrity_check(); + } { lappend results $ic } + adb close + set ::result [join $results " | "] +} {ok} + +#forcedelete $::fa $::fd -# forcedelete $::fa +#unset ::fa ::fd ::tlo ::result ::randints finish_test