From: dan Date: Thu, 22 Apr 2010 19:14:13 +0000 (+0000) Subject: Create a version of the log checksummer that works on big-endian platforms. Remove... X-Git-Tag: version-3.7.2~455^2~63 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=ce4f05f3749f6912a5c5967061d84a480df1237e;p=thirdparty%2Fsqlite.git Create a version of the log checksummer that works on big-endian platforms. Remove the 512KB size limit on the log-summary. FossilOrigin-Name: 5d6d4423d1def39bd2424703120aa985085c3f8e --- diff --git a/manifest b/manifest index 8b101354cd..b451c49511 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Further\stests\sand\schanges\srelated\sto\sswitching\sbetween\sWAL\sand\srollback\smodes. -D 2010-04-22T06:27:05 +C Create\sa\sversion\sof\sthe\slog\schecksummer\sthat\sworks\son\sbig-endian\splatforms.\sRemove\sthe\s512KB\ssize\slimit\son\sthe\slog-summary. +D 2010-04-22T19:14:14 F Makefile.arm-wince-mingw32ce-gcc fcd5e9cd67fe88836360bb4f9ef4cb7f8e2fb5a0 F Makefile.in 4f2f967b7e58a35bb74fb7ec8ae90e0f4ca7868b F Makefile.linux-gcc d53183f4aa6a9192d249731c90dbdffbd2c68654 @@ -131,7 +131,7 @@ F src/journal.c b0ea6b70b532961118ab70301c00a33089f9315c F src/legacy.c a199d7683d60cef73089e892409113e69c23a99f F src/lempar.c 7f026423f4d71d989e719a743f98a1cbd4e6d99e F src/loadext.c 1c7a61ce1281041f437333f366a96aa0d29bb581 -F src/log.c edafa806982a281205e71c439e99766e924de938 +F src/log.c da486aab7abfab8edbc20eb49e2f68e9d3a902b6 F src/log.h b8c45a6348d9ef57c5205a08c611d57d07ee9feb F src/main.c 867de6aa444abd97771b2b70472f448d65c1c77e F src/malloc.c a08f16d134f0bfab6b20c3cd142ebf3e58235a6a @@ -154,7 +154,7 @@ F src/os_common.h 240c88b163b02c21a9f21f87d49678a0aa21ff30 F src/os_os2.c 75a8c7b9a00a2cf1a65f9fa4afbc27d46634bb2f F src/os_unix.c 5bf0015cebe2f21635da2af983c348eb88b3b4c1 F src/os_win.c 1c7453c2df4dab26d90ff6f91272aea18bcf7053 -F src/pager.c 06fb9b8b7c5283c62f5258e098d6d1649dde7635 +F src/pager.c d83d2ea6d2025554e079f891854b133924305e19 F src/pager.h cee4487ab4f0911dd9f22a40e3cd55afdb7ef444 F src/parse.y ace5c7a125d9f2a410e431ee3209034105045f7e F src/pcache.c ace8f6a5ecd4711cc66a1b23053be7109bd437cf @@ -758,7 +758,7 @@ F test/vtabE.test 7c4693638d7797ce2eda17af74292b97e705cc61 F test/vtab_alter.test 9e374885248f69e251bdaacf480b04a197f125e5 F test/vtab_err.test 0d4d8eb4def1d053ac7c5050df3024fd47a3fbd8 F test/vtab_shared.test 0eff9ce4f19facbe0a3e693f6c14b80711a4222d -F test/wal.test fd0e5914599b205172dae33c765368a75feb2462 +F test/wal.test fcb5ec1fbec08c1165b6974f126056f2b4cead49 F test/walcrash.test f022cee7eb7baa5fb898726120a6a4073dd831d1 F test/walhook.test 76a559e262f0715c470bade4a8d8333035f8ee47 F test/walmode.test c2f4e30ad64910b2d8faf6cf4e940b3f201b41df @@ -807,7 +807,7 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P 9f4f933f2c6596064fcfc7fb5add87e8c5b57448 -R cdccc9135123b3edc2fcb17c83fbd857 +P 1236318477787a612f02cc98caf2931bd2e99d94 +R 2eba2ad72744173c34347863efb93bd2 U dan -Z 891f9ffc6892beb06734f78c0f8e5711 +Z b18c4f1fb26d4b385c95cfa4edf72422 diff --git a/manifest.uuid b/manifest.uuid index 0aa3e70b8f..815632c84e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1236318477787a612f02cc98caf2931bd2e99d94 \ No newline at end of file +5d6d4423d1def39bd2424703120aa985085c3f8e \ No newline at end of file diff --git a/src/log.c b/src/log.c index 89b60568b5..f32fdb221d 100644 --- a/src/log.c +++ b/src/log.c @@ -83,6 +83,15 @@ struct LogSummaryHdr { LOG_HDRSIZE + ((iFrame)-1)*((pgsz)+LOG_FRAME_HDRSIZE) \ ) +/* +** If using mmap() to access a shared (or otherwise) log-summary file, then +** the mapping size is incremented in units of the following size. +** +** A 64 KB log-summary mapping corresponds to a log file containing over +** 13000 frames, so the mapping size does not need to be increased often. +*/ +#define LOGSUMMARY_MMAP_INCREMENT (64*1024) + /* ** There is one instance of this structure for each log-summary object ** that this process has a connection to. They are stored in a linked @@ -99,6 +108,7 @@ struct LogSummary { char *zPath; /* Path to associated WAL file */ LogLock *pLock; /* Linked list of locks on this object */ LogSummary *pNext; /* Next in global list */ + int nData; /* Size of aData allocation/mapping */ u32 *aData; /* File body */ }; @@ -288,10 +298,22 @@ static void logChecksumBytes(u8 *aByte, int nByte, u32 *aCksum){ assert( LOG_CKSM_BYTES==2*sizeof(u32) ); assert( (nByte&0x00000003)==0 ); - do { - sum1 += (*a32++); - sum2 += sum1; - } while( a32>24); aCksum[1] = sum2 + (sum2>>24); @@ -333,42 +355,6 @@ static void logNormalizePath(char *zPath){ z[j] = 0; } -/* -** Memory map the first nByte bytes of the summary file opened with -** pSummary->fd at pSummary->aData. If the summary file is smaller than -** nByte bytes in size when this function is called, ftruncate() is -** used to expand it before it is mapped. -** -** It is assumed that an exclusive lock is held on the summary file -** by the caller (to protect the ftruncate()). -*/ -static int logSummaryMap(LogSummary *pSummary, int nByte){ - struct stat sStat; - int rc; - int fd = pSummary->fd; - void *pMap; - - assert( pSummary->aData==0 ); - - /* If the file is less than nByte bytes in size, cause it to grow. */ - rc = fstat(fd, &sStat); - if( rc!=0 ) return SQLITE_IOERR; - if( sStat.st_sizeaData = (u32 *)pMap; - pSummary->nData = nByte; - - return SQLITE_OK; -} - /* ** Unmap the log-summary mapping and close the file-descriptor. If ** the isTruncate argument is non-zero, truncate the log-summary file @@ -523,6 +509,44 @@ static void logMergesort8( } +/* +** Memory map the first nByte bytes of the summary file opened with +** pSummary->fd at pSummary->aData. If the summary file is smaller than +** nByte bytes in size when this function is called, ftruncate() is +** used to expand it before it is mapped. +** +** It is assumed that an exclusive lock is held on the summary file +** by the caller (to protect the ftruncate()). +*/ +static int logSummaryMap(LogSummary *pSummary, int nByte){ + struct stat sStat; + int rc; + int fd = pSummary->fd; + void *pMap; + + assert( pSummary->aData==0 ); + + /* If the file is less than nByte bytes in size, cause it to grow. */ + rc = fstat(fd, &sStat); + if( rc!=0 ) return SQLITE_IOERR; + if( sStat.st_sizeaData = (u32 *)pMap; + pSummary->nData = nByte/4; + + return SQLITE_OK; +} + /* ** Return the index in the LogSummary.aData array that corresponds to ** frame iFrame. The log-summary file consists of a header, followed by @@ -543,6 +567,16 @@ static int logSummaryEntry(u32 iFrame){ static void logSummaryAppend(LogSummary *pSummary, u32 iFrame, u32 iPage){ u32 iSlot = logSummaryEntry(iFrame); + if( (iSlot+128)>=pSummary->nData ){ + int nByte = pSummary->nData*4 + LOGSUMMARY_MMAP_INCREMENT; + + sqlite3_mutex_enter(pSummary->mutex); + munmap(pSummary->aData, pSummary->nData*4); + pSummary->aData = 0; + logSummaryMap(pSummary, nByte); + sqlite3_mutex_leave(pSummary->mutex); + } + /* Set the log-summary entry itself */ pSummary->aData[iSlot] = iPage; @@ -606,7 +640,7 @@ static int logSummaryRecover(LogSummary *pSummary, sqlite3_file *pFd){ ** SQLITE_MAX_PAGE_SIZE, conclude that the log file contains no valid data. */ nPgsz = sqlite3Get4byte(&aBuf[0]); - if( nPgsz&(nPgsz-1) || nPgsz>SQLITE_MAX_PAGE_SIZE ){ + if( nPgsz&(nPgsz-1) || nPgsz>SQLITE_MAX_PAGE_SIZE || nPgsz<512 ){ goto finished; } aCksum[0] = sqlite3Get4byte(&aBuf[4]); @@ -885,7 +919,7 @@ static int logSummaryInit( assert( sqlite3_mutex_held(pSummary->mutex) ); rc = logLockMutex(pSummary, LOG_WRLOCKW); if( rc!=SQLITE_OK ) return rc; - rc = logSummaryMap(pSummary, 512*1024); + rc = logSummaryMap(pSummary, LOGSUMMARY_MMAP_INCREMENT); if( rc!=SQLITE_OK ) goto out; /* Try to obtain an EXCLUSIVE lock on the dead-mans-hand region. If this diff --git a/src/pager.c b/src/pager.c index 157413e9b0..81f0968f54 100644 --- a/src/pager.c +++ b/src/pager.c @@ -3755,8 +3755,8 @@ static int pagerHasWAL(Pager *pPager, int *pExists){ } static int pagerOpenSnapshot(Pager *pPager){ - int rc; - int changed; + int rc; /* Return code */ + int changed = 0; /* True if cache must be reset */ assert( pagerUseLog(pPager) ); @@ -3764,8 +3764,8 @@ static int pagerOpenSnapshot(Pager *pPager){ if( rc==SQLITE_OK ){ int dummy; if( changed ){ - pager_reset(pPager); - assert( pPager->errCode || pPager->dbSizeValid==0 ); + pager_reset(pPager); + assert( pPager->errCode || pPager->dbSizeValid==0 ); } rc = sqlite3PagerPagecount(pPager, &dummy); } diff --git a/test/wal.test b/test/wal.test index f0268e19a2..f69c94020b 100644 --- a/test/wal.test +++ b/test/wal.test @@ -676,7 +676,6 @@ do_test wal-12.3 { execsql { INSERT INTO t2 VALUES('B', 1) } list [expr [file size test.db]/1024] [expr [file size test.db-wal]/1044] } {3 2} - do_test wal-12.4 { file copy -force test.db test2.db file copy -force test.db-wal test2.db-wal @@ -684,7 +683,6 @@ do_test wal-12.4 { execsql { SELECT * FROM t2 } db2 } {B 1} db2 close - do_test wal-12.5 { execsql { PRAGMA checkpoint; @@ -696,14 +694,47 @@ do_test wal-12.5 { SELECT * FROM t2; } } {B 2} - -do_test wal-12.4 { +do_test wal-12.6 { file copy -force test.db test2.db file copy -force test.db-wal test2.db-wal sqlite3_wal db2 test2.db execsql { SELECT * FROM t2 } db2 } {B 2} db2 close +db close + +#------------------------------------------------------------------------- +# Test large log summaries. +# +do_test wal-13.1 { + list [file exists test.db] [file exists test.db-wal] +} {1 0} +do_test wal-13.2 { + set fd [open test.db-wal w] + seek $fd [expr 200*1024*1024] + puts $fd "" + close $fd + sqlite3 db test.db + execsql { SELECT * FROM t2 } +} {B 2} +do_test wal-13.3 { + db close + file exists test.db-wal +} {0} +do_test wal-13.4 { + sqlite3 db test.db + execsql { SELECT count(*) FROM t2 } +} {1} +do_test wal-13.5 { + for {set i 0} {$i < 15} {incr i} { + execsql { INSERT INTO t2 SELECT randomblob(400), randomblob(400) FROM t2 } + } + execsql { SELECT count(*) FROM t2 } +} [expr int(pow(2, 15))] +do_test wal-13.6 { + file size test.db-wal-summary +} [expr 192*1024] + finish_test