From: dan Date: Wed, 11 Oct 2017 18:00:34 +0000 (+0000) Subject: Add new extension "checkfreelist", which uses sqlite_dbpage to check that X-Git-Tag: version-3.21.0~9^2~12^2~4 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=36187fe8c2b90068f3262f962654ba5632b530e7;p=thirdparty%2Fsqlite.git Add new extension "checkfreelist", which uses sqlite_dbpage to check that there are no invalid entries on the database free-list. FossilOrigin-Name: 21930ef5376261d95fa325be7761d327a350d4ae6b4573c83ddb4d294dea51c4 --- diff --git a/ext/misc/checkfreelist.c b/ext/misc/checkfreelist.c new file mode 100644 index 0000000000..cec03b7999 --- /dev/null +++ b/ext/misc/checkfreelist.c @@ -0,0 +1,291 @@ +/* +** 2017 October 11 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This module exports a single C function: +** +** int sqlite3_check_freelist(sqlite3 *db, const char *zDb); +** +** This function checks the free-list in database zDb (one of "main", +** "temp", etc.) and reports any errors by invoking the sqlite3_log() +** function. It returns SQLITE_OK if successful, or an SQLite error +** code otherwise. It is not an error if the free-list is corrupted but +** no IO or OOM errors occur. +** +** If this file is compiled and loaded as an SQLite loadable extension, +** it adds an SQL function "checkfreelist" to the database handle, to +** be invoked as follows: +** +** SELECT checkfreelist(); +** +** This function performs the same checks as sqlite3_check_freelist(), +** except that it returns all error messages as a single text value, +** separated by newline characters. If the freelist is not corrupted +** in any way, an empty string is returned. +** +** To compile this module for use as an SQLite loadable extension: +** +** gcc -Os -fPIC -shared checkfreelist.c -o checkfreelist.so +*/ + +#include "sqlite3ext.h" +SQLITE_EXTENSION_INIT1 + +#ifndef SQLITE_AMALGAMATION +# include +# include +# include +# include +# define ALWAYS(X) 1 +# define NEVER(X) 0 + typedef unsigned char u8; + typedef unsigned short u16; + typedef unsigned int u32; +#define get4byte(x) ( \ + ((u32)((x)[0])<<24) + \ + ((u32)((x)[1])<<16) + \ + ((u32)((x)[2])<<8) + \ + ((u32)((x)[3])) \ +) +#endif + +/* +** Execute a single PRAGMA statement and return the integer value returned +** via output parameter (*pnOut). +** +** The SQL statement passed as the third argument should be a printf-style +** format string containing a single "%s" which will be replace by the +** value passed as the second argument. e.g. +** +** sqlGetInteger(db, "main", "PRAGMA %s.page_count", pnOut) +** +** executes "PRAGMA main.page_count" and stores the results in (*pnOut). +*/ +static int sqlGetInteger( + sqlite3 *db, /* Database handle */ + const char *zDb, /* Database name ("main", "temp" etc.) */ + const char *zFmt, /* SQL statement format */ + u32 *pnOut /* OUT: Integer value */ +){ + int rc, rc2; + char *zSql; + sqlite3_stmt *pStmt = 0; + int bOk = 0; + + zSql = sqlite3_mprintf(zFmt, zDb); + if( zSql==0 ){ + rc = SQLITE_NOMEM; + }else{ + rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0); + sqlite3_free(zSql); + } + + if( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ + *pnOut = (u32)sqlite3_column_int(pStmt, 0); + bOk = 1; + } + + rc2 = sqlite3_finalize(pStmt); + if( rc==SQLITE_OK ) rc = rc2; + if( rc==SQLITE_OK && bOk==0 ) rc = SQLITE_ERROR; + return rc; +} + +/* +** Argument zFmt must be a printf-style format string and must be +** followed by its required arguments. If argument pzOut is NULL, +** then the results of printf()ing the format string are passed to +** sqlite3_log(). Otherwise, they are appended to the string +** at (*pzOut). +*/ +static int checkFreelistError(char **pzOut, const char *zFmt, ...){ + int rc = SQLITE_OK; + char *zErr = 0; + va_list ap; + + va_start(ap, zFmt); + zErr = sqlite3_vmprintf(zFmt, ap); + if( zErr==0 ){ + rc = SQLITE_NOMEM; + }else{ + if( pzOut ){ + *pzOut = sqlite3_mprintf("%s%z%s", *pzOut?"\n":"", *pzOut, zErr); + if( *pzOut==0 ) rc = SQLITE_NOMEM; + }else{ + sqlite3_log(SQLITE_ERROR, "checkfreelist: %s", zErr); + } + sqlite3_free(zErr); + } + va_end(ap); + return rc; +} + +static int checkFreelist( + sqlite3 *db, + const char *zDb, + char **pzOut +){ + /* This query returns one row for each page on the free list. Each row has + ** two columns - the page number and page content. */ + const char *zTrunk = + "WITH freelist_trunk(i, d, n) AS (" + "SELECT 1, NULL, sqlite_readint32(data, 32) " + "FROM sqlite_dbpage(:1) WHERE pgno=1 " + "UNION ALL " + "SELECT n, data, sqlite_readint32(data) " + "FROM freelist_trunk, sqlite_dbpage(:1) WHERE pgno=n " + ")" + "SELECT i, d FROM freelist_trunk WHERE i!=1;"; + + int rc, rc2; /* Return code */ + sqlite3_stmt *pTrunk = 0; /* Compilation of zTrunk */ + u32 nPage = 0; /* Number of pages in db */ + u32 nExpected = 0; /* Expected number of free pages */ + u32 nFree = 0; /* Number of pages on free list */ + + if( zDb==0 ) zDb = "main"; + + if( (rc = sqlGetInteger(db, zDb, "PRAGMA %s.page_count", &nPage)) + || (rc = sqlGetInteger(db, zDb, "PRAGMA %s.freelist_count", &nExpected)) + ){ + return rc; + } + + rc = sqlite3_prepare_v2(db, zTrunk, -1, &pTrunk, 0); + if( rc!=SQLITE_OK ) return rc; + sqlite3_bind_text(pTrunk, 1, zDb, -1, SQLITE_STATIC); + while( rc==SQLITE_OK && sqlite3_step(pTrunk)==SQLITE_ROW ){ + u32 i; + u32 iTrunk = (u32)sqlite3_column_int(pTrunk, 0); + const u8 *aData = (const u8*)sqlite3_column_blob(pTrunk, 1); + int nData = sqlite3_column_bytes(pTrunk, 1); + u32 iNext = get4byte(&aData[0]); + u32 nLeaf = get4byte(&aData[4]); + + nFree += 1+nLeaf; + if( iNext>nPage ){ + rc = checkFreelistError(pzOut, + "trunk page %d is out of range", (int)iNext + ); + } + + for(i=0; rc==SQLITE_OK && inPage ){ + rc = checkFreelistError(pzOut, + "leaf page %d is out of range (child %d of trunk page %d)", + (int)iLeaf, (int)i, (int)iTrunk + ); + } + } + } + + if( rc==SQLITE_OK && nFree!=nExpected ){ + rc = checkFreelistError(pzOut, + "free-list count mismatch: actual=%d header=%d", + (int)nFree, (int)nExpected + ); + } + + rc2 = sqlite3_finalize(pTrunk); + if( rc==SQLITE_OK ) rc = rc2; + return rc; +} + +int sqlite3_check_freelist(sqlite3 *db, const char *zDb){ + return checkFreelist(db, zDb, 0); +} + +static void checkfreelist_function( + sqlite3_context *pCtx, + int nArg, + sqlite3_value **apArg +){ + const char *zDb; + int rc; + char *zOut = 0; + sqlite3 *db = sqlite3_context_db_handle(pCtx); + + assert( nArg==1 ); + zDb = sqlite3_value_text(apArg[0]); + rc = checkFreelist(db, zDb, &zOut); + if( rc==SQLITE_OK ){ + sqlite3_result_text(pCtx, zOut?zOut:"ok", -1, SQLITE_TRANSIENT); + }else{ + sqlite3_result_error_code(pCtx, rc); + } + + sqlite3_free(zOut); +} + +/* +** An SQL function invoked as follows: +** +** sqlite_readint32(BLOB) -- Decode 32-bit integer from start of blob +*/ +static void readint_function( + sqlite3_context *pCtx, + int nArg, + sqlite3_value **apArg +){ + const u8 *zBlob; + int nBlob; + int iOff = 0; + u32 iRet = 0; + + if( nArg!=1 && nArg!=2 ){ + sqlite3_result_error( + pCtx, "wrong number of arguments to function sqlite_readint32()", -1 + ); + return; + } + if( nArg==2 ){ + iOff = sqlite3_value_int(apArg[1]); + } + + zBlob = sqlite3_value_blob(apArg[0]); + nBlob = sqlite3_value_bytes(apArg[0]); + + if( nBlob>=(iOff+4) ){ + iRet = get4byte(&zBlob[iOff]); + } + + sqlite3_result_int64(pCtx, (sqlite3_int64)iRet); +} + +/* +** Register the SQL functions. +*/ +static int cflRegister(sqlite3 *db){ + int rc = sqlite3_create_function( + db, "sqlite_readint32", -1, SQLITE_UTF8, 0, readint_function, 0, 0 + ); + if( rc!=SQLITE_OK ) return rc; + rc = sqlite3_create_function( + db, "checkfreelist", 1, SQLITE_UTF8, 0, checkfreelist_function, 0, 0 + ); + return rc; +} + +/* +** Extension load function. +*/ +#ifdef _WIN32 +__declspec(dllexport) +#endif +int sqlite3_checkfreelist_init( + sqlite3 *db, + char **pzErrMsg, + const sqlite3_api_routines *pApi +){ + SQLITE_EXTENSION_INIT2(pApi); + return cflRegister(db); +} diff --git a/manifest b/manifest index 92b51678a2..d4e28e2d73 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Convert\sthe\simplementation\sof\sthe\s".dbstat"\sdot-command\sof\sthe\scommand-line\nshell\sto\suse\sthe\ssqlite_dbpage\stable. -D 2017-10-11T17:51:08.392 +C Add\snew\sextension\s"checkfreelist",\swhich\suses\ssqlite_dbpage\sto\scheck\sthat\nthere\sare\sno\sinvalid\sentries\son\sthe\sdatabase\sfree-list. +D 2017-10-11T18:00:34.689 F Makefile.in 05d02ce8606a9e46cd413d0bb46873fe597e5e41f52c4110241c11e60adff018 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 148d7cd36e556f5c257232cd93c71a1dd32c880d964c7d714990d677cd094589 @@ -259,6 +259,7 @@ F ext/misc/README.md 8e008c8d2b02e09096b31dfba033253ac27c6c06a18aa5826e299fa7601 F ext/misc/amatch.c 6db4607cb17c54b853a2d7c7c36046d004853f65b9b733e6f019d543d5dfae87 F ext/misc/anycollseq.c 5ffdfde9829eeac52219136ad6aa7cd9a4edb3b15f4f2532de52f4a22525eddb F ext/misc/carray.c ed96c218ea940b85c9a274c4d9c59fe9491c299147a38a8bba537687bd6c6005 +F ext/misc/checkfreelist.c 043fdcc710f4147ff1deaf1bd6ea0a1c3eccb665ddd30d5623823a8eb4817eea w ext/misc/freelistchecker.c F ext/misc/closure.c 0d2a038df8fbae7f19de42e7c7d71f2e4dc88704 F ext/misc/completion.c 52c3f01523e3e387eb321b4739a89d1fe47cbe6025aa1f2d8d3685e9e365df0f F ext/misc/compress.c 122faa92d25033d6c3f07c39231de074ab3d2e83 @@ -647,6 +648,7 @@ F test/capi3e.test 3d49c01ef2a1a55f41d73cba2b23b5059ec460fe F test/cast.test 4c275cbdc8202d6f9c54a3596701719868ac7dc3 F test/cffault.test 9d6b20606afe712374952eec4f8fd74b1a8097ef F test/check.test 33a698e8c63613449d85d624a38ef669bf20331daabebe3891c9405dd6df463a +F test/checkfreelist.test 6324b0a279eb101d698b31c12a65767b25f9b5c66d0d424943ae002e01f0de2f F test/close.test 799ea4599d2f5704b0a30f477d17c2c760d8523fa5d0c8be4a7df2a8cad787d8 F test/closure01.test b1703ba40639cfc9b295cf478d70739415eec6a4 F test/coalesce.test cee0dccb9fbd2d494b77234bccf9dc6c6786eb91 @@ -1658,7 +1660,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 01bf856c424c20b464f26973720bf5dcd3e89509c5b02c3625d4828f0385d3db -R 32cd47bb05391f34b12cc1216f3e5e53 -U drh -Z ef5ad8ed0ad3bdbe43b1de17eb74fe58 +P 497409e167c7c025fbddc319b4fa9a8b965f70d05ac88c060dee469f70321388 +R 7024a507e1ac7d6985bebba168f4b31f +U dan +Z 54a66c878cf2157bd3590abc0ee2a612 diff --git a/manifest.uuid b/manifest.uuid index 12bf79e9d8..e6ec495019 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -497409e167c7c025fbddc319b4fa9a8b965f70d05ac88c060dee469f70321388 \ No newline at end of file +21930ef5376261d95fa325be7761d327a350d4ae6b4573c83ddb4d294dea51c4 \ No newline at end of file diff --git a/test/checkfreelist.test b/test/checkfreelist.test new file mode 100644 index 0000000000..f16d8b57bb --- /dev/null +++ b/test/checkfreelist.test @@ -0,0 +1,114 @@ +# 2017-10-11 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for SQLite library. The +# focus of this file is testing the checkfreelist extension. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix checkfreelist + +ifcapable !vtab||!compound { + finish_test + return +} + +if {[file exists ../checkfreelist.so]==0} { + finish_test + return +} + +do_execsql_test 1.0 { + CREATE TABLE t1(a, b); +} + +db enable_load_extension 1 +do_execsql_test 1.1 { + SELECT load_extension('../checkfreelist.so'); +} {{}} + +do_execsql_test 1.2 { SELECT checkfreelist('main') } {ok} +do_execsql_test 1.3 { + WITH s(i) AS ( + SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<10000 + ) + INSERT INTO t1 SELECT randomblob(400), randomblob(400) FROM s; + DELETE FROM t1 WHERE rowid%3; + PRAGMA freelist_count; +} {6726} + +do_execsql_test 1.4 { SELECT checkfreelist('main') } {ok} +do_execsql_test 1.5 { + WITH freelist_trunk(i, d, n) AS ( + SELECT 1, NULL, sqlite_readint32(data, 32) FROM sqlite_dbpage WHERE pgno=1 + UNION ALL + SELECT n, data, sqlite_readint32(data) + FROM freelist_trunk, sqlite_dbpage WHERE pgno=n + ) + SELECT i FROM freelist_trunk WHERE i!=1; +} { + 10010 9716 9344 8970 8596 8223 7848 7475 7103 6728 6355 5983 5609 5235 + 4861 4488 4113 3741 3368 2993 2620 2248 1873 1500 1126 753 378 5 +} + +do_execsql_test 1.6 { SELECT checkfreelist('main') } {ok} + +proc set_int {blob idx newval} { + binary scan $blob I* ints + lset ints $idx $newval + binary format I* $ints +} +db func set_int set_int + +proc get_int {blob idx} { + binary scan $blob I* ints + lindex $ints $idx +} +db func get_int get_int + +do_execsql_test 1.7 { + BEGIN; + UPDATE sqlite_dbpage + SET data = set_int(data, 1, get_int(data, 1)-1) + WHERE pgno=4861; + SELECT checkfreelist('main'); + ROLLBACK; +} {{free-list count mismatch: actual=6725 header=6726}} + +do_execsql_test 1.8 { + BEGIN; + UPDATE sqlite_dbpage + SET data = set_int(data, 5, (SELECT * FROM pragma_page_count)+1) + WHERE pgno=4861; + SELECT checkfreelist('main'); + ROLLBACK; +} {{leaf page 10093 is out of range (child 3 of trunk page 4861)}} + +do_execsql_test 1.9 { + BEGIN; + UPDATE sqlite_dbpage + SET data = set_int(data, 5, 0) + WHERE pgno=4861; + SELECT checkfreelist('main'); + ROLLBACK; +} {{leaf page 0 is out of range (child 3 of trunk page 4861)}} + +do_execsql_test 1.10 { + BEGIN; + UPDATE sqlite_dbpage + SET data = set_int(data, get_int(data, 1)+1, 0) + WHERE pgno=5; + SELECT checkfreelist('main'); + ROLLBACK; +} {{leaf page 0 is out of range (child 247 of trunk page 5)}} + +finish_test +