From: dan Date: Mon, 22 Apr 2019 20:52:12 +0000 (+0000) Subject: Enhance the ".recover" command. Fix a problem with overflow pages in dbdata.c. X-Git-Tag: version-3.29.0~176^2~13 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=b40af49d735f7ea8111bf37dab5c3f041e549290;p=thirdparty%2Fsqlite.git Enhance the ".recover" command. Fix a problem with overflow pages in dbdata.c. FossilOrigin-Name: f193ca587f9e4f925f4f2343b0b07053bd6f93dd87fc6f8f41cf4479e90cf562 --- diff --git a/ext/misc/dbdata.c b/ext/misc/dbdata.c index aae862fc7f..7662417ae5 100644 --- a/ext/misc/dbdata.c +++ b/ext/misc/dbdata.c @@ -506,6 +506,7 @@ static int dbdataNext(sqlite3_vtab_cursor *pCursor){ memcpy(&pCsr->pRec[nPayload-nRem], &aOvfl[4], nCopy); nRem -= nCopy; + pgnoOvfl = get_uint32(aOvfl); sqlite3_free(aOvfl); } } diff --git a/manifest b/manifest index c103da1624..b830ce2b1e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\s".recovery"\scommand\sto\sthe\sshell\stool.\sFor\srecovering\sthe\smaximum\samount\sdata\sfrom\scorrupt\sdatabases.\sStill\sneeds\swork. -D 2019-04-20T20:57:28.136 +C Enhance\sthe\s".recover"\scommand.\sFix\sa\sproblem\swith\soverflow\spages\sin\sdbdata.c. +D 2019-04-22T20:52:12.850 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -284,7 +284,7 @@ F ext/misc/closure.c dbfd8543b2a017ae6b1a5843986b22ddf99ff126ec9634a2f4047cd14c8 F ext/misc/completion.c cec672d40604075bb341a7f11ac48393efdcd90a979269b8fe7977ea62d0547f F ext/misc/compress.c dd4f8a6d0baccff3c694757db5b430f3bbd821d8686d1fc24df55cf9f035b189 F ext/misc/csv.c 7f047aeb68f5802e7ce6639292095d622a488bb43526ed04810e0649faa71ceb -F ext/misc/dbdata.c 8f74f25565e1f57942f75134ef94ab73058849bed87d92be619be7d3b8a7d054 +F ext/misc/dbdata.c 6a0ccc33e8c5ef3b6164eec7839cb4dff44e27bee0717f1660dbb8ee77ee4cf8 F ext/misc/dbdump.c baf6e37447c9d6968417b1cd34cbedb0b0ab3f91b5329501d8a8d5be3287c336 F ext/misc/eval.c 4b4757592d00fd32e44c7a067e6a0e4839c81a4d57abc4131ee7806d1be3104e F ext/misc/explain.c d5c12962d79913ef774b297006872af1fccda388f61a11d37758f9179a09551f @@ -520,7 +520,7 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 F src/resolve.c 567888ee3faec14dae06519b4306201771058364a37560186a3e0e755ebc4cb8 F src/rowset.c d977b011993aaea002cab3e0bb2ce50cf346000dff94e944d547b989f4b1fe93 F src/select.c 9263f5c30dd44c7ac2eb29f40a7ec64322a96885b71c00de6bc30b756c2e1c49 -F src/shell.c.in 3646e448cc207fa3118266c2a23f177306b068e42aae6fee64323ffb13307680 +F src/shell.c.in 7dd0babc42bc136b8dd070152981ee2325b787571f58326e68e6ad35d764aa33 F src/sqlite.h.in 38390767acc1914d58930e03149595ee4710afa4e3c43ab6c3a8aea3f1a6b8cd F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 9ecc93b8493bd20c0c07d52e2ac0ed8bab9b549c7f7955b59869597b650dd8b5 @@ -787,7 +787,7 @@ F test/cursorhint2.test 6f3aa9cb19e7418967a10ec6905209bcbb5968054da855fc36c8beee F test/dataversion1.test 6e5e86ac681f0782e766ebcb56c019ae001522d114e0e111e5ebf68ccf2a7bb8 F test/date.test 9b73bbeb1b82d9c1f44dec5cf563bf7da58d2373 F test/date2.test 74c234bece1b016e94dd4ef9c8cc7a199a8806c0e2291cab7ba64bace6350b10 -F test/dbdata.test 573fa3347744863e47d011e4e8e9b87c6795ba92a759bf5d5c68da975900ddd4 +F test/dbdata.test c8d97bafd1b2efb1e445871c4641208dcd91e686d2dfbb6463d83934adbd1ac5 F test/dbfuzz.c 73047c920d6210e5912c87cdffd9a1c281d4252e F test/dbfuzz001.test e32d14465f1c77712896fda6a1ccc0f037b481c191c1696a9c44f6c9e4964faf F test/dbfuzz2-seed1.db e6225c6f3d7b63f9c5b6867146a5f329d997ab105bee64644dc2b3a2f2aebaee @@ -1820,7 +1820,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 3213a15f2133afbb0a4fec3b8f6e0eeca8c0befafd6658c41074e84f589d5d32 -R 9a7e323fe9ebfe9eb0759e1c70136795 +P 7461d2e120f2149315ddac2676d51d7445bcdb8e97543effd9c30603517ef9da +R 14050e9b3d7e7a784dc8a127e6275222 U dan -Z 100e3ccaddf1d19d0bacd74d773a93b7 +Z abcbf2aebf4cbfcbf65f71e558364487 diff --git a/manifest.uuid b/manifest.uuid index 0b854fa40a..65b5fdd250 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -7461d2e120f2149315ddac2676d51d7445bcdb8e97543effd9c30603517ef9da \ No newline at end of file +f193ca587f9e4f925f4f2343b0b07053bd6f93dd87fc6f8f41cf4479e90cf562 \ No newline at end of file diff --git a/src/shell.c.in b/src/shell.c.in index c38bfb64b9..abba5b0168 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -6073,9 +6073,9 @@ typedef struct RecoverTable RecoverTable; struct RecoverTable { char *zName; /* Name of table */ char *zQuoted; /* Quoted version of zName */ - char *zCreate; /* SQL to create table in default schema */ int nCol; /* Number of columns in table */ char **azlCol; /* Array of column lists */ + int iPk; }; /* @@ -6085,7 +6085,6 @@ static void recoverFreeTable(RecoverTable *pTab){ if( pTab ){ sqlite3_free(pTab->zName); sqlite3_free(pTab->zQuoted); - sqlite3_free(pTab->zCreate); if( pTab->azlCol ){ int i; for(i=0; inCol; i++){ @@ -6097,17 +6096,141 @@ static void recoverFreeTable(RecoverTable *pTab){ } } +static void recoverOldTable( + int *pRc, /* IN/OUT: Error code */ + RecoverTable *pTab, + const char *zName, /* Name of table */ + const char *zSql, /* CREATE TABLE statement */ + int bIntkey, + int nCol +){ + sqlite3 *dbtmp = 0; /* sqlite3 handle for testing CREATE TABLE */ + int rc = *pRc; + + if( rc==SQLITE_OK ){ + int nSqlCol = 0; + int bSqlIntkey = 0; + sqlite3_stmt *pStmt = 0; + + rc = sqlite3_open("", &dbtmp); + if( rc==SQLITE_OK ){ + rc = sqlite3_exec(dbtmp, zSql, 0, 0, 0); + if( rc==SQLITE_ERROR ){ + rc = SQLITE_OK; + goto finished; + } + } + shellPreparePrintf(dbtmp, &rc, &pStmt, + "SELECT count(*) FROM pragma_table_info(%Q)", zName + ); + if( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ + nSqlCol = sqlite3_column_int(pStmt, 0); + } + shellFinalize(&rc, pStmt); + + if( rc!=SQLITE_OK || nSqlColiPk = sqlite3_column_int(pPkFinder, 0); + zPk = (const char*)sqlite3_column_text(pPkFinder, 1); + } + + pTab->zName = shellMPrintf(&rc, "%s", zName); + pTab->zQuoted = shellMPrintf(&rc, "%Q", pTab->zName); + pTab->azlCol = (char**)shellMalloc(&rc, sizeof(char*) * nSqlCol); + pTab->nCol = nSqlCol; + + if( nSqlCol==1 && pTab->iPk==0 ){ + pTab->azlCol[0] = shellMPrintf(&rc, "%Q", zPk); + }else{ + shellPreparePrintf(dbtmp, &rc, &pStmt, + "SELECT -1+row_number() OVER (ORDER BY cid)," + " %Q||%Q||group_concat(name, ', ') FILTER (WHERE cid!=%d) " + " OVER (ORDER BY cid) " + "FROM pragma_table_info(%Q)", + (bIntkey ? zPk : ""), (bIntkey ? ", " : ""), + pTab->iPk, zName + ); + while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ + int idx = sqlite3_column_int(pStmt, 0); + const char *zText = (const char*)sqlite3_column_text(pStmt, 1); + pTab->azlCol[idx] = shellMPrintf(&rc, "%s", zText); + } + shellFinalize(&rc, pStmt); + } + shellFinalize(&rc, pPkFinder); + } + } + + finished: + sqlite3_close(dbtmp); + *pRc = rc; +} + static RecoverTable *recoverNewTable( ShellState *pState, int *pRc, int iRoot, + int bIntkey, int nCol ){ + sqlite3_stmt *pStmt = 0; RecoverTable *pRet = 0; + int bNoop = 0; + const char *zSql = 0; + const char *zName = 0; pRet = (RecoverTable*)shellMalloc(pRc, sizeof(RecoverTable)); - if( pRet ){ + if( pRet ) pRet->iPk = -2; + + /* Search the recovered schema for an object with root page iRoot. */ + shellPreparePrintf(pState->db, pRc, &pStmt, + "SELECT type, name, sql FROM recovery.schema WHERE rootpage=%d", iRoot + ); + while( *pRc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ + const char *zType = (const char*)sqlite3_column_text(pStmt, 0); + if( bIntkey==0 && sqlite3_stricmp(zType, "index")==0 ){ + bNoop = 1; + break; + } + if( sqlite3_stricmp(zType, "table")==0 ){ + zName = (const char*)sqlite3_column_text(pStmt, 1); + zSql = (const char*)sqlite3_column_text(pStmt, 2); + recoverOldTable(pRc, pRet, zName, zSql, bIntkey, nCol); + break; + } + } + shellFinalize(pRc, pStmt); + if( bNoop ){ + sqlite3_free(pRet); + return 0; + } + + if( pRet && pRet->zName==0 ){ sqlite3_stmt *pStmt = 0; + pRet->zName = shellMPrintf(pRc, "orphan_%d_%d", nCol, iRoot); pRet->zQuoted = shellMPrintf(pRc, "%Q", pRet->zName); pRet->azlCol = (char**)shellMalloc(pRc, sizeof(char*) * nCol); @@ -6117,8 +6240,8 @@ static RecoverTable *recoverNewTable( "WITH s(i) AS (" " SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<%d" ")" - "SELECT i-1, group_concat('c' || i, ', ') OVER (ORDER BY i) FROM s", - nCol + "SELECT i-1, %Q || group_concat('c' || i, ', ') OVER (ORDER BY i) FROM s", + nCol, (bIntkey ? "id, " : "") ); while( *pRc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ int idx = sqlite3_column_int(pStmt, 0); @@ -6127,9 +6250,15 @@ static RecoverTable *recoverNewTable( } shellFinalize(pRc, pStmt); - pRet->zCreate = shellMPrintf(pRc, "CREATE TABLE %Q (id, %s)", + if( *pRc==SQLITE_OK ){ + char *zCreate = shellMPrintf(pRc, "CREATE TABLE %Q (%s)", pRet->zName, pRet->azlCol[nCol-1] - ); + ); + if( zCreate ){ + raw_printf(pState->out, "%s;\n", zCreate); + sqlite3_free(zCreate); + } + } } if( *pRc!=SQLITE_OK ){ @@ -6146,7 +6275,6 @@ static RecoverTable *recoverNewTable( ** on stream pState->out. */ static int recoverDatabaseCmd(ShellState *pState){ - const char *zSql; int rc = SQLITE_OK; sqlite3_stmt *pLoop = 0; /* Loop through all root pages */ @@ -6173,7 +6301,9 @@ static int recoverDatabaseCmd(ShellState *pState){ /* Create the "map" table that will (eventually) contain instructions ** for dealing with each page in the db that contains one or more ** records. */ - "CREATE TABLE recovery.map(pgno INTEGER PRIMARY KEY, maxlen INT, root INT);" + "CREATE TABLE recovery.map(" + "pgno INTEGER PRIMARY KEY, maxlen INT, intkey, root INT" + ");" /* Populate table [map]. If there are circular loops of pages in the ** database, the following adds all pages in such a loop to the map @@ -6182,10 +6312,12 @@ static int recoverDatabaseCmd(ShellState *pState){ " SELECT page_count, max(field+1) " " FROM pragma_page_count, sqlite_dbdata WHERE pgno=page_count" " UNION ALL" - " SELECT * FROM (SELECT i-1, max(field+1)" - " FROM pages, sqlite_dbdata WHERE pgno=i-1 AND i>=2)" + " SELECT i-1, (" + " SELECT max(field+1) FROM sqlite_dbdata WHERE pgno=i-1" + " ) FROM pages WHERE i>=2" ")" - "INSERT INTO recovery.map(pgno, maxlen, root) SELECT i, maxlen, (" + "INSERT INTO recovery.map(pgno, maxlen, intkey, root) " + " SELECT i, maxlen, NULL, (" " WITH p(orig, pgno, parent) AS (" " SELECT 0, i, (SELECT pgno FROM recovery.dbptr WHERE child=i)" " UNION ALL" @@ -6195,6 +6327,9 @@ static int recoverDatabaseCmd(ShellState *pState){ " SELECT pgno FROM p WHERE (parent IS NULL OR pgno = orig)" ") " "FROM pages WHERE maxlen > 0;" + "UPDATE recovery.map AS o SET intkey = (" + " SELECT substr(data, 1, 1)==X'0D' FROM sqlite_dbpage WHERE pgno=o.pgno" + ");" /* Extract data from page 1 and any linked pages into table ** recovery.schema. With the same schema as an sqlite_master table. */ @@ -6221,36 +6356,53 @@ static int recoverDatabaseCmd(ShellState *pState){ return rc; #endif + /* Open a transaction, then print out all non-virtual, non-"sqlite_%" + ** CREATE TABLE statements that extracted from the existing schema. */ + if( rc==SQLITE_OK ){ + sqlite3_stmt *pStmt = 0; + raw_printf(pState->out, "BEGIN;\n"); + shellPrepare(pState->db, &rc, + "SELECT sql FROM recovery.schema " + "WHERE type='table' " + " AND length(sql)>6" + " AND sql LIKE 'create table%'" + " AND name NOT LIKE 'sqliteX_%' ESCAPE 'X'", &pStmt + ); + while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ + const char *zCreateTable = (const char*)sqlite3_column_text(pStmt, 0); + raw_printf(pState->out, "%s;\n", zCreateTable); + } + shellFinalize(&rc, pStmt); + } + /* Loop through each root page. */ - zSql = "SELECT root,max(maxlen) FROM recovery.map WHERE root>1 GROUP BY root"; - shellPrepare(pState->db, &rc, zSql, &pLoop); + shellPrepare(pState->db, &rc, + "SELECT root, intkey, max(maxlen) FROM recovery.map" + " WHERE root>1 GROUP BY root, intkey", &pLoop + ); while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pLoop) ){ int iRoot = sqlite3_column_int(pLoop, 0); - int nCol = sqlite3_column_int(pLoop, 1); + int bIntkey = sqlite3_column_int(pLoop, 1); + int nCol = sqlite3_column_int(pLoop, 2); RecoverTable *pTab; - pTab = recoverNewTable(pState, &rc, iRoot, nCol); + pTab = recoverNewTable(pState, &rc, iRoot, bIntkey, nCol); if( pTab ){ sqlite3_stmt *pData = 0; - raw_printf(pState->out, "%s;\n", pTab->zCreate); shellPreparePrintf(pState->db, &rc, &pData, "SELECT max(field), group_concat(quote(value), ', ') " "FROM sqlite_dbdata WHERE pgno IN (" " SELECT pgno FROM recovery.map WHERE root=%d" ")" - "GROUP BY pgno, cell;", iRoot + " AND field!=%d " + "GROUP BY pgno, cell;", iRoot, pTab->iPk ); while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pData) ){ int iMax = sqlite3_column_int(pData, 0); const char *zVal = (const char*)sqlite3_column_text(pData, 1); - if( iMax+1==pTab->nCol ){ - raw_printf(pState->out, "INSERT INTO %s VALUES( %s );\n", - pTab->zQuoted, zVal); - }else{ - raw_printf(pState->out, "INSERT INTO %s(%s) VALUES( %s );\n", - pTab->zQuoted, pTab->azlCol[iMax], zVal - ); - } + raw_printf(pState->out, "INSERT INTO %s(%s) VALUES( %s );\n", + pTab->zQuoted, pTab->azlCol[iMax>0?iMax:0], zVal + ); } shellFinalize(&rc, pData); } @@ -6258,6 +6410,9 @@ static int recoverDatabaseCmd(ShellState *pState){ } shellFinalize(&rc, pLoop); + if( rc==SQLITE_OK ){ + raw_printf(pState->out, "COMMIT;\n"); + } sqlite3_exec(pState->db, "DETACH recovery", 0, 0, 0); return rc; } diff --git a/test/dbdata.test b/test/dbdata.test index 5a0df5dc86..391b278025 100644 --- a/test/dbdata.test +++ b/test/dbdata.test @@ -67,6 +67,16 @@ do_execsql_test 1.3 { SELECT value FROM sqlite_dbdata WHERE pgno=2 AND cell=2 AND field=1; } $big +do_execsql_test 1.4 { + DELETE FROM t1; + INSERT INTO t1 VALUES(NULL, randomblob(5050)); +} +do_test 1.5 { + execsql { + SELECT quote(value) FROM sqlite_dbdata WHERE pgno=2 AND cell=0 AND field=1; + } +} [db one {SELECT quote(b) FROM t1}] + #------------------------------------------------------------------------- reset_db db enable_load_extension 1