Further work on making recovery extension compatible with the shell tool ".recover...

FossilOrigin-Name: 8df7c7d0dcd1b2fcdad00e765a9868407f0ced02ac4432ee2cdf25d83b130759

diff --git a/ext/recover/recover1.test b/ext/recover/recover1.test
index c4348baea0ec39b527688e44918449b790919a93..73832c8a1244571369ff1ab3cf052ccc09be86da 100644 (file)
--- a/ext/recover/recover1.test
+++ b/ext/recover/recover1.test
@@ -40,9 +40,11 @@ proc compare_dbs {db1 db2} {
 
 proc do_recover_test {tn} {
   forcedelete test.db2
+  forcedelete rstate.db
 
   uplevel [list do_test $tn.1 {
     set R [sqlite3_recover_init db main test.db2]
+    $R config testdb rstate.db
     $R step
     $R finish
   } {}]

diff --git a/ext/recover/sqlite3recover.c b/ext/recover/sqlite3recover.c
index 6c7828c0070a6c7c57627d97fe66f422aa9d2750..d8b151616b781df4007d8eb004a066900c60ef8c 100644 (file)
--- a/ext/recover/sqlite3recover.c
+++ b/ext/recover/sqlite3recover.c
@@ -48,6 +48,12 @@ struct RecoverTable {
   RecoverTable *pNext;
 };
 
+typedef struct RecoverBitmap RecoverBitmap;
+struct RecoverBitmap {
+  i64 nPg;                        /* Size of bitmap */
+  u32 aElem[0];                   /* Array of 32-bit bitmasks */
+};
+
 /*
 ** 
 */
@@ -59,7 +65,7 @@ struct RecoverTable {
 "      pgno, child, PRIMARY KEY(child, pgno)"                  \
 "  ) WITHOUT ROWID;"                                           \
 "  CREATE TABLE recovery.map("                                 \
-"      pgno INTEGER PRIMARY KEY, maxlen INT, intkey, root INT" \
+"      pgno INTEGER PRIMARY KEY, parent INT"                   \
 "  );"                                                         \
 "  CREATE TABLE recovery.schema("                              \
 "      type, name, tbl_name, rootpage, sql"                    \
@@ -75,11 +81,14 @@ struct sqlite3_recover {
   char *zDb;
   char *zUri;
   RecoverTable *pTblList;
+  RecoverBitmap *pUsed;           /* Used by recoverWriteLostAndFound() */
 
   int errCode;                    /* For sqlite3_recover_errcode() */
   char *zErrMsg;                  /* For sqlite3_recover_errmsg() */
 
   char *zStateDb;
+  int bLostAndFound;
+
 };
 
 /*
@@ -124,6 +133,41 @@ static int recoverError(
   return errCode;
 }
 
+
+static RecoverBitmap *recoverBitmapAlloc(sqlite3_recover *p, i64 nPg){
+  int nElem = (nPg+1+31) / 32;
+  int nByte = sizeof(RecoverBitmap) + nElem*sizeof(u32);
+  RecoverBitmap *pRet = (RecoverBitmap*)recoverMalloc(p, nByte);
+
+  if( pRet ){
+    pRet->nPg = nPg;
+  }
+  return pRet;
+}
+
+static void recoverBitmapFree(RecoverBitmap *pMap){
+  sqlite3_free(pMap);
+}
+
+static void recoverBitmapSet(RecoverBitmap *pMap, i64 iPg){
+  if( iPg<=pMap->nPg ){
+    int iElem = (iPg / 32);
+    int iBit = (iPg % 32);
+    pMap->aElem[iElem] |= (((u32)1) << iBit);
+  }
+}
+
+static int recoverBitmapQuery(RecoverBitmap *pMap, i64 iPg){
+  int ret = 1;
+  if( iPg<=pMap->nPg ){
+    int iElem = (iPg / 32);
+    int iBit = (iPg % 32);
+    ret = (pMap->aElem[iElem] & (((u32)1) << iBit)) ? 1 : 0;
+  }
+  return ret;
+}
+
+
 static int recoverDbError(sqlite3_recover *p, sqlite3 *db){
   return recoverError(p, sqlite3_errcode(db), "%s", sqlite3_errmsg(db));
 }
@@ -194,6 +238,24 @@ static int recoverExec(sqlite3_recover *p, sqlite3 *db, const char *zSql){
   return p->errCode;
 }
 
+/*
+** Execute "PRAGMA page_count" against the input database. If successful,
+** return the integer result. Or, if an error occurs, leave an error code 
+** and error message in the sqlite3_recover handle.
+*/
+static i64 recoverPageCount(sqlite3_recover *p){
+  i64 nPg = 0;
+  if( p->errCode==SQLITE_OK ){
+    sqlite3_stmt *pStmt = 0;
+    pStmt = recoverPreparePrintf(p, p->dbIn, "PRAGMA %Q.page_count", p->zDb);
+    if( pStmt && SQLITE_ROW==sqlite3_step(pStmt) ){
+      nPg = sqlite3_column_int64(pStmt, 0);
+    }
+    recoverFinalize(p, pStmt);
+  }
+  return nPg;
+}
+
 /*
 ** The implementation of a user-defined SQL function invoked by the 
 ** sqlite_dbdata and sqlite_dbptr virtual table modules to access pages
@@ -219,24 +281,23 @@ static void recoverGetPage(
 
   assert( nArg==1 );
   if( pgno==0 ){
-    pStmt = recoverPreparePrintf(p, p->dbIn, "PRAGMA %Q.page_count", p->zDb);
-  }else if( p->pGetPage==0 ){
-    pStmt = recoverPreparePrintf(
-        p, p->dbIn, "SELECT data FROM sqlite_dbpage(%Q) WHERE pgno=?", p->zDb
-    );
+    sqlite3_result_int64(pCtx, recoverPageCount(p));
+    return;
   }else{
-    pStmt = p->pGetPage;
-  }
-
-  if( pStmt ){
-    if( pgno ) sqlite3_bind_int64(pStmt, 1, pgno);
-    if( SQLITE_ROW==sqlite3_step(pStmt) ){
-      sqlite3_result_value(pCtx, sqlite3_column_value(pStmt, 0));
-    }
-    if( pgno ){
-      p->pGetPage = recoverReset(p, pStmt);
+    if( p->pGetPage==0 ){
+      pStmt = recoverPreparePrintf(
+          p, p->dbIn, "SELECT data FROM sqlite_dbpage(%Q) WHERE pgno=?", p->zDb
+          );
     }else{
-      recoverFinalize(p, pStmt);
+      pStmt = p->pGetPage;
+    }
+
+    if( pStmt ){
+      sqlite3_bind_int64(pStmt, 1, pgno);
+      if( SQLITE_ROW==sqlite3_step(pStmt) ){
+        sqlite3_result_value(pCtx, sqlite3_column_value(pStmt, 0));
+      }
+        p->pGetPage = recoverReset(p, pStmt);
     }
   }
 
@@ -311,7 +372,7 @@ static int recoverCacheSchema(sqlite3_recover *p){
     "WITH RECURSIVE pages(p) AS ("
     "  SELECT 1"
     "    UNION"
-    "  SELECT child FROM recovery.dbptr, pages WHERE pgno=p"
+    "  SELECT child FROM sqlite_dbptr('getpage()'), pages WHERE pgno=p"
     ")"
     "INSERT INTO recovery.schema SELECT"
     "  max(CASE WHEN field=0 THEN value ELSE NULL END),"
@@ -504,6 +565,77 @@ static RecoverTable *recoverFindTable(sqlite3_recover *p, u32 iRoot){
   return pRet;
 }
 
+static int recoverWriteLostAndFound(sqlite3_recover *p){
+  i64 nPg = 0;
+  RecoverBitmap *pMap = 0;
+
+  nPg = recoverPageCount(p);
+  pMap = p->pUsed = recoverBitmapAlloc(p, nPg);
+  if( pMap ){
+    sqlite3_stmt *pStmt = 0;
+    char *zField = 0;
+    const char *zSep = 0;
+    int ii;
+
+    sqlite3_stmt *pStmt = recoverPrepare(
+        p, p->dbOut,
+        "WITH RECURSIVE used(page) AS ("
+        "  SELECT rootpage FROM recovery.schema WHERE rootpage>0"
+        "    UNION"
+        "  SELECT child FROM sqlite_dbptr('getpage()'), used "
+        "    WHERE pgno=page"
+        ") "
+        "SELECT page FROM used"
+    );
+    while( pStmt && sqlite3_step(pStmt) ){
+      i64 iPg = sqlite3_column_int64(pStmt);
+      recoverBitmapSet(pMap, iPg);
+    }
+    recoverFinalize(pStmt);
+
+    pStmt = recoverPreparePrintf(
+        p, p->dbOut,
+        "WITH RECURSIVE seq(ii) AS ("
+        "  SELECT 1 UNION ALL SELECT ii+1 FROM seq WHERE ii<%lld"
+        ")"
+        "INSERT INTO recover.map(pgno) "
+        "    SELECT ii FROM seq WHERE !page_is_used(ii)"
+    );
+    sqlite3_step(pStmt);
+    recoverFinalize(pStmt);
+
+    pStmt = recoverPrepare(
+        p, p->dbOut,
+        "UPDATE recover.map SET parent = ptr.pgno "
+        "    FROM sqlite_dbptr('getpage()') WHERE recover.map.pgno=ptr.child"
+    );
+    sqlite3_step(pStmt);
+    recoverFinalize(pStmt);
+      
+    pStmt = recoverPrepare(
+        p, p->dbOut,
+        "SELECT max(field) FROM sqlite_dbdata('getpage') WHERE pgno IN ("
+        "  SELECT pgno FROM recover.map"
+        ")"
+    );
+    if( pStmt && sqlite3_step(pStmt) ){
+      nMaxField = sqlite3_column_int64(pStmt, 0);
+    }
+    recoverFinalize(pStmt);
+
+    if( nMaxField==0 || p->errCode!=SQLITE_OK ) return p->errCode;
+
+    zSep = "rootpgno INTEGER, pgno INTEGER, nfield INTEGER, id INTEGER, ";
+    for(ii=0; ii<nMaxField; ii++){
+      zField = sqlite3_mprintf("%z%sc%d", zField, zSep, ii);
+      zSep = ", ";
+      if( zField==0 ){
+        p->errCode = SQLITE_NOMEM;
+      }
+    }
+  }
+}
+
 static int recoverWriteData(sqlite3_recover *p){
   RecoverTable *pTbl;
   int nMax = 0;
@@ -522,7 +654,8 @@ static int recoverWriteData(sqlite3_recover *p){
       "WITH RECURSIVE pages(root, page) AS ("
       "  SELECT rootpage, rootpage FROM recovery.schema"
       "    UNION"
-      "   SELECT root, child FROM recovery.dbptr, pages WHERE pgno=page"
+      "   SELECT root, child FROM sqlite_dbptr('getpage()'), pages "
+      "    WHERE pgno=page"
       ") "
       "SELECT root, page, cell, field, value "
       "FROM sqlite_dbdata('getpage()') d, pages p WHERE p.page=d.pgno "
@@ -677,6 +810,10 @@ int sqlite3_recover_config(sqlite3_recover *p, int op, void *pArg){
       p->zStateDb = sqlite3_mprintf("%s", (char*)pArg);
       break;
 
+    case SQLITE_RECOVER_LOST_AND_FOUND:
+      p->bLostAndFound = (pArg ? 1 : 0);
+      break;
+
     default:
       rc = SQLITE_NOTFOUND;
       break;
@@ -695,6 +832,7 @@ static void recoverStep(sqlite3_recover *p){
     if( recoverCacheSchema(p) ) return;
     if( recoverWriteSchema1(p) ) return;
     if( recoverWriteData(p) ) return;
+    if( p->bLostAndFound && recoverWriteLostAndFound(p) ) return;
     if( recoverWriteSchema2(p) ) return;
   }
 }
@@ -722,6 +860,7 @@ int sqlite3_recover_finish(sqlite3_recover *p){
   p->pGetPage = 0;
   rc = p->errCode;
 
+  sqlite3_free(p->zStateDb);
   sqlite3_free(p);
   return rc;
 }

diff --git a/ext/recover/sqlite3recover.h b/ext/recover/sqlite3recover.h
index 401f83ea2800781a7a2b3aae66913eb50b26e674..0c83f8dea579215423f0c0eaee1148d276288b2a 100644 (file)
--- a/ext/recover/sqlite3recover.h
+++ b/ext/recover/sqlite3recover.h
@@ -38,7 +38,8 @@ sqlite3_recover *sqlite3_recover_init(
 /* Details TBD. */
 int sqlite3_recover_config(sqlite3_recover*, int op, void *pArg);
 
-#define SQLITE_RECOVER_TESTDB 789
+#define SQLITE_RECOVER_TESTDB         789
+#define SQLITE_RECOVER_LOST_AND_FOUND 790
 
 /* Step the recovery object. Return SQLITE_DONE if recovery is complete,
 ** SQLITE_OK if recovery is not complete but no error has occurred, or

diff --git a/manifest b/manifest
index 0fab53a46f3272923625d0888dd8cd046d2d6263..7f8f333f25414724a4e480cbf8bb42ee461cc843 100644 (file)
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Add\snew\sfiles\sfor\san\sextension\sto\srecover\sdata\sfrom\scorrupted\sdatabases.
-D 2022-08-31T20:45:43.730
+C Further\swork\son\smaking\srecovery\sextension\scompatible\swith\sthe\sshell\stool\s".recover"\scode.
+D 2022-09-01T21:00:39.747
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -387,10 +387,10 @@ F ext/rbu/rbuvacuum4.test a78898e438a44803eb2bc897ba3323373c9f277418e2d6d76e90f2
 F ext/rbu/sqlite3rbu.c 8737cabdfbee84bb25a7851ecef8b1312be332761238da9be6ddb10c62ad4291
 F ext/rbu/sqlite3rbu.h 1dc88ab7bd32d0f15890ea08d23476c4198d3da3056985403991f8c9cd389812
 F ext/rbu/test_rbu.c 03f6f177096a5f822d68d8e4069ad8907fe572c62ff2d19b141f59742821828a
-F ext/recover/recover1.test 861ad5140566102a8c5a3d1f936a7d6da569f34c86597c274de695f597031bac
+F ext/recover/recover1.test a848af8c82fe0731af835ff99475724f8654d2f24f772cc4e6f7ec4eb2ab71ea
 F ext/recover/recover_common.tcl 6679af7dffc858e345053a91c9b0a897595b4a13007aceffafca75304ccb137c
-F ext/recover/sqlite3recover.c 594fb45777a14f0b88b944b9fb2ccb3e85a29ef5b17522b8dac3e3944c4c27ea
-F ext/recover/sqlite3recover.h 3255f6491007e57be310aedb72a848c88f79fc14e7222bda4b8d4dab1a2450c3
+F ext/recover/sqlite3recover.c d81b430f968d838035ebf5ca168b43ae8a0bec1e7c2c950b74ec4fd5e16ca47b
+F ext/recover/sqlite3recover.h 94e277a9b314a03df46b5e94cc44b70ed6c6893d2776d09c7ea0b55c969ad854
 F ext/recover/test_recover.c 919f61df54776598b350250057fd2d3ea9cc2cef1aeac0dbb760958d26fe1afb
 F ext/repair/README.md 92f5e8aae749a4dae14f02eea8e1bb42d4db2b6ce5e83dbcdd6b1446997e0c15
 F ext/repair/checkfreelist.c e21f06995ff4efdc1622dcceaea4dcba2caa83ca2f31a1607b98a8509168a996
@@ -2004,11 +2004,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 5007742886bd20de20be3973737cf46b010359911615eb3da69cd262bd9a2435
-R 563b8320bf923831e4768bc403655fc2
-T *branch * recover-extension
-T *sym-recover-extension *
-T -sym-trunk *
+P f8298eeba01cb5b02ac4d642c06f3801331ca90edea533ea898a3283981a9e49
+R 185cdcc7bc591773e93e55f9de4bfb4c
 U dan
-Z 1c7612740eb933f84d589533d182c6df
+Z 5d96e275b1b3f6a2d23b93e2a033e9c9
 # Remove this line to create a well-formed Fossil manifest.

diff --git a/manifest.uuid b/manifest.uuid
index 16703e1b2f38e31fefc90c4e626a3de5a40f173d..5c5b60080fefa389b6d3e1bf07774f11f3ce32c2 100644 (file)
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-f8298eeba01cb5b02ac4d642c06f3801331ca90edea533ea898a3283981a9e49
\ No newline at end of file
+8df7c7d0dcd1b2fcdad00e765a9868407f0ced02ac4432ee2cdf25d83b130759
\ No newline at end of file