From: drh <> Date: Thu, 12 Sep 2024 12:04:53 +0000 (+0000) Subject: Improved debugging output. X-Git-Tag: version-3.47.0~118^2~19 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=f1fb5ce10f4d320fc9b012df8e6b7119f508ef42;p=thirdparty%2Fsqlite.git Improved debugging output. FossilOrigin-Name: 80461e0d724963aaf2646005298f1194c5f1c4c9ae41c1085d4d137ed485bd9f --- diff --git a/Makefile.in b/Makefile.in index 0cfae780cc..8a070ebda8 100644 --- a/Makefile.in +++ b/Makefile.in @@ -706,6 +706,7 @@ RSYNC_SRC = \ sqlite3.c RSYNC_OPT = \ + -DSQLITE_ENABLE_DBPAGE_VTAB \ -DSQLITE_THREADSAFE=0 \ -DSQLITE_OMIT_LOAD_EXTENSION \ -DSQLITE_OMIT_DEPRECATED diff --git a/Makefile.msc b/Makefile.msc index 461859d1f1..40f8dc0f82 100644 --- a/Makefile.msc +++ b/Makefile.msc @@ -1873,6 +1873,7 @@ RSYNC_SRC = \ $(SQLITE3C) RSYNC_OPT = \ + -DSQLITE_ENABLE_DBPAGE_VTAB \ -DSQLITE_THREADSAFE=0 \ -DSQLITE_OMIT_LOAD_EXTENSION \ -DSQLITE_OMIT_DEPRECATED diff --git a/main.mk b/main.mk index 7414e7488e..3ab5047b5a 100644 --- a/main.mk +++ b/main.mk @@ -574,6 +574,7 @@ RSYNC_SRC = \ sqlite3.c RSYNC_OPT = \ + -DSQLITE_ENABLE_DBPAGE_VTAB \ -DSQLITE_THREADSAFE=0 \ -DSQLITE_OMIT_LOAD_EXTENSION \ -DSQLITE_OMIT_DEPRECATED diff --git a/manifest b/manifest index c99d1cd05f..bb7f3a7267 100644 --- a/manifest +++ b/manifest @@ -1,11 +1,11 @@ -C Progress\son\sthe\ssqlite3-rsync\sutility.\s\sThis\sis\san\sincremental\scheck-in.\s\sIt\ndoes\scompile,\sbut\sit\sdoes\snot\swork. -D 2024-09-11T17:02:44.010 +C Improved\sdebugging\soutput. +D 2024-09-12T12:04:53.132 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 -F Makefile.in 54e80b016b0e58db383c0f08d340ef795b8b709ffb6f53a51a8ba7bf0c5e288f +F Makefile.in 167583cd37df435b3cd7e87de7a04247d341db83ffd363bd0240ddcc776c55d6 F Makefile.linux-gcc f3842a0b1efbfbb74ac0ef60e56b301836d05b4d867d014f714fa750048f1ab6 -F Makefile.msc a86e0f3fe5f807daa82d44b5056e3dbc311e569bd4748646a776a994124ec58b +F Makefile.msc 4af481bae608f19f869f7709d93ba04876480844044e14ce97f89e5ee2e51759 F README.md c3c0f19532ce28f6297a71870f3c7b424729f0e6d9ab889616d3587dd2332159 F VERSION 0db40f92c04378404eb45bff93e9e42c148c7e54fd3da99469ed21e22411f5a6 F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50 @@ -687,7 +687,7 @@ F ext/wasm/wasmfs.make 8a4955882aaa0783b3f60a9484a1f0f3d8b6f775c0fcd17c082f31966 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 F magic.txt 5ade0bc977aa135e79e3faaea894d5671b26107cc91e70783aa7dc83f22f3ba0 -F main.mk 9ffe4a14bdb4a0b856217a5465ca6b1ef4bef66a3c45e2da3fdb6d9bfc8d583e +F main.mk f6424b8011c62b707fca5153a71a5d5a373f36ea6458908cc8858f7c5118c9f1 F mptest/config01.test 3c6adcbc50b991866855f1977ff172eb6d901271 F mptest/config02.test 4415dfe36c48785f751e16e32c20b077c28ae504 F mptest/crash01.test 61e61469e257df0850df4293d7d4d6c2af301421 @@ -2174,7 +2174,7 @@ F tool/speedtest8inst1.c 7ce07da76b5e745783e703a834417d725b7d45fd F tool/spellsift.tcl 52b4b04dc4333c7ab024f09d9d66ed6b6f7c6eb00b38497a09f338fa55d40618 x F tool/split-sqlite3c.tcl 5aa60643afca558bc732b1444ae81a522326f91e1dc5665b369c54f09e20de60 F tool/sqldiff.c 847fc8fcfddf5ce4797b7394cad6372f2f5dc17d8186e2ef8fb44d50fae4f44a -F tool/sqlite3-rsync.c eb75a24e3a47fe7b5a4d5cbd2eadd4f4b6b6d6038ec7c94eb98a879ccd1bf8c5 +F tool/sqlite3-rsync.c d9f8803f79c66dbc213761a345e24ae22c7de14fd334150086519c611ff1a705 F tool/sqlite3_analyzer.c.in 8da2b08f56eeac331a715036cf707cc20f879f231362be0c22efd682e2b89b4f F tool/sqltclsh.c.in 1bcc2e9da58fadf17b0bf6a50e68c1159e602ce057210b655d50bad5aaaef898 F tool/sqltclsh.tcl 862f4cf1418df5e1315b5db3b5ebe88969e2a784525af5fbf9596592f14ed848 @@ -2213,8 +2213,8 @@ F vsixtest/vsixtest.tcl 6195aba1f12a5e10efc2b8c0009532167be5e301abe5b31385638080 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 9a1a95f523a96303aad57e2422c2b51ea7e125f5490f32f7a2929d49b6c69ef8 -R f1d49deccd29ff432816ec80f04a6271 +P fa06977b6db7fa745720561ec0b10570cf7e71598dc7a7c5ee650640e5bdf6f5 +R 7d94be6de6803646ba12e0250789553f U drh -Z f5a21c2123f254d15ff8790517b1651d +Z 2b03e42bffc925c056d94789b2d4ed35 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 6113137f18..af581bf8ac 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -fa06977b6db7fa745720561ec0b10570cf7e71598dc7a7c5ee650640e5bdf6f5 +80461e0d724963aaf2646005298f1194c5f1c4c9ae41c1085d4d137ed485bd9f diff --git a/tool/sqlite3-rsync.c b/tool/sqlite3-rsync.c index 68bfe5d668..276b9af90f 100644 --- a/tool/sqlite3-rsync.c +++ b/tool/sqlite3-rsync.c @@ -29,6 +29,8 @@ static const char zUsage[] = "copy of ORIGIN\n" ; +typedef unsigned char u8; + /* Context for the run */ typedef struct SQLiteRsync SQLiteRsync; struct SQLiteRsync { @@ -36,13 +38,17 @@ struct SQLiteRsync { const char *zReplica; /* Name of the replica */ FILE *pOut; /* Transmit to the other side */ FILE *pIn; /* Receive from the other side */ - sqlite3_uint64 nOut; /* Bytes transmitted */ - sqlite3_uint64 nIn; /* Bytes received */ sqlite3 *db; /* Database connection */ int nErr; /* Number of errors encountered */ - int eVerbose; /* Bigger for more output. 0 means none. */ - int bCommCheck; /* True to debug the communication protocol */ - int isRemote; /* On the remote side of a connection */ + u8 eVerbose; /* Bigger for more output. 0 means none. */ + u8 bCommCheck; /* True to debug the communication protocol */ + u8 isRemote; /* On the remote side of a connection */ + sqlite3_uint64 nOut; /* Bytes transmitted */ + sqlite3_uint64 nIn; /* Bytes received */ + unsigned int nPage; /* Total number of pages in the database */ + unsigned int szPage; /* Database page size */ + unsigned int nHashSent; /* Hashes sent (replica to origin) */ + unsigned int nPageSent; /* Page contents sent (origin to replica) */ }; @@ -504,6 +510,7 @@ static int readUint32(SQLiteRsync *p, unsigned int *pU){ unsigned char buf[4]; if( fread(buf, sizeof(buf), 1, p->pIn)==1 ){ *pU = (buf[0]<<24) | (buf[1]<<16) | (buf[2]<<8) | buf[3]; + p->nIn += 4; return 0; }else{ p->nErr++; @@ -527,9 +534,45 @@ static int writeUint32(SQLiteRsync *p, unsigned int x){ p->nErr++; return 1; } + p->nOut += 4; return 0; } +/* Read a single byte from the wire. +*/ +int readByte(SQLiteRsync *p){ + int c = fgetc(p->pIn); + if( c!=EOF ) p->nIn++; + return c; +} + +/* Write a single byte into the wire. +*/ +void writeByte(SQLiteRsync *p, int c){ + fputc(c, p->pOut); + p->nOut++; +} + +/* Read an array of bytes from the wire. +*/ +void readBytes(SQLiteRsync *p, int nByte, void *pData){ + if( fread(pData, 1, nByte, p->pIn)==nByte ){ + p->nIn += nByte; + }else{ + p->nErr++; + } +} + +/* Write an array of bytes onto the wire. +*/ +void writeBytes(SQLiteRsync *p, int nByte, const void *pData){ + if( fwrite(pData, 1, nByte, p->pOut)==nByte ){ + p->nOut += nByte; + }else{ + p->nErr++; + } +} + /* Report an error. ** ** If this happens on the remote side, we send back a REMOTE_ERROR @@ -550,7 +593,7 @@ static void reportError(SQLiteRsync *p, const char *zFormat, ...){ putc(ORIGIN_ERROR, p->pOut); } writeUint32(p, nMsg); - fwrite(zMsg, nMsg, 1, p->pOut); + writeBytes(p, nMsg, zMsg); fflush(p->pOut); }else{ fprintf(stderr, "%s\n", zMsg); @@ -574,7 +617,7 @@ static void readAndDisplayError(SQLiteRsync *p){ return; } memset(zMsg, 0, n+1); - fread(zMsg, 1, n, p->pIn); + readBytes(p, n, zMsg); fprintf(stderr,"ERROR: %s\n", zMsg); sqlite3_free(zMsg); } @@ -639,6 +682,7 @@ static void runSql(SQLiteRsync *p, char *zSql, ...){ va_end(ap); if( pStmt ){ int rc = sqlite3_step(pStmt); + if( rc==SQLITE_ROW ) rc = sqlite3_step(pStmt); if( rc!=SQLITE_OK && rc!=SQLITE_DONE ){ reportError(p, "SQL statement [%s] failed: %s", zSql, sqlite3_errmsg(p->db)); @@ -732,21 +776,37 @@ static void closeDb(SQLiteRsync *p){ /* ** Run the origin-side protocol. ** -** 1. Send the origin-begin message -** 2. Receive replica-begin message -** - Error check and abort if necessary -** 3. Receive replica-hash messages -** 4. BEGIN -** 5. Send changed pages -** 6. COMMIT -** 7. Send origin-end message +** Begin by sending the ORIGIN_BEGIN message with two arguments, +** nPage, and szPage. Then enter a loop responding to message from +** the replica: +** +** REPLICA_ERROR size text +** +** Report an error from the replica and quit +** +** REPLICA_END +** +** The replica is terminating. Stop processing now. +** +** REPLICA_HASH hash +** +** The argument is the 20-byte SHA1 hash for the next page +** page hashes appear in sequential order with no gaps. +** +** REPLICA_READY +** +** The replica has sent all the hashes that it intends to send. +** This side (the origin) can now start responding with page +** content for pages that do not have a matching hash. */ static void originSide(SQLiteRsync *p){ int rc = 0; int c = 0; unsigned int nPage = 0; + unsigned int iPage = 0; unsigned int szPg = 0; - char buf[100]; + sqlite3_stmt *pCkHash = 0; + char buf[200]; if( p->bCommCheck ){ fprintf(p->pOut, "sqlite3-rsync origin-begin %s\n", p->zOrigin); @@ -759,7 +819,7 @@ static void originSide(SQLiteRsync *p){ } /* Open the ORIGIN database. */ - rc = sqlite3_open_v2(p->zOrigin, &p->db, SQLITE_OPEN_READONLY, 0); + rc = sqlite3_open_v2(p->zOrigin, &p->db, SQLITE_OPEN_READWRITE, 0); if( rc ){ reportError(p, "unable to open origin database file \"%s\": %s", sqlite3_errmsg(p->db)); @@ -777,30 +837,68 @@ static void originSide(SQLiteRsync *p){ if( p->nErr==0 ){ /* Send the ORIGIN_BEGIN message */ - fputc(ORIGIN_BEGIN, p->pOut); + writeByte(p, ORIGIN_BEGIN); writeUint32(p, nPage); writeUint32(p, szPg); fflush(p->pOut); + p->nPage = nPage; + p->szPage = szPg; } /* Respond to message from the replica */ - while( p->nErr==0 && (c = fgetc(p->pIn))!=EOF ){ + while( p->nErr==0 && (c = readByte(p))!=EOF && c!=REPLICA_END ){ switch( c ){ case REPLICA_ERROR: { readAndDisplayError(p); break; } - case REPLICA_BEGIN: { - break; - } - case REPLICA_END: { - break; - } case REPLICA_HASH: { + if( pCkHash==0 ){ + runSql(p, "CREATE TEMP TABLE badHash(pgno INTEGER PRIMARY KEY)"); + pCkHash = prepareStmt(p, + "INSERT INTO badHash SELECT pgno FROM sqlite_dbpage('main')" + " WHERE pgno=?1 AND sha1b(data)!=?2" + ); + if( pCkHash==0 ) break; + } + p->nHashSent++; + iPage++; + sqlite3_bind_int64(pCkHash, 1, iPage); + readBytes(p, 20, buf); + sqlite3_bind_blob(pCkHash, 2, buf, 20, SQLITE_STATIC); + rc = sqlite3_step(pCkHash); + if( rc!=SQLITE_DONE ){ + reportError(p, "SQL statement [%s] failed: %s", + sqlite3_sql(pCkHash), sqlite3_errmsg(p->db)); + } + sqlite3_reset(pCkHash); break; } case REPLICA_READY: { - break; + sqlite3_stmt *pStmt; + sqlite3_finalize(pCkHash); + pCkHash = 0; + pStmt = prepareStmt(p, + "SELECT pgno, data" + " FROM badHash JOIN sqlite_dbpage('main') USING(pgno) " + "UNION ALL " + "SELECT pgno, data" + " FROM sqlite_dbpage('main')" + " WHERE pgno>%d", + iPage); + if( pStmt==0 ) break; + while( sqlite3_step(pStmt)==SQLITE_ROW && p->nErr==0 ){ + const void *pContent = sqlite3_column_blob(pStmt, 1); + writeByte(p, ORIGIN_PAGE); + writeUint32(p, (unsigned int)sqlite3_column_int64(pStmt, 0)); + writeBytes(p, szPg, pContent); + p->nPageSent++; + } + sqlite3_finalize(pStmt); + writeByte(p, ORIGIN_TXN); + writeUint32(p, nPage); + writeByte(p, ORIGIN_END); + goto origin_end; } default: { reportError(p, "Origin side received unknown message: 0x%02x", c); @@ -809,24 +907,43 @@ static void originSide(SQLiteRsync *p){ } } +origin_end: + if( pCkHash ) sqlite3_finalize(pCkHash); closeDb(p); } /* -** Run the replica-side protocol. +** Run the replica-side protocol. The protocol is passive in the sense +** that it only response to message from the origin side. +** +** ORIGIN_BEGIN nPage szPage +** +** The origin is reporting the number of pages and the size of each +** pages. This procedure checks compatibility, and if everything is +** ok, it sends hash for all its extant pages. ** -** 1. Receive the origin-begin message -** - Error check. If unable to continue, send replica-error and quit -** 2. BEGIN IMMEDIATE -** 3. Send replica-begin message -** 4. Send replica-hash messages -** 5. Receive changed pages and apply them -** 6. Receive origin-end message -** 7. COMMIT +** ORIGIN_ERROR size text +** +** Report the received error and quit. +** +** ORIGIN_PAGE pgno content +** +** Update the content of the given page. +** +** ORIGIN_TXN pgno +** +** Close the update transaction. The total database size is pgno +** pages. +** +** ORIGIN_END +** +** Expect no more transmissions from the origin. */ static void replicaSide(SQLiteRsync *p){ int c; - char buf[100]; + sqlite3_stmt *pIns = 0; + unsigned int szOPage = 0; + char buf[65536]; if( p->bCommCheck ){ echoOneLine(p); fprintf(p->pOut, "replica-begin %s\n", p->zReplica); @@ -840,14 +957,14 @@ static void replicaSide(SQLiteRsync *p){ /* Respond to message from the origin. The origin will initiate the ** the conversation with an ORIGIN_BEGIN message. */ - while( p->nErr==0 && (c = fgetc(p->pIn))!=EOF ){ + while( p->nErr==0 && (c = readByte(p))!=EOF && c!=ORIGIN_END ){ switch( c ){ case ORIGIN_ERROR: { readAndDisplayError(p); break; } case ORIGIN_BEGIN: { - unsigned int nOPage = 0, szOPage = 0; + unsigned int nOPage = 0; unsigned int nRPage = 0, szRPage = 0; int rc = 0; sqlite3_stmt *pStmt = 0; @@ -856,6 +973,8 @@ static void replicaSide(SQLiteRsync *p){ readUint32(p, &nOPage); readUint32(p, &szOPage); if( p->nErr ) break; + p->nPage = nOPage; + p->szPage = szOPage; rc = sqlite3_open(p->zReplica, &p->db); if( rc ){ reportError(p, "cannot open replica database \"%s\": %s", @@ -885,15 +1004,64 @@ static void replicaSide(SQLiteRsync *p){ break; } pStmt = prepareStmt(p, - "SELECT pgno, sha1(data) FROM sqlite_dbpage" - " WHERE pgno<=min(%d,%d)", nRPage, nOPage); + "SELECT sha1b(data) FROM sqlite_dbpage" + " WHERE pgno<=min(%d,%d)" + " ORDER BY pgno", nRPage, nOPage); + while( sqlite3_step(pStmt)==SQLITE_ROW && p->nErr==0 ){ + const unsigned char *a = sqlite3_column_blob(pStmt, 0); + writeByte(p, REPLICA_HASH); + writeBytes(p, 20, a); + p->nHashSent++; + } sqlite3_finalize(pStmt); + writeByte(p, REPLICA_READY); + fflush(p->pOut); break; } - case ORIGIN_END: { + case ORIGIN_TXN: { + unsigned int nOPage = 0; + readUint32(p, &nOPage); + if( pIns==0 ){ + /* Nothing has changed */ + runSql(p, "COMMIT"); + }else if( p->nErr ){ + runSql(p, "ROLLBACK"); + }else{ + int rc; + sqlite3_bind_int64(pIns, 1, nOPage); + sqlite3_bind_null(pIns, 2); + rc = sqlite3_step(pIns); + if( rc!=SQLITE_DONE ){ + reportError(p, "SQL statement [%s] failed: %s", + sqlite3_sql(pIns), sqlite3_errmsg(p->db)); + } + sqlite3_reset(pIns); + runSql(p, "COMMIT"); + } break; } case ORIGIN_PAGE: { + unsigned int pgno = 0; + int rc; + readUint32(p, &pgno); + if( p->nErr ) break; + if( pIns==0 ){ + pIns = prepareStmt(p, + "INSERT INTO sqlite_dbpage(pgno,data,schema) VALUES(?1,?2,'main')" + ); + if( pIns==0 ) break; + } + readBytes(p, szOPage, buf); + if( p->nErr ) break; + p->nPageSent++; + sqlite3_bind_int64(pIns, 1, pgno); + sqlite3_bind_blob(pIns, 2, buf, szOPage, SQLITE_STATIC); + rc = sqlite3_step(pIns); + if( rc!=SQLITE_DONE ){ + reportError(p, "SQL statement [%s] failed: %s", + sqlite3_sql(pIns), sqlite3_errmsg(p->db)); + } + sqlite3_reset(pIns); break; } default: { @@ -903,9 +1071,24 @@ static void replicaSide(SQLiteRsync *p){ } } + if( pIns ) sqlite3_finalize(pIns); closeDb(p); } +/* +** The argument might be -vvv...vv with any number of "v"s. Return +** the number of "v"s. Return 0 if the argument is not a -vvv...v. +*/ +static int numVs(const char *z){ + int n = 0; + if( z[0]!='-' ) return 0; + z++; + if( z[0]=='-' ) z++; + while( z[0]=='v' ){ n++; z++; } + if( z[0]==0 ) return n; + return 0; +} + /* ** Parse command-line arguments. Dispatch subroutines to do the @@ -957,8 +1140,8 @@ int main(int argc, char **argv){ isReplica = 1; continue; } - if( strcmp(z, "-v")==0 ){ - ctx.eVerbose++; + if( numVs(z) ){ + ctx.eVerbose += numVs(z); continue; } if( strcmp(z, "--ssh")==0 ){ @@ -1066,7 +1249,7 @@ int main(int argc, char **argv){ fprintf(stderr, "Could not start auxiliary process: %s\n", zCmd); return 1; } - originSide(&ctx); + replicaSide(&ctx); }else if( (zDiv = strchr(ctx.zReplica,':'))!=0 ){ /* Local ORIGIN and remote REPLICA */ sqlite3_str *pStr = sqlite3_str_new(0); @@ -1091,7 +1274,7 @@ int main(int argc, char **argv){ }else{ /* Local ORIGIN and REPLICA */ sqlite3_str *pStr = sqlite3_str_new(0); - append_escaped_arg(pStr, zExe, 1); + append_escaped_arg(pStr, argv[0], 1); append_escaped_arg(pStr, "--replica", 0); if( ctx.bCommCheck ){ append_escaped_arg(pStr, "--commcheck", 0); @@ -1105,6 +1288,16 @@ int main(int argc, char **argv){ } originSide(&ctx); } + if( ctx.eVerbose ){ + if( ctx.nErr ) printf("%d errors, ", ctx.nErr); + printf("%lld bytes sent, %lld bytes received\n", ctx.nOut, ctx.nIn); + if( ctx.eVerbose>=2 ){ + printf("Database is %u pages of %u bytes each.\n", + ctx.nPage, ctx.szPage); + printf("Sent %u hashes, %u page contents\n", + ctx.nHashSent, ctx.nPageSent); + } + } sqlite3_free(zCmd); if( pIn!=0 && pOut!=0 ){ pclose2(pIn, pOut, childPid);