From: drh Date: Wed, 8 Jun 2016 01:03:05 +0000 (+0000) Subject: An initial attempt at a "dbhash" command-line utility. X-Git-Tag: version-3.14.0~102^2~2 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=290fcaa2d130bc488c22e615d37c240186714260;p=thirdparty%2Fsqlite.git An initial attempt at a "dbhash" command-line utility. FossilOrigin-Name: 2247649ca215c06205b33b2250eb809baf39263a --- diff --git a/Makefile.in b/Makefile.in index 115cfac58f..29c9174afa 100644 --- a/Makefile.in +++ b/Makefile.in @@ -539,7 +539,8 @@ TESTPROGS = \ testfixture$(TEXE) \ sqlite3$(TEXE) \ sqlite3_analyzer$(TEXE) \ - sqldiff$(TEXE) + sqldiff$(TEXE) \ + dbhash$(TEXE) # Databases containing fuzzer test cases # @@ -590,6 +591,9 @@ sqlite3$(TEXE): $(TOP)/src/shell.c sqlite3.c sqldiff$(TEXE): $(TOP)/tool/sqldiff.c sqlite3.c sqlite3.h $(LTLINK) -o $@ $(TOP)/tool/sqldiff.c sqlite3.c $(TLIBS) +dbhash$(TEXE): $(TOP)/tool/dbhash.c sqlite3.c sqlite3.h + $(LTLINK) -o $@ $(TOP)/tool/dbhash.c sqlite3.c $(TLIBS) + scrub$(TEXE): $(TOP)/ext/misc/scrub.c sqlite3.o $(LTLINK) -o $@ -I. -DSCRUB_STANDALONE \ $(TOP)/ext/misc/scrub.c sqlite3.o $(TLIBS) @@ -1251,6 +1255,7 @@ clean: rm -f fuzzershell fuzzershell.exe rm -f fuzzcheck fuzzcheck.exe rm -f sqldiff sqldiff.exe + rm -f dbhash dbhash.exe rm -f fts5.* fts5parse.* distclean: clean diff --git a/Makefile.msc b/Makefile.msc index 9bf1e4f8fe..93116cd0dc 100644 --- a/Makefile.msc +++ b/Makefile.msc @@ -1378,7 +1378,8 @@ TESTPROGS = \ testfixture.exe \ $(SQLITE3EXE) \ sqlite3_analyzer.exe \ - sqldiff.exe + sqldiff.exe \ + dbhash.exe # Databases containing fuzzer test cases # @@ -1456,6 +1457,9 @@ $(SQLITE3EXE): $(TOP)\src\shell.c $(SHELL_CORE_DEP) $(LIBRESOBJS) $(SHELL_CORE_S sqldiff.exe: $(TOP)\tool\sqldiff.c $(SQLITE3C) $(SQLITE3H) $(LTLINK) $(NO_WARN) $(TOP)\tool\sqldiff.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) +dbhash.exe: $(TOP)\tool\dbhash.c $(SQLITE3C) $(SQLITE3H) + $(LTLINK) $(NO_WARN) $(TOP)\tool\dbhash.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) + scrub.exe: $(TOP)\ext\misc\scrub.c $(SQLITE3C) $(SQLITE3H) $(LTLINK) $(NO_WARN) $(TOP)\ext\misc\scrub.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) @@ -2105,6 +2109,6 @@ clean: del /Q shell.c sqlite3ext.h sqlite3session.h 2>NUL del /Q sqlite3_analyzer.exe sqlite3_analyzer.c 2>NUL del /Q sqlite-*-output.vsix 2>NUL - del /Q fuzzershell.exe fuzzcheck.exe sqldiff.exe 2>NUL + del /Q fuzzershell.exe fuzzcheck.exe sqldiff.exe dbhash.exe 2>NUL del /Q fts5.* fts5parse.* 2>NUL # <> diff --git a/main.mk b/main.mk index 624bf77a06..ba710bfa46 100644 --- a/main.mk +++ b/main.mk @@ -451,7 +451,8 @@ TESTPROGS = \ testfixture$(EXE) \ sqlite3$(EXE) \ sqlite3_analyzer$(EXE) \ - sqldiff$(EXE) + sqldiff$(EXE) \ + dbhash$(EXE) # Databases containing fuzzer test cases # @@ -488,6 +489,10 @@ sqldiff$(EXE): $(TOP)/tool/sqldiff.c sqlite3.c sqlite3.h $(TCCX) -o sqldiff$(EXE) -DSQLITE_THREADSAFE=0 \ $(TOP)/tool/sqldiff.c sqlite3.c $(TLIBS) $(THREADLIB) +dbhash$(EXE): $(TOP)/tool/dbhash.c sqlite3.c sqlite3.h + $(TCCX) -o dbhash$(EXE) -DSQLITE_THREADSAFE=0 \ + $(TOP)/tool/dbhash.c sqlite3.c $(TLIBS) $(THREADLIB) + scrub$(EXE): $(TOP)/ext/misc/scrub.c sqlite3.o $(TCC) -I. -DSCRUB_STANDALONE -o scrub$(EXE) $(TOP)/ext/misc/scrub.c sqlite3.o $(THREADLIB) diff --git a/manifest b/manifest index 36bb44109e..7d7f2a3501 100644 --- a/manifest +++ b/manifest @@ -1,8 +1,8 @@ -C Fix\sthe\swalcrash4.test\stest\smodule\sso\sthat\sit\sworks\son\swindows. -D 2016-06-07T20:25:19.931 -F Makefile.in 7321ef0b584224781ec7731408857fa8962c32cc +C An\sinitial\sattempt\sat\sa\s"dbhash"\scommand-line\sutility. +D 2016-06-08T01:03:05.100 +F Makefile.in f3f7d2060ce03af4584e711ef3a626ef0b1d6340 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 -F Makefile.msc 831503fc4e988f571590af1405645fff121b5f1e +F Makefile.msc 50149765ef72f4e652b9a0f1f6462c4784bb9423 F README.md 8ecc12493ff9f820cdea6520a9016001cb2e59b7 F VERSION cb29eb11e493dd85b3eeec4053c03949bf98478e F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50 @@ -307,7 +307,7 @@ F ext/userauth/userauth.c 5fa3bdb492f481bbc1709fc83c91ebd13460c69e F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60 -F main.mk 2b90646ca027cc21dbae209a0fee68dfedfe0e83 +F main.mk 3f669c06db5c4a53ff21dda639247c6310497180 F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83 F mptest/config01.test 3c6adcbc50b991866855f1977ff172eb6d901271 F mptest/config02.test 4415dfe36c48785f751e16e32c20b077c28ae504 @@ -1420,6 +1420,7 @@ F tool/build-all-msvc.bat 3e4e4043b53f1aede4308e0d2567bbd773614630 x F tool/build-shell.sh 950f47c6174f1eea171319438b93ba67ff5bf367 F tool/cg_anno.tcl 692ce4b8693d59e3a3de77ca97f4139ecfa641b0 x F tool/checkSpacing.c 810e51703529a204fc4e1eb060e9ab663e3c06d2 +F tool/dbhash.c 11ae59057c1ebbd54ac10bdc59c8fc7e0a43a156 F tool/extract.c 054069d81b095fbdc189a6f5d4466e40380505e2 F tool/fast_vacuum.c 5ba0d6f5963a0a63bdc42840f678bad75b2ebce1 F tool/fragck.tcl 5265a95126abcf6ab357f7efa544787e5963f439 @@ -1500,7 +1501,10 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P e404ad705d0e2d96025d05cc88348ffcd0703533 -R 17828d50ca8a72aedc66939901a97177 +P 2091a4c9231c7871f27661adc27dd7df26500f6c +R 2cb484710443ce45b289f472a9c90e73 +T *branch * dbhash +T *sym-dbhash * +T -sym-trunk * U drh -Z c43be01e1ddab6118f1ee2a4247dca79 +Z 1329a111dd8194eb5ec209c8b1ad3288 diff --git a/manifest.uuid b/manifest.uuid index 459a559a5d..ca0df0561c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -2091a4c9231c7871f27661adc27dd7df26500f6c \ No newline at end of file +2247649ca215c06205b33b2250eb809baf39263a \ No newline at end of file diff --git a/tool/dbhash.c b/tool/dbhash.c new file mode 100644 index 0000000000..0e1b02b673 --- /dev/null +++ b/tool/dbhash.c @@ -0,0 +1,424 @@ +/* +** 2016-06-07 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This is a utility program that computes a hash on the content +** of an SQLite database. +** +** The hash is computed over just the content of the database. Free +** space inside of the database file, and alternative on-disk representations +** of the same content (ex: UTF8 vs UTF16) do not affect the hash. So, +** for example, the database file page size, encoding, and auto_vacuum setting +** can all be changed without changing the hash. +*/ +#include +#include +#include +#include +#include +#include +#include "sqlite3.h" + +/* Context for the SHA1 hash */ +typedef struct SHA1Context SHA1Context; +struct SHA1Context { + unsigned int state[5]; + unsigned int count[2]; + unsigned char buffer[64]; +}; + +/* +** All global variables are gathered into the "g" singleton. +*/ +struct GlobalVars { + const char *zArgv0; /* Name of program */ + int bSchemaPK; /* Use the schema-defined PK, not the true PK */ + unsigned fDebug; /* Debug flags */ + sqlite3 *db; /* The database connection */ + SHA1Context cx; /* SHA1 hash context */ +} g; + +/****************************************************************************** +** The Hash Engine +** +** Modify these routines (and appropriate state fields in global variable 'g') +** in order to compute a different (better?) hash of the database. +*/ +/* + * blk0() and blk() perform the initial expand. + * I got the idea of expanding during the round function from SSLeay + * + * blk0le() for little-endian and blk0be() for big-endian. + */ +#if __GNUC__ && (defined(__i386__) || defined(__x86_64__)) +/* + * GCC by itself only generates left rotates. Use right rotates if + * possible to be kinder to dinky implementations with iterative rotate + * instructions. + */ +#define SHA_ROT(op, x, k) \ + ({ unsigned int y; asm(op " %1,%0" : "=r" (y) : "I" (k), "0" (x)); y; }) +#define rol(x,k) SHA_ROT("roll", x, k) +#define ror(x,k) SHA_ROT("rorl", x, k) + +#else +/* Generic C equivalent */ +#define SHA_ROT(x,l,r) ((x) << (l) | (x) >> (r)) +#define rol(x,k) SHA_ROT(x,k,32-(k)) +#define ror(x,k) SHA_ROT(x,32-(k),k) +#endif + + +#define blk0le(i) (block[i] = (ror(block[i],8)&0xFF00FF00) \ + |(rol(block[i],8)&0x00FF00FF)) +#define blk0be(i) block[i] +#define blk(i) (block[i&15] = rol(block[(i+13)&15]^block[(i+8)&15] \ + ^block[(i+2)&15]^block[i&15],1)) + +/* + * (R0+R1), R2, R3, R4 are the different operations (rounds) used in SHA1 + * + * Rl0() for little-endian and Rb0() for big-endian. Endianness is + * determined at run-time. + */ +#define Rl0(v,w,x,y,z,i) \ + z+=((w&(x^y))^y)+blk0le(i)+0x5A827999+rol(v,5);w=ror(w,2); +#define Rb0(v,w,x,y,z,i) \ + z+=((w&(x^y))^y)+blk0be(i)+0x5A827999+rol(v,5);w=ror(w,2); +#define R1(v,w,x,y,z,i) \ + z+=((w&(x^y))^y)+blk(i)+0x5A827999+rol(v,5);w=ror(w,2); +#define R2(v,w,x,y,z,i) \ + z+=(w^x^y)+blk(i)+0x6ED9EBA1+rol(v,5);w=ror(w,2); +#define R3(v,w,x,y,z,i) \ + z+=(((w|x)&y)|(w&x))+blk(i)+0x8F1BBCDC+rol(v,5);w=ror(w,2); +#define R4(v,w,x,y,z,i) \ + z+=(w^x^y)+blk(i)+0xCA62C1D6+rol(v,5);w=ror(w,2); + +/* + * Hash a single 512-bit block. This is the core of the algorithm. + */ +#define a qq[0] +#define b qq[1] +#define c qq[2] +#define d qq[3] +#define e qq[4] + +void SHA1Transform(unsigned int state[5], const unsigned char buffer[64]){ + unsigned int qq[5]; /* a, b, c, d, e; */ + static int one = 1; + unsigned int block[16]; + memcpy(block, buffer, 64); + memcpy(qq,state,5*sizeof(unsigned int)); + + /* Copy g.cx.state[] to working vars */ + /* + a = state[0]; + b = state[1]; + c = state[2]; + d = state[3]; + e = state[4]; + */ + + /* 4 rounds of 20 operations each. Loop unrolled. */ + if( 1 == *(unsigned char*)&one ){ + Rl0(a,b,c,d,e, 0); Rl0(e,a,b,c,d, 1); Rl0(d,e,a,b,c, 2); Rl0(c,d,e,a,b, 3); + Rl0(b,c,d,e,a, 4); Rl0(a,b,c,d,e, 5); Rl0(e,a,b,c,d, 6); Rl0(d,e,a,b,c, 7); + Rl0(c,d,e,a,b, 8); Rl0(b,c,d,e,a, 9); Rl0(a,b,c,d,e,10); Rl0(e,a,b,c,d,11); + Rl0(d,e,a,b,c,12); Rl0(c,d,e,a,b,13); Rl0(b,c,d,e,a,14); Rl0(a,b,c,d,e,15); + }else{ + Rb0(a,b,c,d,e, 0); Rb0(e,a,b,c,d, 1); Rb0(d,e,a,b,c, 2); Rb0(c,d,e,a,b, 3); + Rb0(b,c,d,e,a, 4); Rb0(a,b,c,d,e, 5); Rb0(e,a,b,c,d, 6); Rb0(d,e,a,b,c, 7); + Rb0(c,d,e,a,b, 8); Rb0(b,c,d,e,a, 9); Rb0(a,b,c,d,e,10); Rb0(e,a,b,c,d,11); + Rb0(d,e,a,b,c,12); Rb0(c,d,e,a,b,13); Rb0(b,c,d,e,a,14); Rb0(a,b,c,d,e,15); + } + R1(e,a,b,c,d,16); R1(d,e,a,b,c,17); R1(c,d,e,a,b,18); R1(b,c,d,e,a,19); + R2(a,b,c,d,e,20); R2(e,a,b,c,d,21); R2(d,e,a,b,c,22); R2(c,d,e,a,b,23); + R2(b,c,d,e,a,24); R2(a,b,c,d,e,25); R2(e,a,b,c,d,26); R2(d,e,a,b,c,27); + R2(c,d,e,a,b,28); R2(b,c,d,e,a,29); R2(a,b,c,d,e,30); R2(e,a,b,c,d,31); + R2(d,e,a,b,c,32); R2(c,d,e,a,b,33); R2(b,c,d,e,a,34); R2(a,b,c,d,e,35); + R2(e,a,b,c,d,36); R2(d,e,a,b,c,37); R2(c,d,e,a,b,38); R2(b,c,d,e,a,39); + R3(a,b,c,d,e,40); R3(e,a,b,c,d,41); R3(d,e,a,b,c,42); R3(c,d,e,a,b,43); + R3(b,c,d,e,a,44); R3(a,b,c,d,e,45); R3(e,a,b,c,d,46); R3(d,e,a,b,c,47); + R3(c,d,e,a,b,48); R3(b,c,d,e,a,49); R3(a,b,c,d,e,50); R3(e,a,b,c,d,51); + R3(d,e,a,b,c,52); R3(c,d,e,a,b,53); R3(b,c,d,e,a,54); R3(a,b,c,d,e,55); + R3(e,a,b,c,d,56); R3(d,e,a,b,c,57); R3(c,d,e,a,b,58); R3(b,c,d,e,a,59); + R4(a,b,c,d,e,60); R4(e,a,b,c,d,61); R4(d,e,a,b,c,62); R4(c,d,e,a,b,63); + R4(b,c,d,e,a,64); R4(a,b,c,d,e,65); R4(e,a,b,c,d,66); R4(d,e,a,b,c,67); + R4(c,d,e,a,b,68); R4(b,c,d,e,a,69); R4(a,b,c,d,e,70); R4(e,a,b,c,d,71); + R4(d,e,a,b,c,72); R4(c,d,e,a,b,73); R4(b,c,d,e,a,74); R4(a,b,c,d,e,75); + R4(e,a,b,c,d,76); R4(d,e,a,b,c,77); R4(c,d,e,a,b,78); R4(b,c,d,e,a,79); + + /* Add the working vars back into context.state[] */ + state[0] += a; + state[1] += b; + state[2] += c; + state[3] += d; + state[4] += e; +} + + +/* Initialize the SHA1 hash */ +static void hash_init(void){ + /* SHA1 initialization constants */ + g.cx.state[0] = 0x67452301; + g.cx.state[1] = 0xEFCDAB89; + g.cx.state[2] = 0x98BADCFE; + g.cx.state[3] = 0x10325476; + g.cx.state[4] = 0xC3D2E1F0; + g.cx.count[0] = g.cx.count[1] = 0; +} + +/* Add new content to the SHA1 hash */ +static void hash_step(const unsigned char *data, unsigned int len){ + unsigned int i, j; + + j = g.cx.count[0]; + if( (g.cx.count[0] += len << 3) < j ){ + g.cx.count[1] += (len>>29)+1; + } + j = (j >> 3) & 63; + if( (j + len) > 63 ){ + (void)memcpy(&g.cx.buffer[j], data, (i = 64-j)); + SHA1Transform(g.cx.state, g.cx.buffer); + for(; i + 63 < len; i += 64){ + SHA1Transform(g.cx.state, &data[i]); + } + j = 0; + }else{ + i = 0; + } + (void)memcpy(&g.cx.buffer[j], &data[i], len - i); +} + + +/* Add padding and compute and output the message digest. */ +static void hash_finish(void){ + unsigned int i; + unsigned char finalcount[8]; + unsigned char digest[20]; + static const char zEncode[] = "0123456789abcdef"; + char zOut[40]; + + for (i = 0; i < 8; i++){ + finalcount[i] = (unsigned char)((g.cx.count[(i >= 4 ? 0 : 1)] + >> ((3-(i & 3)) * 8) ) & 255); /* Endian independent */ + } + hash_step((const unsigned char *)"\200", 1); + while ((g.cx.count[0] & 504) != 448){ + hash_step((const unsigned char *)"\0", 1); + } + hash_step(finalcount, 8); /* Should cause a SHA1Transform() */ + for (i = 0; i < 20; i++){ + digest[i] = (unsigned char)((g.cx.state[i>>2] >> ((3-(i & 3)) * 8) ) & 255); + } + for(i=0; i<20; i++){ + zOut[i*2] = zEncode[(digest[i]>>4)&0xf]; + zOut[i*2+1] = zEncode[digest[i] & 0xf]; + } + zOut[i*2]= 0; + printf("%s\n", zOut); +} +/* End of the hashing logic +*******************************************************************************/ + +/* +** Print an error resulting from faulting command-line arguments and +** abort the program. +*/ +static void cmdlineError(const char *zFormat, ...){ + va_list ap; + fprintf(stderr, "%s: ", g.zArgv0); + va_start(ap, zFormat); + vfprintf(stderr, zFormat, ap); + va_end(ap); + fprintf(stderr, "\n\"%s --help\" for more help\n", g.zArgv0); + exit(1); +} + +/* +** Print an error message for an error that occurs at runtime, then +** abort the program. +*/ +static void runtimeError(const char *zFormat, ...){ + va_list ap; + fprintf(stderr, "%s: ", g.zArgv0); + va_start(ap, zFormat); + vfprintf(stderr, zFormat, ap); + va_end(ap); + fprintf(stderr, "\n"); + exit(1); +} + +/* +** Prepare a new SQL statement. Print an error and abort if anything +** goes wrong. +*/ +static sqlite3_stmt *db_vprepare(const char *zFormat, va_list ap){ + char *zSql; + int rc; + sqlite3_stmt *pStmt; + + zSql = sqlite3_vmprintf(zFormat, ap); + if( zSql==0 ) runtimeError("out of memory"); + rc = sqlite3_prepare_v2(g.db, zSql, -1, &pStmt, 0); + if( rc ){ + runtimeError("SQL statement error: %s\n\"%s\"", sqlite3_errmsg(g.db), + zSql); + } + sqlite3_free(zSql); + return pStmt; +} +static sqlite3_stmt *db_prepare(const char *zFormat, ...){ + va_list ap; + sqlite3_stmt *pStmt; + va_start(ap, zFormat); + pStmt = db_vprepare(zFormat, ap); + va_end(ap); + return pStmt; +} + +/* +** Compute the hash for a single table named zTab +*/ +static void hash_one_table(const char *zTab){ + sqlite3_stmt *pStmt; + int nCol; + int i; + pStmt = db_prepare("SELECT * FROM \"%w\";", zTab); + nCol = sqlite3_column_count(pStmt); + while( SQLITE_ROW==sqlite3_step(pStmt) ){ + for(i=0; i=0; j--){ + x[j] = u & 0xff; + u >>= 8; + } + hash_step((const unsigned char*)"1",1); + hash_step(x,8); + break; + } + case SQLITE_FLOAT: { + sqlite3_uint64 u; + int j; + unsigned char x[8]; + double r = sqlite3_column_double(pStmt,i); + memcpy(&u, &r, 8); + for(j=7; j>=0; j--){ + x[j] = u & 0xff; + u >>= 8; + } + hash_step((const unsigned char*)"2",1); + hash_step(x,8); + break; + } + case SQLITE_TEXT: { + int n = sqlite3_column_bytes(pStmt, i); + const unsigned char *z = sqlite3_column_text(pStmt, i); + hash_step((const unsigned char*)"3", 1); + hash_step(z, n); + break; + } + case SQLITE_BLOB: { + int n = sqlite3_column_bytes(pStmt, i); + const unsigned char *z = sqlite3_column_blob(pStmt, i); + hash_step((const unsigned char*)"4", 1); + hash_step(z, n); + break; + } + } + } + } + sqlite3_finalize(pStmt); +} + + +/* +** Print sketchy documentation for this utility program +*/ +static void showHelp(void){ + printf("Usage: %s DB\n", g.zArgv0); + printf( +"Compute a hash on the content of database DB\n" + ); +} + +int main(int argc, char **argv){ + const char *zDb = 0; + int i; + int rc; + char *zErrMsg; + sqlite3_stmt *pStmt; + + g.zArgv0 = argv[0]; + sqlite3_config(SQLITE_CONFIG_SINGLETHREAD); + for(i=1; i