From: drh Date: Thu, 13 Dec 2018 15:06:26 +0000 (+0000) Subject: Add the "dbtotxt" utility program and the ability to read "dbtotxt" output X-Git-Tag: version-3.27.0~301 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=3374648545262810849a3b17fdb4435bbb255a07;p=thirdparty%2Fsqlite.git Add the "dbtotxt" utility program and the ability to read "dbtotxt" output as a deserialized input database in the CLI, using the --hexdb option to the ".open" command. FossilOrigin-Name: e3bf1d3ea5f748c5142c2403813fdace5aedc1fc68f0dcd5eae40a2fe763fedb --- diff --git a/Makefile.in b/Makefile.in index b200ed6628..69371076fe 100644 --- a/Makefile.in +++ b/Makefile.in @@ -1292,6 +1292,9 @@ dbdump$(TEXE): $(TOP)/ext/misc/dbdump.c sqlite3.lo $(LTLINK) -DDBDUMP_STANDALONE -o $@ \ $(TOP)/ext/misc/dbdump.c sqlite3.lo $(TLIBS) +dbtotxt$(TEXE): $(TOP)/tool/dbtotxt.c + $(LTLINK)-o $@ $(TOP)/tool/dbtotxt.c + showdb$(TEXE): $(TOP)/tool/showdb.c sqlite3.lo $(LTLINK) -o $@ $(TOP)/tool/showdb.c sqlite3.lo $(TLIBS) diff --git a/Makefile.msc b/Makefile.msc index 3b95d088d2..f5d47240f5 100644 --- a/Makefile.msc +++ b/Makefile.msc @@ -2425,6 +2425,9 @@ testloadext.lo: $(TOP)\src\test_loadext.c $(SQLITE3H) testloadext.dll: testloadext.lo $(LD) $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) /DLL /OUT:$@ testloadext.lo +dbtotxt.exe: $(TOP)\tool\dbtotxt.c + $(LTLINK) $(NO_WARN) $(TOP)\tool\dbtotxt.c /link $(LDFLAGS) $(LTLINKOPTS) + showdb.exe: $(TOP)\tool\showdb.c $(SQLITE3C) $(SQLITE3H) $(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION \ $(TOP)\tool\showdb.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) diff --git a/main.mk b/main.mk index 7a85619732..65c2c9dc23 100644 --- a/main.mk +++ b/main.mk @@ -976,6 +976,9 @@ $(TEST_EXTENSION): $(TOP)/src/test_loadext.c extensiontest: testfixture$(EXE) $(TEST_EXTENSION) ./testfixture$(EXE) $(TOP)/test/loadext.test +dbtotxt$(EXE): $(TOP)/tool/dbtotxt.c + $(TCC) -o dbtotxt$(EXE) $(TOP)/tool/dbtotxt.c + showdb$(EXE): $(TOP)/tool/showdb.c sqlite3.o $(TCC) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -o showdb$(EXE) \ $(TOP)/tool/showdb.c sqlite3.o $(THREADLIB) diff --git a/manifest b/manifest index 3cda5a3847..127fdbd6ec 100644 --- a/manifest +++ b/manifest @@ -1,10 +1,10 @@ -C Include\sOPTS\sin\sthe\sbuild\sof\sdbfuzz2. -D 2018-12-13T12:37:13.440 +C Add\sthe\s"dbtotxt"\sutility\sprogram\sand\sthe\sability\sto\sread\s"dbtotxt"\soutput\nas\sa\sdeserialized\sinput\sdatabase\sin\sthe\sCLI,\susing\sthe\s--hexdb\soption\sto\nthe\s".open"\scommand. +D 2018-12-13T15:06:26.121 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea -F Makefile.in 6fbf65894d820ac4a5b624552febce4283cd6d9db052a85310868be78ceddddb +F Makefile.in 2f1b61ac62689ca4e9cbff9fdb359578ea37ddd9252355ec0b7b9700ad56fe90 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 -F Makefile.msc b7d4a710fa3f0b8cfc532ff195b85dc1ba2a8ad34343cb3d67639f28f0a24306 +F Makefile.msc 2ef13d6845b899eaaa6122c69b74175656a97e26666567af795f4cfe41b7a673 F README.md 377233394b905d3b2e2b33741289e093bc93f2e7adbe00923b2c5958c9a9edee F VERSION 453e2f4529ca208196d5567db28d549d7151f79efd33f6e6cfe6e613e583a0be F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50 @@ -429,7 +429,7 @@ F ext/userauth/userauth.c f81aa5a3ecacf406f170c62a144405858f6f6de51dbdc0920134e6 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60 -F main.mk 55f94164ecc194b067d9c55e106f37fd3c9b39f9668e8b568c98f008b6f9ec90 +F main.mk cca1ecdd5dfe4579ded84e5abdd38e1866bbb8b8cf58d1a24496e9b1b65580d7 F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83 F mptest/config01.test 3c6adcbc50b991866855f1977ff172eb6d901271 F mptest/config02.test 4415dfe36c48785f751e16e32c20b077c28ae504 @@ -508,7 +508,7 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 F src/resolve.c abd65c518c198400193c6319a70c0d722fa30a35be89dc898917ff6489edf017 F src/rowset.c d977b011993aaea002cab3e0bb2ce50cf346000dff94e944d547b989f4b1fe93 F src/select.c 8c7317d5ee920516a56b8b4ca79fbfca70a1f8b52d67e884c808ea3a016c04e3 -F src/shell.c.in 5f38bd0e127c2cc4e506b5c3565c10879ddfae6c2d867bb5972563e40717c19c +F src/shell.c.in 6c06ff4077ab38b30a307c058ecc650e958bfdb6da1c255ebbd8779447e0cae9 F src/sqlite.h.in 92fd656c26cc76de9fa8c5bf1a473066e3b5c6da345a447679f0f44de1aa4edd F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 960f1b86c3610fa23cb6a267572a97dcf286e77aa0dd3b9b23292ffaa1ea8683 @@ -1694,6 +1694,8 @@ F tool/build-shell.sh 950f47c6174f1eea171319438b93ba67ff5bf367 F tool/cg_anno.tcl f95b0006c52cf7f0496b506343415b6ee3cdcdd3 x F tool/checkSpacing.c 810e51703529a204fc4e1eb060e9ab663e3c06d2 F tool/dbhash.c a06228aa21ebc4e6ea8daa486601d938499238a5 +F tool/dbtotxt.c 1655f60fd7b24a3e7a25d01cdb3a6a4785f30112213b08ff83a27ae7ef2dd13e +F tool/dbtotxt.md c9a57af8739957ef36d2cfad5c4b1443ff3688ed33e4901ee200c8b651f43f3c F tool/extract.c 054069d81b095fbdc189a6f5d4466e40380505e2 F tool/fast_vacuum.c 5ba0d6f5963a0a63bdc42840f678bad75b2ebce1 F tool/fragck.tcl 5265a95126abcf6ab357f7efa544787e5963f439 @@ -1784,7 +1786,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 54231ac4ca506e6c34c31bc02bb8d3db22f14e1b01679bc70529b165d7dabb83 -R 3965f390ed5089653fa0ae080a56637b +P eac9c0c49650b008951318a2225b932709a0a8dffbb01ed57684c40357b2e25c +R 4375ea249bc2f1867ed511e156cfaedb U drh -Z 3b3f257dcedf459d870badc7f1cd6ad5 +Z 5d7eea8bc66404385fe54bd1e1b7369f diff --git a/manifest.uuid b/manifest.uuid index 13bd2abd32..4226c78984 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -eac9c0c49650b008951318a2225b932709a0a8dffbb01ed57684c40357b2e25c \ No newline at end of file +e3bf1d3ea5f748c5142c2403813fdace5aedc1fc68f0dcd5eae40a2fe763fedb \ No newline at end of file diff --git a/src/shell.c.in b/src/shell.c.in index fd9cefa7a8..70cbfe9eb9 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -1066,6 +1066,7 @@ struct ShellState { #define SHELL_OPEN_ZIPFILE 3 /* Use the zipfile virtual table */ #define SHELL_OPEN_READONLY 4 /* Open a normal database read-only */ #define SHELL_OPEN_DESERIALIZE 5 /* Open using sqlite3_deserialize() */ +#define SHELL_OPEN_HEXDB 6 /* Use "dbtotxt" output as data source */ /* Allowed values for ShellState.eTraceType */ @@ -3444,6 +3445,7 @@ static const char *(azHelp[]) = { " --append Use appendvfs to append database to the end of FILE", #ifdef SQLITE_ENABLE_DESERIALIZE " --deserialize Load into memory useing sqlite3_deserialize()", + " --hexdb Load the output of \"dbtotxt\" as an in-memory database", #endif " --new Initialize FILE to an empty database", " --readonly Open FILE readonly", @@ -3723,6 +3725,87 @@ int deduceDatabaseType(const char *zName, int dfltZip){ return rc; } +#ifdef SQLITE_ENABLE_DESERIALIZE +/* +** Reconstruct an in-memory database using the output from the "dbtotxt" +** program. Read content from the file in p->zDbFilename. If p->zDbFilename +** is 0, then read from standard input. +*/ +static unsigned char *readHexDb(ShellState *p, int *pnData){ + unsigned char *a = 0; + int nLine = 1; + int n = 0; + int pgsz = 0; + int iOffset = 0; + int j, k; + int rc; + FILE *in; + unsigned char x[16]; + char zLine[100]; + if( p->zDbFilename ){ + in = fopen(p->zDbFilename, "r"); + if( in==0 ){ + utf8_printf(stderr, "cannot open \"%s\" for reading\n", p->zDbFilename); + return 0; + } + }else{ + in = stdin; + } + *pnData = 0; + if( fgets(zLine, sizeof(zLine), in)==0 ) goto readHexDb_error; + rc = sscanf(zLine, "| size %d pagesize %d", &n, &pgsz); + if( rc!=2 ) goto readHexDb_error; + if( n<=0 ) goto readHexDb_error; + a = sqlite3_malloc( n ); + if( a==0 ){ + utf8_printf(stderr, "Out of memory!\n"); + goto readHexDb_error; + } + memset(a, 0, n); + if( pgsz<512 || pgsz>65536 || (pgsz & (pgsz-1))!=0 ){ + utf8_printf(stderr, "invalid pagesize\n"); + goto readHexDb_error; + } + for(nLine=2; fgets(zLine, sizeof(zLine), in)!=0; nLine++){ + rc = sscanf(zLine, "| page %d offset %d", &j, &k); + if( rc==2 ){ + iOffset = k; + continue; + } + if( strncmp(zLine, "| end ", 6)==0 ){ + break; + } + rc = sscanf(zLine,"| %d: %hhx %hhx %hhx %hhx %hhx %hhx %hhx %hhx" + " %hhx %hhx %hhx %hhx %hhx %hhx %hhx %hhx", + &j, &x[0], &x[1], &x[2], &x[3], &x[4], &x[5], &x[6], &x[7], + &x[8], &x[9], &x[10], &x[11], &x[12], &x[13], &x[14], &x[15]); + if( rc==17 ){ + k = iOffset+j; + if( k+16>n ){ + utf8_printf(stderr, "continue exceeds file size\n"); + goto readHexDb_error; + } + memcpy(a+k, x, 16); + } + } + *pnData = n; + if( in!=stdin ) fclose(in); + return a; + +readHexDb_error: + if( in!=stdin ){ + fclose(in); + }else{ + while( fgets(zLine, sizeof(zLine), in)!=0 ){ + if(strncmp(zLine, "| end ", 6)==0 ) break; + } + } + sqlite3_free(a); + utf8_printf(stderr,"Error on line %d of --hexdb input\n", nLine); + return 0; +} +#endif /* SQLITE_ENABLE_DESERIALIZE */ + /* Flags for open_db(). ** ** The default behavior of open_db() is to exit(1) if the database fails to @@ -3756,6 +3839,7 @@ static void open_db(ShellState *p, int openFlags){ SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE, "apndvfs"); break; } + case SHELL_OPEN_HEXDB: case SHELL_OPEN_DESERIALIZE: { sqlite3_open(0, &p->db); break; @@ -3810,9 +3894,19 @@ static void open_db(ShellState *p, int openFlags){ sqlite3_free(zSql); } #ifdef SQLITE_ENABLE_DESERIALIZE - else if( p->openMode==SHELL_OPEN_DESERIALIZE ){ + else + if( p->openMode==SHELL_OPEN_DESERIALIZE || p->openMode==SHELL_OPEN_HEXDB ){ int nData = 0; - unsigned char *aData = (unsigned char*)readFile(p->zDbFilename, &nData); + unsigned char *aData; + if( p->openMode==SHELL_OPEN_DESERIALIZE ){ + aData = (unsigned char*)readFile(p->zDbFilename, &nData); + }else{ + aData = readHexDb(p, &nData); + if( aData==0 ){ + utf8_printf(stderr, "Error in hexdb input\n"); + return; + } + } int rc = sqlite3_deserialize(p->db, "main", aData, nData, nData, SQLITE_DESERIALIZE_RESIZEABLE | SQLITE_DESERIALIZE_FREEONCLOSE); @@ -6749,7 +6843,9 @@ static int do_meta_command(char *zLine, ShellState *p){ #ifdef SQLITE_ENABLE_DESERIALIZE }else if( optionMatch(z, "deserialize") ){ p->openMode = SHELL_OPEN_DESERIALIZE; -#endif + }else if( optionMatch(z, "hexdb") ){ + p->openMode = SHELL_OPEN_HEXDB; +#endif /* SQLITE_ENABLE_DESERIALIZE */ }else if( z[0]=='-' ){ utf8_printf(stderr, "unknown option: %s\n", z); rc = 1; @@ -6758,7 +6854,7 @@ static int do_meta_command(char *zLine, ShellState *p){ } /* If a filename is specified, try to open it first */ zNewFilename = nArg>iName ? sqlite3_mprintf("%s", azArg[iName]) : 0; - if( zNewFilename ){ + if( zNewFilename || p->openMode==SHELL_OPEN_HEXDB ){ if( newFlag ) shellDeleteFile(zNewFilename); p->zDbFilename = zNewFilename; open_db(p, OPEN_DB_KEEPALIVE); diff --git a/tool/dbtotxt.c b/tool/dbtotxt.c new file mode 100644 index 0000000000..f28e209ce2 --- /dev/null +++ b/tool/dbtotxt.c @@ -0,0 +1,140 @@ +/* +** Copyright 2008 D. Richard Hipp and Hipp, Wyrick & Company, Inc. +** All Rights Reserved +** +****************************************************************************** +** +** This file implements a stand-alone utility program that converts +** a binary file (usually an SQLite database) into a text format that +** is compact and friendly to human-readers. +** +** Usage: +** +** dbtotxt [--pagesize N] FILENAME +** +** The translation of the database appears on standard output. If the +** --pagesize command-line option is omitted, then the page size is taken +** from the database header. +** +** Compactness is achieved by suppressing lines of all zero bytes. This +** works well at compressing test databases that are mostly empty. But +** the output will probably be lengthy for a real database containing lots +** of real content. For maximum compactness, it is suggested that test +** databases be constructed with "zeroblob()" rather than "randomblob()" +** used for filler content and with "PRAGMA secure_delete=ON" selected to +** zero-out deleted content. +*/ +#include +#include +#include + +/* Return true if the line is all zeros */ +static int allZero(unsigned char *aLine){ + int i; + for(i=0; i<16 && aLine[i]==0; i++){} + return i==16; +} + +int main(int argc, char **argv){ + int pgsz = 0; /* page size */ + long szFile; /* Size of the input file in bytes */ + FILE *in; /* Input file */ + int i, j; /* Loop counters */ + int nErr = 0; /* Number of errors */ + const char *zInputFile = 0; /* Name of the input file */ + const char *zBaseName = 0; /* Base name of the file */ + int lastPage = 0; /* Last page number shown */ + int iPage; /* Current page number */ + unsigned char aLine[16]; /* A single line of the file */ + unsigned char aHdr[100]; /* File header */ + for(i=1; i65536 || (pgsz&(pgsz-1))!=0 ){ + fprintf(stderr, "Page size must be a power of two between" + " 512 and 65536.\n"); + nErr++; + } + continue; + } + fprintf(stderr, "Unknown option: %s\n", argv[i]); + nErr++; + }else if( zInputFile ){ + fprintf(stderr, "Already using a different input file: [%s]\n", argv[i]); + nErr++; + }else{ + zInputFile = argv[i]; + } + } + if( zInputFile==0 ){ + fprintf(stderr, "No input file specified.\n"); + nErr++; + } + if( nErr ){ + fprintf(stderr, "Usage: %s [--pagesize N] FILENAME\n", argv[0]); + exit(1); + } + in = fopen(zInputFile, "rb"); + if( in==0 ){ + fprintf(stderr, "Cannot open input file [%s]\n", zInputFile); + exit(1); + } + fseek(in, 0, SEEK_END); + szFile = ftell(in); + rewind(in); + if( szFile<512 ){ + fprintf(stderr, "File too short. Minimum size is 512 bytes.\n"); + exit(1); + } + if( fread(aHdr, 100, 1, in)!=1 ){ + fprintf(stderr, "Cannot read file header\n"); + exit(1); + } + rewind(in); + if( pgsz==0 ){ + pgsz = (aHdr[16]<<8) | aHdr[17]; + if( pgsz==1 ) pgsz = 65536; + if( pgsz<512 || (pgsz&(pgsz-1))!=0 ){ + fprintf(stderr, "Invalid page size in header: %d\n", pgsz); + exit(1); + } + } + zBaseName = zInputFile; + for(i=0; zInputFile[i]; i++){ + if( zInputFile[i]=='/' && zInputFile[i+1]!=0 ) zBaseName = zInputFile+1; + } + printf("| size %d pagesize %d filename %s\n",(int)szFile,pgsz,zBaseName); + for(i=0; i=0x20 && c<=0x7e ? c : '.', stdout); + } + fputc('\n', stdout); + } + fclose(in); + printf("| end %s\n", zBaseName); + return 0; +} diff --git a/tool/dbtotxt.md b/tool/dbtotxt.md new file mode 100644 index 0000000000..f2bd7c9c25 --- /dev/null +++ b/tool/dbtotxt.md @@ -0,0 +1,56 @@ +

The dbtotxt Tool

+ +The dbtotxt utility program reads an SQLite database file and writes its +raw binary content to screen as a hex dump for testing and debugging +purposes. + +The hex-dump output is formatted in such a way as to be easily readable +both by humans and by software. The dbtotxt utility has long been a part +of the TH3 test suite. The output of dbtotxt can be embedded in TH3 test +scripts and used to generate very specific database files, perhaps with +deliberately introduced corruption. The cov1/corrupt*.test modules in +TH3 make extensive use of dbtotxt. + +More recently (2018-12-13) the dbtotxt utility has been added to the SQLite +core and the command-line shell (CLI) has been augmented to be able to read +dbtotxt output. The CLI dot-command is: + +> .open --hexdb ?OPTIONAL-FILENAME? + +If the OPTIONAL-FILENAME is included, then content is read from that file. +If OPTIONAL-FILENAME is omitted, then the text is taken from the input stream, +terminated by the "| end" line of the dbtotxt text. This allows small test +databases to be embedded directly in scripts. Consider this example: + +> + .open --hexdb + | size 8192 pagesize 4096 filename x9.db + | page 1 offset 0 + | 0: 53 51 4c 69 74 65 20 66 6f 72 6d 61 74 20 33 00 SQLite format 3. + | 16: 10 00 01 01 00 40 20 20 00 00 00 04 00 00 00 02 .....@ ........ + | 32: 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 04 ................ + | 48: 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 ................ + | 80: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 04 ................ + | 96: 00 2e 30 38 0d 00 00 00 01 0f c0 00 0f c0 00 00 ..08............ + | 4032: 3e 01 06 17 11 11 01 69 74 61 62 6c 65 74 31 74 >......itablet1t + | 4048: 31 02 43 52 45 41 54 45 20 54 41 42 4c 45 20 74 1.CREATE TABLE t + | 4064: 31 28 78 2c 79 20 44 45 46 41 55 4c 54 20 78 27 1(x,y DEFAULT x' + | 4080: 66 66 27 2c 7a 20 44 45 46 41 55 4c 54 20 30 29 ff',z DEFAULT 0) + | page 2 offset 4096 + | 0: 0d 08 14 00 04 00 10 00 0e 05 0a 0f 04 15 00 10 ................ + | 16: 88 02 03 05 90 04 0e 08 00 00 00 00 00 00 00 00 ................ + | 1040: 00 00 00 00 ff 87 7c 02 05 8f 78 0e 08 00 00 00 ......|...x..... + | 2064: 00 00 00 ff 0c 0a 01 fb 00 00 00 00 00 00 00 00 ................ + | 2560: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 83 ................ + | 2576: 78 01 05 87 70 0e 08 00 00 00 00 00 00 00 00 00 x...p........... + | 3072: 00 00 00 00 00 00 00 00 00 ff 00 00 01 fb 00 00 ................ + | 3584: 00 00 00 00 00 83 78 00 05 87 70 0e 08 00 00 00 ......x...p..... + | 4080: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ff ................ + | end x9.db + SELECT rowid FROM t1; + PRAGMA integrity_check; + +You can run this script to see that the database file is correctly decoded +and loaded. Furthermore, you can make subtle corruptions to the input +database simply by editing the hexadecimal description, then rerun the +script to verify that SQLite correctly handles the corruption.