From: drh Date: Mon, 9 Mar 2020 15:39:39 +0000 (+0000) Subject: Enhancements to the ".import" command of the CLI. X-Git-Tag: version-3.32.0~126 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=ccb37816731acf41cb60648c0eb73ced069a8915;p=thirdparty%2Fsqlite.git Enhancements to the ".import" command of the CLI. FossilOrigin-Name: cab1834cfc71f71bfed3c5170a0ba40a39385c3b2c50b7c6b6f09cc830dd1b1e --- diff --git a/manifest b/manifest index 6c007c55b5..04435eea73 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\stypos\sin\sRowSet. -D 2020-03-09T03:21:33.427 +C Enhancements\sto\sthe\s".import"\scommand\sof\sthe\sCLI. +D 2020-03-09T15:39:39.029 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -532,7 +532,7 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 F src/resolve.c 38e3a5636f5bdc92e3683e4cafbba6418c0aa15e0d89ca5b28bd0b621dbb80bf F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 F src/select.c c94eec317c8ba929bc228392eb3cac8124f2d0fbe3fc1bddecb44dfc7057bc78 -F src/shell.c.in 3897f3f7302914da1f6df3a2a09ac4aafa14a571d7d18c51500cfb2ff04f05eb +F src/shell.c.in f76590931c0cbbfef347f44f81ade6b335f80c46bc6e59b8b6114383a8df30e0 F src/sqlite.h.in 802957feeb249ede54f8dfe99b72aa19e70a0b7737969c46e625dc2f9f2d42b0 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 9c5269260409eb3275324ccace6a13a96f4ad330c708415f70ca6097901ff4ee @@ -1333,11 +1333,11 @@ F test/sharedA.test 49d87ec54ab640fbbc3786ee3c01de94aaa482a3a9f834ad3fe92770eb69 F test/sharedB.test 16cc7178e20965d75278f410943109b77b2e645e F test/shared_err.test 32634e404a3317eeb94abc7a099c556a346fdb8fb3858dbe222a4cbb8926a939 F test/sharedlock.test 5ede3c37439067c43b0198f580fd374ebf15d304 -F test/shell1.test 3c9707dce15e8fdca529503378660f099777d3ddcedccf801a37589a405c5942 +F test/shell1.test 43e12c7d4ff65f041803ad24a93fd3818deb2cb83e721810f27d0fde61d64a13 F test/shell2.test e242a9912f44f4c23c3d1d802a83e934e84c853b F test/shell3.test ac8c2b744014c3e9a0e26bfd829ab65f00923dc1a91ffd044863e9423cc91494 F test/shell4.test 1c6aef11daaa2d6830acaba3ac9cbec93fbc1c3d5530743a637f39b3987d08ce -F test/shell5.test 23939a4c51f0421330ea61dbd3c74f9c215f5f8d3d1a94846da6ffc777a35458 +F test/shell5.test 84a30b55722a95a5b72989e691c469a999ca7591e7aa00b7fabc783ea5c9a6fe F test/shell6.test 1ceb51b2678c472ba6cf1e5da96679ce8347889fe2c3bf93a0e0fa73f00b00d3 F test/shell7.test 115132f66d0463417f408562cc2cf534f6bbc6d83a6d50f0072a9eb171bae97f F test/shell8.test 96be02ea0c21f05b24c1883d7b711a1fa8525a68ab7b636aacf6057876941013 @@ -1860,7 +1860,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 35f1f151ac478d6b46f3685d2565c35108ef74bd33ce96fb65300d3c303b289b -R 3ad66a51f32de0dab569042c4aa4c3c6 -U pdr -Z ab82beefd8209a83b9f274edf5605106 +P 86465c08f4d629a296332a7985937326ac43ea2822c5651bf03862cd79d370fc +R 59ca84b4f9c863c565a73d91cc01fad5 +U drh +Z ca4df02a4f4403d62ad30a31c55381b8 diff --git a/manifest.uuid b/manifest.uuid index 7deff47aa2..39e7d10901 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -86465c08f4d629a296332a7985937326ac43ea2822c5651bf03862cd79d370fc \ No newline at end of file +cab1834cfc71f71bfed3c5170a0ba40a39385c3b2c50b7c6b6f09cc830dd1b1e \ No newline at end of file diff --git a/src/shell.c.in b/src/shell.c.in index 149ba7ee1d..728a4d051d 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -3575,6 +3575,18 @@ static const char *(azHelp[]) = { ".headers on|off Turn display of headers on or off", ".help ?-all? ?PATTERN? Show help text for PATTERN", ".import FILE TABLE Import data from FILE into TABLE", + " Options:", + " --ascii Use \\037 and \\036 as column and row separators", + " --csv Use , and \\n as column and row separators", + " --skip N Skip the first N rows of input", + " -v \"Verbose\" - increase auxiliary output", + " Notes:", + " * If TABLE does not exist, it is created. The first row of input", + " determines the column names.", + " * If neither --csv or --ascii are used, the input mode is derived", + " from the \".mode\" output mode", + " * If FILE begins with \"|\" then it is a command that generates the", + " input text.", #ifndef SQLITE_OMIT_TEST_CONTROL ".imposter INDEX TABLE Create imposter table TABLE on index INDEX", #endif @@ -4563,6 +4575,8 @@ struct ImportCtx { int n; /* Number of bytes in z */ int nAlloc; /* Space allocated for z[] */ int nLine; /* Current line number */ + int nRow; /* Number of rows imported */ + int nErr; /* Number of errors encountered */ int bNotFirst; /* True if one or more bytes already read */ int cTerm; /* Character that terminated the most recent field */ int cColSep; /* The column separator character. (Usually ",") */ @@ -7623,8 +7637,8 @@ static int do_meta_command(char *zLine, ShellState *p){ }else if( c=='i' && strncmp(azArg[0], "import", n)==0 ){ - char *zTable; /* Insert data into this table */ - char *zFile; /* Name of file to extra content from */ + char *zTable = 0; /* Insert data into this table */ + char *zFile = 0; /* Name of file to extra content from */ sqlite3_stmt *pStmt = NULL; /* A statement */ int nCol; /* Number of columns in the table */ int nByte; /* Number of bytes in an SQL string */ @@ -7635,51 +7649,108 @@ static int do_meta_command(char *zLine, ShellState *p){ ImportCtx sCtx; /* Reader context */ char *(SQLITE_CDECL *xRead)(ImportCtx*); /* Func to read one value */ int (SQLITE_CDECL *xCloser)(FILE*); /* Func to close file */ + int eVerbose = 0; /* Larger for more console output */ + int nSkip = 0; /* Initial lines to skip */ + int useOutputMode = 1; /* Use output mode to determine separators */ - if( nArg!=3 ){ - raw_printf(stderr, "Usage: .import FILE TABLE\n"); - goto meta_command_exit; - } - zFile = azArg[1]; - zTable = azArg[2]; - seenInterrupt = 0; memset(&sCtx, 0, sizeof(sCtx)); - open_db(p, 0); - nSep = strlen30(p->colSeparator); - if( nSep==0 ){ - raw_printf(stderr, - "Error: non-null column separator required for import\n"); - return 1; + if( p->mode==MODE_Ascii ){ + xRead = ascii_read_one_field; + }else{ + xRead = csv_read_one_field; } - if( nSep>1 ){ - raw_printf(stderr, "Error: multi-character column separators not allowed" - " for import\n"); - return 1; + for(i=1; iout, "ERROR: extra argument: \"%s\". Usage:\n", z); + showHelp(p->out, "import"); + rc = 1; + goto meta_command_exit; + } + }else if( strcmp(z,"-v")==0 ){ + eVerbose++; + }else if( strcmp(z,"-skip")==0 && iout, "ERROR: unknown option: \"%s\". Usage:\n", z); + showHelp(p->out, "import"); + rc = 1; + goto meta_command_exit; + } } - nSep = strlen30(p->rowSeparator); - if( nSep==0 ){ - raw_printf(stderr, "Error: non-null row separator required for import\n"); - return 1; + if( zTable==0 ){ + utf8_printf(p->out, "ERROR: missing %s argument. Usage:\n", + zFile==0 ? "FILE" : "TABLE"); + showHelp(p->out, "import"); + rc = 1; + goto meta_command_exit; } - if( nSep==2 && p->mode==MODE_Csv && strcmp(p->rowSeparator, SEP_CrLf)==0 ){ - /* When importing CSV (only), if the row separator is set to the - ** default output row separator, change it to the default input - ** row separator. This avoids having to maintain different input - ** and output row separators. */ - sqlite3_snprintf(sizeof(p->rowSeparator), p->rowSeparator, SEP_Row); + seenInterrupt = 0; + open_db(p, 0); + if( useOutputMode ){ + /* If neither the --csv or --ascii options are specified, then set + ** the column and row separator characters from the output mode. */ + nSep = strlen30(p->colSeparator); + if( nSep==0 ){ + raw_printf(stderr, + "Error: non-null column separator required for import\n"); + rc = 1; + goto meta_command_exit; + } + if( nSep>1 ){ + raw_printf(stderr, + "Error: multi-character column separators not allowed" + " for import\n"); + rc = 1; + goto meta_command_exit; + } nSep = strlen30(p->rowSeparator); - } - if( nSep>1 ){ - raw_printf(stderr, "Error: multi-character row separators not allowed" - " for import\n"); - return 1; + if( nSep==0 ){ + raw_printf(stderr, + "Error: non-null row separator required for import\n"); + rc = 1; + goto meta_command_exit; + } + if( nSep==2 && p->mode==MODE_Csv && strcmp(p->rowSeparator,SEP_CrLf)==0 ){ + /* When importing CSV (only), if the row separator is set to the + ** default output row separator, change it to the default input + ** row separator. This avoids having to maintain different input + ** and output row separators. */ + sqlite3_snprintf(sizeof(p->rowSeparator), p->rowSeparator, SEP_Row); + nSep = strlen30(p->rowSeparator); + } + if( nSep>1 ){ + raw_printf(stderr, "Error: multi-character row separators not allowed" + " for import\n"); + rc = 1; + goto meta_command_exit; + } + sCtx.cColSep = p->colSeparator[0]; + sCtx.cRowSep = p->rowSeparator[0]; } sCtx.zFile = zFile; sCtx.nLine = 1; if( sCtx.zFile[0]=='|' ){ #ifdef SQLITE_OMIT_POPEN raw_printf(stderr, "Error: pipes are not supported in this OS\n"); - return 1; + rc = 1; + goto meta_command_exit; #else sCtx.in = popen(sCtx.zFile+1, "r"); sCtx.zFile = ""; @@ -7689,17 +7760,26 @@ static int do_meta_command(char *zLine, ShellState *p){ sCtx.in = fopen(sCtx.zFile, "rb"); xCloser = fclose; } - if( p->mode==MODE_Ascii ){ - xRead = ascii_read_one_field; - }else{ - xRead = csv_read_one_field; - } if( sCtx.in==0 ){ utf8_printf(stderr, "Error: cannot open \"%s\"\n", zFile); - return 1; + rc = 1; + goto meta_command_exit; + } + if( eVerbose>=2 || (eVerbose>=1 && useOutputMode) ){ + char zSep[2]; + zSep[1] = 0; + zSep[0] = sCtx.cColSep; + utf8_printf(p->out, "Column separator "); + output_c_string(p->out, zSep); + utf8_printf(p->out, ", row separator "); + zSep[0] = sCtx.cRowSep; + output_c_string(p->out, zSep); + utf8_printf(p->out, "\n"); + } + while( (nSkip--)>0 ){ + while( xRead(&sCtx) && sCtx.cTerm==sCtx.cColSep ){} + sCtx.nLine++; } - sCtx.cColSep = p->colSeparator[0]; - sCtx.cRowSep = p->rowSeparator[0]; zSql = sqlite3_mprintf("SELECT * FROM %s", zTable); if( zSql==0 ){ xCloser(sCtx.in); @@ -7721,9 +7801,13 @@ static int do_meta_command(char *zLine, ShellState *p){ sqlite3_free(sCtx.z); xCloser(sCtx.in); utf8_printf(stderr,"%s: empty file\n", sCtx.zFile); - return 1; + rc = 1; + goto meta_command_exit; } zCreate = sqlite3_mprintf("%z\n)", zCreate); + if( eVerbose>=1 ){ + utf8_printf(p->out, "%s\n", zCreate); + } rc = sqlite3_exec(p->db, zCreate, 0, 0, 0); sqlite3_free(zCreate); if( rc ){ @@ -7731,7 +7815,8 @@ static int do_meta_command(char *zLine, ShellState *p){ sqlite3_errmsg(p->db)); sqlite3_free(sCtx.z); xCloser(sCtx.in); - return 1; + rc = 1; + goto meta_command_exit; } rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0); } @@ -7740,7 +7825,8 @@ static int do_meta_command(char *zLine, ShellState *p){ if (pStmt) sqlite3_finalize(pStmt); utf8_printf(stderr,"Error: %s\n", sqlite3_errmsg(p->db)); xCloser(sCtx.in); - return 1; + rc = 1; + goto meta_command_exit; } nCol = sqlite3_column_count(pStmt); sqlite3_finalize(pStmt); @@ -7759,13 +7845,17 @@ static int do_meta_command(char *zLine, ShellState *p){ } zSql[j++] = ')'; zSql[j] = 0; + if( eVerbose>=2 ){ + utf8_printf(p->out, "Insert using: %s\n", zSql); + } rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0); sqlite3_free(zSql); if( rc ){ utf8_printf(stderr, "Error: %s\n", sqlite3_errmsg(p->db)); if (pStmt) sqlite3_finalize(pStmt); xCloser(sCtx.in); - return 1; + rc = 1; + goto meta_command_exit; } needCommit = sqlite3_get_autocommit(p->db); if( needCommit ) sqlite3_exec(p->db, "BEGIN", 0, 0, 0); @@ -7808,6 +7898,9 @@ static int do_meta_command(char *zLine, ShellState *p){ if( rc!=SQLITE_OK ){ utf8_printf(stderr, "%s:%d: INSERT failed: %s\n", sCtx.zFile, startLine, sqlite3_errmsg(p->db)); + sCtx.nErr++; + }else{ + sCtx.nRow++; } } }while( sCtx.cTerm!=EOF ); @@ -7816,6 +7909,11 @@ static int do_meta_command(char *zLine, ShellState *p){ sqlite3_free(sCtx.z); sqlite3_finalize(pStmt); if( needCommit ) sqlite3_exec(p->db, "COMMIT", 0, 0, 0); + if( eVerbose>0 ){ + utf8_printf(p->out, + "Added %d rows with %d errors using %d lines of input\n", + sCtx.nRow, sCtx.nErr, sCtx.nLine-1); + } }else #ifndef SQLITE_UNTESTABLE diff --git a/test/shell1.test b/test/shell1.test index 734a21a2f8..9273adeeb8 100644 --- a/test/shell1.test +++ b/test/shell1.test @@ -388,17 +388,14 @@ do_test shell1-3.10.2 { # .import FILE TABLE Import data from FILE into TABLE do_test shell1-3.11.1 { catchcmd "test.db" ".import" -} {1 {Usage: .import FILE TABLE}} +} {/1 .ERROR: missing FILE argument.*/} do_test shell1-3.11.2 { catchcmd "test.db" ".import FOO" -} {1 {Usage: .import FILE TABLE}} -#do_test shell1-3.11.2 { -# catchcmd "test.db" ".import FOO BAR" -#} {1 {Error: no such table: BAR}} +} {/1 .ERROR: missing TABLE argument.*/} do_test shell1-3.11.3 { # too many arguments catchcmd "test.db" ".import FOO BAR BAD" -} {1 {Usage: .import FILE TABLE}} +} {/1 .ERROR: extra argument: "BAD".*./} # .indexes ?TABLE? Show names of all indexes # If TABLE specified, only show indexes for tables diff --git a/test/shell5.test b/test/shell5.test index 72f20ca3a6..8ec9a632bf 100644 --- a/test/shell5.test +++ b/test/shell5.test @@ -32,17 +32,14 @@ forcedelete test.db test.db-journal test.db-wal # .import FILE TABLE Import data from FILE into TABLE do_test shell5-1.1.1 { catchcmd "test.db" ".import" -} {1 {Usage: .import FILE TABLE}} +} {/1 .ERROR: missing FILE argument.*/} do_test shell5-1.1.2 { catchcmd "test.db" ".import FOO" -} {1 {Usage: .import FILE TABLE}} -#do_test shell5-1.1.2 { -# catchcmd "test.db" ".import FOO BAR" -#} {1 {Error: no such table: BAR}} +} {/1 .ERROR: missing TABLE argument.*/} do_test shell5-1.1.3 { # too many arguments catchcmd "test.db" ".import FOO BAR BAD" -} {1 {Usage: .import FILE TABLE}} +} {/1 .ERROR: extra argument.*/} # .separator STRING Change separator used by output mode and .import do_test shell5-1.2.1 {