Update the ".import" command of the command-line shell so that it can

accept field values that span multiple lines and so that it issues
error messages if the input text does not strictly conform to RFC4180.

FossilOrigin-Name: 93f632152e464a89322a0130adaf9f342411bf7d

diff --git a/manifest b/manifest
index 5872eec583544f1e3336184b1f182f6be6058663..febb5a72c92b320810cb8f25944a1b3697f2e37c 100644 (file)
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Add\sthe\s"vtshim"\sextension,\simplementing\sa\swrapper\saround\sthe\svirtual\ntable\sinterface\sto\smake\sit\sDisposable\sfor\sthe\sCLR.\s\sNo\schanges\sto\sthe\ncore.
-D 2013-06-26T18:04:19.083
+C Update\sthe\s".import"\scommand\sof\sthe\scommand-line\sshell\sso\sthat\sit\scan\naccept\sfield\svalues\sthat\sspan\smultiple\slines\sand\sso\sthat\sit\sissues\nerror\smessages\sif\sthe\sinput\stext\sdoes\snot\sstrictly\sconform\sto\sRFC4180.
+D 2013-06-26T22:46:00.198
 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
 F Makefile.in 5e41da95d92656a5004b03d3576e8b226858a28e
 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@@ -217,7 +217,7 @@ F src/random.c cd4a67b3953b88019f8cd4ccd81394a8ddfaba50
 F src/resolve.c 89f9003e8316ee3a172795459efc2a0274e1d5a8
 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0
 F src/select.c 91b62654caf8dfe292fb8882715e575d34ad3874
-F src/shell.c a02544af6697c5782d29ec3204616f35ed9e8458
+F src/shell.c 92cbe95eadc1c423422d36beac3609a9422889d1
 F src/sqlite.h.in 5f86553f4c1d8b4a9069285ed19e7981451ea77a
 F src/sqlite3.rc fea433eb0a59f4c9393c8e6d76a6e2596b1fe0c0
 F src/sqlite3ext.h d936f797812c28b81b26ed18345baf8db28a21a5
@@ -773,11 +773,11 @@ F test/shared9.test 5f2a8f79b4d6c7d107a01ffa1ed05ae7e6333e21
 F test/sharedA.test 0cdf1a76dfa00e6beee66af5b534b1e8df2720f5
 F test/shared_err.test 0079c05c97d88cfa03989b7c20a8b266983087aa
 F test/sharedlock.test 927a4b6da11978c82b857dbdb20a932aad732123
-F test/shell1.test 4a2f57952719972c6f862134463f8712e953c038
+F test/shell1.test 338f51e6ff543720c609178bda81c2606df8df8d
 F test/shell2.test 037d6ad16e873354195d30bb2dc4b5321788154a
 F test/shell3.test 9196c42772d575685e722c92b4b39053c6ebba59
 F test/shell4.test aa4eef8118b412d1a01477a53426ece169ea86a9
-F test/shell5.test fa2188bbb13fe2d183fd04a5f7b512650c35ef5d
+F test/shell5.test 0fed7823d57e80f79da2c5f350e50aa86011f24f
 F test/shortread1.test bb591ef20f0fd9ed26d0d12e80eee6d7ac8897a3
 F test/shrink.test 8c70f62b6e8eb4d54533de6d65bd06b1b9a17868
 F test/sidedelete.test f0ad71abe6233e3b153100f3b8d679b19a488329
@@ -1098,7 +1098,7 @@ F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381
 F tool/wherecosttest.c f407dc4c79786982a475261866a161cd007947ae
 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac
-P ebac5afa471526dffc8026e66753263476137a3b b4a0d5327addd90bef758e6a1403ac69f61b3886
-R a3237e173c634535eb67beba24018ada
+P 6c3839ef311a53076650c6479c932e545a26b96f
+R 20f28da1c568459e4d7e332d42754a7d
 U drh
-Z 356f790fa566cdf1861055fde4a499e2
+Z a56fe80d4e15f5a77335d76e21c16046

diff --git a/manifest.uuid b/manifest.uuid
index ac526c0c7891b798d02bc654849e401433257cb5..2ca371dd708e9ddc8b278f12d57ae3cc32ce5cf7 100644 (file)
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-6c3839ef311a53076650c6479c932e545a26b96f
\ No newline at end of file
+93f632152e464a89322a0130adaf9f342411bf7d
\ No newline at end of file

diff --git a/src/shell.c b/src/shell.c
index 9a8a906944adf99e1c602b7d3cd9aca6ee04152d..e66125b23a67c53df65850dfe14fc5e46d9066e7 100644 (file)
--- a/src/shell.c
+++ b/src/shell.c
@@ -1650,6 +1650,107 @@ static void test_breakpoint(void){
   nCall++;
 }
 
+/*
+** An object used to read a CSV file
+*/
+typedef struct CSVReader CSVReader;
+struct CSVReader {
+  const char *zFile;  /* Name of the input file */
+  FILE *in;           /* Read the CSV text from this input stream */
+  char *z;            /* Accumulated text for a field */
+  int n;              /* Number of bytes in z */
+  int nAlloc;         /* Space allocated for z[] */
+  int nLine;          /* Current line number */
+  int cTerm;          /* Character that terminated the most recent field */
+  int cSeparator;     /* The separator character.  (Usually ",") */
+};
+
+/* Append a single byte to z[] */
+static void csv_append_char(CSVReader *p, int c){
+  if( p->n+1>=p->nAlloc ){
+    p->nAlloc += p->nAlloc + 100;
+    p->z = sqlite3_realloc(p->z, p->nAlloc);
+    if( p->z==0 ){
+      fprintf(stderr, "out of memory\n");
+      exit(1);
+    }
+  }
+  p->z[p->n++] = (char)c;
+}
+
+/* Read a single field of CSV text.  Compatible with rfc4180 and extended
+** with the option of having a separator other than ",".
+**
+**   +  Input comes from p->in.
+**   +  Store results in p->z of length p->n.  Space to hold p->z comes
+**      from sqlite3_malloc().
+**   +  Use p->cSep as the separator.  The default is ",".
+**   +  Keep track of the line number in p->nLine.
+**   +  Store the character that terminates the field in p->cTerm.  Store
+**      EOF on end-of-file.
+**   +  Report syntax errors on stderr
+*/
+static char *csv_read_one_field(CSVReader *p){
+  int c, pc;
+  int cSep = p->cSeparator;
+  p->n = 0;
+  c = fgetc(p->in);
+  if( c==EOF || seenInterrupt ){
+    p->cTerm = EOF;
+    return 0;
+  }
+  if( c=='"' ){
+    int startLine = p->nLine;
+    int cQuote = c;
+    pc = 0;
+    while( 1 ){
+      c = fgetc(p->in);
+      if( c=='\n' ) p->nLine++;
+      if( c==cQuote ){
+        if( pc==cQuote ){
+          pc = 0;
+          continue;
+        }
+      }
+      if( (c==cSep && pc==cQuote)
+       || (c=='\n' && pc==cQuote)
+       || (c=='\n' && pc=='\r' && p->n>2 && p->z[p->n-2]==cQuote)
+       || (c==EOF && pc==cQuote)
+      ){
+        do{ p->n--; }while( p->z[p->n]!=cQuote );
+        p->z[p->n] = 0;
+        p->cTerm = c;
+        break;
+      }
+      if( pc==cQuote && c!='\r' ){
+        fprintf(stderr, "%s:%d: unescaped %c character\n",
+                p->zFile, p->nLine, cQuote);
+      }
+      if( c==EOF ){
+        fprintf(stderr, "%s:%d: unterminated %c-quoted field\n",
+                p->zFile, startLine, cQuote);
+        p->z[p->n] = 0;
+        p->cTerm = EOF;
+        break;
+      }
+      csv_append_char(p, c);
+      pc = c;
+    }      
+  }else{
+    csv_append_char(p, c);
+    while( (c = fgetc(p->in))!=EOF && c!=cSep && c!='\n' ){
+      csv_append_char(p, c);
+    }
+    if( c=='\n' ){
+      p->nLine++;
+      if( p->n>1 && p->z[p->n-1]=='\r' ) p->n--;
+    }
+    p->z[p->n] = 0;
+    p->cTerm = c;
+  }
+  return p->z;
+}
+
 /*
 ** If an input line begins with "." then invoke this routine to
 ** process that line.
@@ -1888,48 +1989,81 @@ static int do_meta_command(char *zLine, struct callback_data *p){
 
   if( c=='i' && strncmp(azArg[0], "import", n)==0 && nArg==3 ){
     char *zTable = azArg[2];    /* Insert data into this table */
-    char *zFile = azArg[1];     /* The file from which to extract data */
     sqlite3_stmt *pStmt = NULL; /* A statement */
     int nCol;                   /* Number of columns in the table */
     int nByte;                  /* Number of bytes in an SQL string */
     int i, j;                   /* Loop counters */
     int nSep;                   /* Number of bytes in p->separator[] */
     char *zSql;                 /* An SQL statement */
-    char *zLine;                /* A single line of input from the file */
-    char **azCol;               /* zLine[] broken up into columns */
-    char *zCommit;              /* How to commit changes */   
-    FILE *in;                   /* The input file */
-    int lineno = 0;             /* Line number of input file */
+    CSVReader sCsv;             /* Reader context */
 
+    seenInterrupt = 0;
+    memset(&sCsv, 0, sizeof(sCsv));
+    sCsv.zFile = azArg[1];
+    sCsv.nLine = 1;
     open_db(p);
     nSep = strlen30(p->separator);
     if( nSep==0 ){
       fprintf(stderr, "Error: non-null separator required for import\n");
       return 1;
     }
+    if( nSep>1 ){
+      fprintf(stderr, "Error: multi-character separators not allowed"
+                      " for import\n");
+      return 1;
+    }
+    sCsv.in = fopen(sCsv.zFile, "rb");
+    if( sCsv.in==0 ){
+      fprintf(stderr, "Error: cannot open \"%s\"\n", sCsv.zFile);
+      return 1;
+    }
+    sCsv.cSeparator = p->separator[0];
     zSql = sqlite3_mprintf("SELECT * FROM %s", zTable);
     if( zSql==0 ){
       fprintf(stderr, "Error: out of memory\n");
+      fclose(sCsv.in);
       return 1;
     }
     nByte = strlen30(zSql);
     rc = sqlite3_prepare(p->db, zSql, -1, &pStmt, 0);
+    if( rc && sqlite3_strglob("no such table: *", sqlite3_errmsg(db))==0 ){
+      char *zCreate = sqlite3_mprintf("CREATE TABLE %s", zTable);
+      char cSep = '(';
+      while( csv_read_one_field(&sCsv) ){
+        zCreate = sqlite3_mprintf("%z%c\n  \"%s\" TEXT", zCreate, cSep, sCsv.z);
+        cSep = ',';
+        if( sCsv.cTerm!=sCsv.cSeparator ) break;
+      }
+      zCreate = sqlite3_mprintf("%z\n)", zCreate);
+      rc = sqlite3_exec(p->db, zCreate, 0, 0, 0);
+      sqlite3_free(zCreate);
+      if( rc ){
+        fprintf(stderr, "CREATE TABLE %s(...) failed: %s\n", zTable,
+                sqlite3_errmsg(db));
+        sqlite3_free(sCsv.z);
+        fclose(sCsv.in);
+        return 1;
+      }
+      rc = sqlite3_prepare(p->db, zSql, -1, &pStmt, 0);
+    }
     sqlite3_free(zSql);
     if( rc ){
       if (pStmt) sqlite3_finalize(pStmt);
       fprintf(stderr,"Error: %s\n", sqlite3_errmsg(db));
+      fclose(sCsv.in);
       return 1;
     }
     nCol = sqlite3_column_count(pStmt);
     sqlite3_finalize(pStmt);
     pStmt = 0;
     if( nCol==0 ) return 0; /* no columns, no error */
-    zSql = malloc( nByte + 20 + nCol*2 );
+    zSql = sqlite3_malloc( nByte*2 + 20 + nCol*2 );
     if( zSql==0 ){
       fprintf(stderr, "Error: out of memory\n");
+      fclose(sCsv.in);
       return 1;
     }
-    sqlite3_snprintf(nByte+20, zSql, "INSERT INTO %s VALUES(?", zTable);
+    sqlite3_snprintf(nByte+20, zSql, "INSERT INTO \"%w\" VALUES(?", zTable);
     j = strlen30(zSql);
     for(i=1; i<nCol; i++){
       zSql[j++] = ',';
@@ -1938,79 +2072,50 @@ static int do_meta_command(char *zLine, struct callback_data *p){
     zSql[j++] = ')';
     zSql[j] = 0;
     rc = sqlite3_prepare(p->db, zSql, -1, &pStmt, 0);
-    free(zSql);
+    sqlite3_free(zSql);
     if( rc ){
       fprintf(stderr, "Error: %s\n", sqlite3_errmsg(db));
       if (pStmt) sqlite3_finalize(pStmt);
+      fclose(sCsv.in);
       return 1;
     }
-    in = fopen(zFile, "rb");
-    if( in==0 ){
-      fprintf(stderr, "Error: cannot open \"%s\"\n", zFile);
-      sqlite3_finalize(pStmt);
-      return 1;
-    }
-    azCol = malloc( sizeof(azCol[0])*(nCol+1) );
-    if( azCol==0 ){
-      fprintf(stderr, "Error: out of memory\n");
-      fclose(in);
-      sqlite3_finalize(pStmt);
-      return 1;
-    }
-    sqlite3_exec(p->db, "BEGIN", 0, 0, 0);
-    zCommit = "COMMIT";
-    while( (zLine = local_getline(0, in, 1))!=0 ){
-      char *z, c;
-      int inQuote = 0;
-      lineno++;
-      azCol[0] = zLine;
-      for(i=0, z=zLine; (c = *z)!=0; z++){
-        if( c=='"' ) inQuote = !inQuote;
-        if( c=='\n' ) lineno++;
-        if( !inQuote && c==p->separator[0] && strncmp(z,p->separator,nSep)==0 ){
-          *z = 0;
+    do{
+      int startLine = sCsv.nLine;
+      for(i=0; i<nCol; i++){
+        char *z = csv_read_one_field(&sCsv);
+        if( z==0 && i==0 ) break;
+        sqlite3_bind_text(pStmt, i+1, z, -1, SQLITE_TRANSIENT);
+        if( i<nCol-1 && sCsv.cTerm!=sCsv.cSeparator ){
+          fprintf(stderr, "%s:%d: expected %d columns but found %d - "
+                          "filling the rest with NULL\n",
+                          sCsv.zFile, startLine, nCol, i+1);
           i++;
-          if( i<nCol ){
-            azCol[i] = &z[nSep];
-            z += nSep-1;
-          }
+          while( i<nCol ){ sqlite3_bind_null(pStmt, i); i++; }
         }
-      } /* end for */
-      *z = 0;
-      if( i+1!=nCol ){
-        fprintf(stderr,
-                "Error: %s line %d: expected %d columns of data but found %d\n",
-                zFile, lineno, nCol, i+1);
-        zCommit = "ROLLBACK";
-        free(zLine);
-        rc = 1;
-        break; /* from while */
       }
-      for(i=0; i<nCol; i++){
-        if( azCol[i][0]=='"' ){
-          int k;
-          for(z=azCol[i], j=1, k=0; z[j]; j++){
-            if( z[j]=='"' ){ j++; if( z[j]==0 ) break; }
-            z[k++] = z[j];
-          }
-          z[k] = 0;
+      if( sCsv.cTerm==sCsv.cSeparator ){
+        do{
+          csv_read_one_field(&sCsv);
+          i++;
+        }while( sCsv.cTerm==sCsv.cSeparator );
+        fprintf(stderr, "%s:%d: expected %d columns but found %d - "
+                        "extras ignored\n",
+                        sCsv.zFile, startLine, nCol, i);
+      }
+      if( i>=nCol ){
+        sqlite3_step(pStmt);
+        rc = sqlite3_reset(pStmt);
+        if( rc!=SQLITE_OK ){
+          fprintf(stderr, "%s:%d: INSERT failed: %s\n", sCsv.zFile, startLine,
+                  sqlite3_errmsg(db));
         }
-        sqlite3_bind_text(pStmt, i+1, azCol[i], -1, SQLITE_STATIC);
-      }
-      sqlite3_step(pStmt);
-      rc = sqlite3_reset(pStmt);
-      free(zLine);
-      if( rc!=SQLITE_OK ){
-        fprintf(stderr,"Error: %s\n", sqlite3_errmsg(db));
-        zCommit = "ROLLBACK";
-        rc = 1;
-        break; /* from while */
       }
-    } /* end while */
-    free(azCol);
-    fclose(in);
+    }while( sCsv.cTerm!=EOF );
+
+    fclose(sCsv.in);
+    sqlite3_free(sCsv.z);
     sqlite3_finalize(pStmt);
-    sqlite3_exec(p->db, zCommit, 0, 0, 0);
+    sqlite3_exec(p->db, "COMMIT", 0, 0, 0);
   }else
 
   if( c=='i' && strncmp(azArg[0], "indices", n)==0 && nArg<3 ){

diff --git a/test/shell1.test b/test/shell1.test
index c60f3af7026a02e11863be5229471ecb7ec2bd09..798a7e696fd6ac0d526506c5a2c486b3377be590 100644 (file)
--- a/test/shell1.test
+++ b/test/shell1.test
@@ -397,9 +397,9 @@ do_test shell1-3.11.1 {
 do_test shell1-3.11.2 {
   catchcmd "test.db" ".import FOO"
 } {1 {Error: unknown command or invalid arguments:  "import". Enter ".help" for help}}
-do_test shell1-3.11.2 {
-  catchcmd "test.db" ".import FOO BAR"
-} {1 {Error: no such table: BAR}}
+#do_test shell1-3.11.2 {
+#  catchcmd "test.db" ".import FOO BAR"
+#} {1 {Error: no such table: BAR}}
 do_test shell1-3.11.3 {
   # too many arguments
   catchcmd "test.db" ".import FOO BAR BAD"

diff --git a/test/shell5.test b/test/shell5.test
index d90cedf949ef834d9b41cf19c453c164925dd8dd..343e4cfcd1666696ed3c18cd1569f4522503c9c0 100644 (file)
--- a/test/shell5.test
+++ b/test/shell5.test
@@ -45,9 +45,9 @@ do_test shell5-1.1.1 {
 do_test shell5-1.1.2 {
   catchcmd "test.db" ".import FOO"
 } {1 {Error: unknown command or invalid arguments:  "import". Enter ".help" for help}}
-do_test shell5-1.1.2 {
-  catchcmd "test.db" ".import FOO BAR"
-} {1 {Error: no such table: BAR}}
+#do_test shell5-1.1.2 {
+#  catchcmd "test.db" ".import FOO BAR"
+#} {1 {Error: no such table: BAR}}
 do_test shell5-1.1.3 {
   # too many arguments
   catchcmd "test.db" ".import FOO BAR BAD"
@@ -101,7 +101,7 @@ do_test shell5-1.4.3 {
   puts $in "1"
   close $in
   set res [catchcmd "test.db" {.import shell5.csv t1}]
-} {1 {Error: shell5.csv line 1: expected 2 columns of data but found 1}}
+} {1 {shell5.csv:1: expected 2 columns but found 1 - filling the rest with NULL}}
 
 # import file with 1 row, 3 columns (expecting 2 cols)
 do_test shell5-1.4.4 {
@@ -109,14 +109,15 @@ do_test shell5-1.4.4 {
   puts $in "1|2|3"
   close $in
   set res [catchcmd "test.db" {.import shell5.csv t1}]
-} {1 {Error: shell5.csv line 1: expected 2 columns of data but found 3}}
+} {1 {shell5.csv:1: expected 2 columns but found 3 - extras ignored}}
 
 # import file with 1 row, 2 columns
 do_test shell5-1.4.5 {
   set in [open shell5.csv w]
   puts $in "1|2"
   close $in
-  set res [catchcmd "test.db" {.import shell5.csv t1
+  set res [catchcmd "test.db" {DELETE FROM t1;
+.import shell5.csv t1
 SELECT COUNT(*) FROM t1;}]
 } {0 1}
 
@@ -197,15 +198,15 @@ SELECT length(b) FROM t1 WHERE a='8';}]
 # This is limited by SQLITE_MAX_VARIABLE_NUMBER, which defaults to 999.
 set cols 999
 do_test shell5-1.6.1 {
-  set sql {CREATE TABLE t2(}
   set data {}
   for {set i 1} {$i<$cols} {incr i} {
-    append sql "c$i,"
+    append data "c$i|"
+  }
+  append data "c$cols\n";
+  for {set i 1} {$i<$cols} {incr i} {
     append data "$i|"
   }
-  append sql "c$cols);"
   append data "$cols"
-  catchcmd "test.db" $sql
   set in [open shell5.csv w]
   puts $in $data
   close $in
@@ -214,14 +215,15 @@ SELECT COUNT(*) FROM t2;}]
 } {0 1}
 
 # try importing a large number of rows
-set rows 999999
+set rows 99999
 do_test shell5-1.7.1 {
   set in [open shell5.csv w]
+  puts $in a
   for {set i 1} {$i<=$rows} {incr i} {
     puts $in $i
   }
   close $in
-  set res [catchcmd "test.db" {CREATE TABLE t3(a);
+  set res [catchcmd "test.db" {.mode csv
 .import shell5.csv t3
 SELECT COUNT(*) FROM t3;}]
 } [list 0 $rows]