For CLI .sha3sum, emit warning to stderr for any invalidly encoded text fields.

FossilOrigin-Name: 8e833ecc81367658e81acd1d3a0242ab954a62d6f719af56f1d185656d0b73e6

diff --git a/manifest b/manifest
index 34ffa10eccf27889946448b7a705665c868d1624..56f9cf403194b2fd5cf4437d41f6a2e1e4b8ab91 100644 (file)
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Cause\sCLI\s.sha3sum\sto\swarn\sof\stext\sfields\sthat\sdo\snot\ssurvive\sCAST(CAST(t\sas\sBLOB)\sAS\sTEXT)\sdue\sto\sinvalid\sUTF\sencoding.
-D 2022-12-03T16:09:32.554
+C For\sCLI\s.sha3sum,\semit\swarning\sto\sstderr\sfor\sany\sinvalidly\sencoded\stext\sfields.
+D 2022-12-06T18:48:37.123
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -646,7 +646,7 @@ F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c
 F src/resolve.c efea4e5fbecfd6d0a9071b0be0d952620991673391b6ffaaf4c277b0bb674633
 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92
 F src/select.c 6983de0e6b8b68c97f82f9fca27ffb8f17161cff4d0d48fdf9eb40b213da0cc6
-F src/shell.c.in aa3e4c78ae25c24562b0c3497a88875baa61425714437143170b2c462842035d
+F src/shell.c.in cf32c16e1b4ff8e434b51fe38753bfddccd75d4a3ecb1fb838a6f96a70e2ff3a
 F src/sqlite.h.in 3439711b72cf1a541716da3671ac40f8d5957cdecfc192d47d32f7aed94207c2
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h c4b9fa7a7e2bcdf850cfeb4b8a91d5ec47b7a00033bc996fd2ee96cbf2741f5f
@@ -2065,11 +2065,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P d9807656f8a7c2a893d3f68ee5592f44826b8e999ae66f7d9000674b5c1b0207
-R 9f83c1fb14dddfc272778480afabb0a3
-T *branch * sha3sum_text_validation
-T *sym-sha3sum_text_validation *
-T -sym-trunk *
+P 123f2a0785790addf9c60a0fd09077dda9cb84d33a2594901a2b22bb555be491
+R 1a108a993a6e5c4e1463804542f95600
 U larrybr
-Z 1ba97ab620561a8dccaf913734ec9094
+Z 23502144cb7b8e35b977c4658f2c6efa
 # Remove this line to create a well-formed Fossil manifest.

diff --git a/manifest.uuid b/manifest.uuid
index b35a727cc37bd3d04575f1099e264ed0ec7b891f..b0fd6bc454cdaf689ae4a5ed4c9d3ff914a80fd4 100644 (file)
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-123f2a0785790addf9c60a0fd09077dda9cb84d33a2594901a2b22bb555be491
\ No newline at end of file
+8e833ecc81367658e81acd1d3a0242ab954a62d6f719af56f1d185656d0b73e6
\ No newline at end of file

diff --git a/src/shell.c.in b/src/shell.c.in
index 1f0ac24c81d28b4605ded900ad5cff387aead6da..a74614eaacbc96937530930d4171f22c900d469c 100644 (file)
--- a/src/shell.c.in
+++ b/src/shell.c.in
@@ -10077,7 +10077,6 @@ static int do_meta_command(char *zLine, ShellState *p){
     sqlite3_stmt *pStmt;     /* For querying tables names */
     char *zSql;              /* SQL to be run */
     char *zSep;              /* Separator */
-    char *zRevText;          /* Query for reversible to-blob-to-text check */
     ShellText sSql;          /* Complete SQL for the query to run the hash */
     ShellText sQuery;        /* Set of queries used to read all content */
     open_db(p, 0);
@@ -10126,24 +10125,6 @@ static int do_meta_command(char *zLine, ShellState *p){
              " ORDER BY 1 collate nocase";
     }
     sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0);
-    zRevText = sqlite3_mprintf(
-        /* lower-case query is first run, producing upper-case query. */
-        "with text_cols as materialized("
-        "select tname, cname "
-        "from ("
-        " select ss.tname as tname, ti.name as cname, lower(type) as lca"
-        " from (%s) ss join pragma_table_info(tname) ti"
-        " where (instr(lca,'int')=0 and "
-        " instr(lca,'char')+instr(lca,'clob')+instr(lca,'text')>0)"
-        "))"
-        "select 'SELECT total(bad_text_count) AS bad_text_count"
-        " FROM ('||group_concat(query, ' UNION ALL ')||')' as btc_query"
-        " from (select 'SELECT COUNT(*) AS bad_text_count"
-        " FROM '||tname||' WHERE '"
-        " ||group_concat('CAST(CAST('||cname||' AS BLOB) AS TEXT)<>'||cname,"
-        " ' OR ') as query, tname from text_cols group by tname)",
-        zSql);
-    shell_check_oom(zRevText);
     initText(&sQuery);
     initText(&sSql);
     appendText(&sSql, "WITH [sha3sum$query](a,b) AS(",0);
@@ -10200,34 +10181,56 @@ static int do_meta_command(char *zLine, ShellState *p){
       shell_exec(p, zSql, 0);
     }
     {
-      int lrc = sqlite3_prepare_v2(p->db, zRevText, -1, &pStmt, 0);
+      int lrc;
+      char *zRevText = /* Query for reversible to-blob-to-text check */
+       "SELECT lower(name) as tname FROM sqlite_schema\n"
+       "WHERE type='table' AND coalesce(rootpage,0)>1\n"
+       "AND name NOT LIKE 'sqlite_%%'%s\n"
+       "ORDER BY 1 collate nocase";
+      zRevText = sqlite3_mprintf(zRevText, zLike? " AND name LIKE $tspec" : "");
+      zRevText = sqlite3_mprintf(
+         /* lower-case query is first run, producing upper-case query. */
+         "with tabcols as materialized(\n"
+         "select tname, cname\n"
+         "from ("
+         " select ss.tname as tname, ti.name as cname\n"
+         " from (%z) ss\n inner join pragma_table_info(tname) ti))\n"
+         "select 'SELECT total(bad_text_count) AS bad_text_count\n"
+         "FROM ('||group_concat(query, ' UNION ALL ')||')' as btc_query\n"
+         " from (select 'SELECT COUNT(*) AS bad_text_count\n"
+         "FROM '||tname||' WHERE '\n"
+         "||group_concat('CAST(CAST('||cname||' AS BLOB) AS TEXT)<>'||cname\n"
+         "|| ' AND typeof('||cname||')=''text'' ',\n"
+         "' OR ') as query, tname from tabcols group by tname)"
+         , zRevText);
+      shell_check_oom(zRevText);
       if( bDebug ) utf8_printf(p->out, "%s\n", zRevText);
+      lrc = sqlite3_prepare_v2(p->db, zRevText, -1, &pStmt, 0);
       assert(lrc==SQLITE_OK);
+      if( zLike ) sqlite3_bind_text(pStmt,1,zLike,-1,SQLITE_STATIC);
       lrc = SQLITE_ROW==sqlite3_step(pStmt);
-      assert(lrc!=0);
       if( lrc ){
         const char *zGenQuery = sqlite3_column_text(pStmt,0);
         sqlite3_stmt *pCheckStmt;
-        double countIrreversible;
         lrc = sqlite3_prepare_v2(p->db, zGenQuery, -1, &pCheckStmt, 0);
         if( bDebug ) utf8_printf(p->out, "%s\n", zGenQuery);
-        assert(lrc==SQLITE_OK);
-        lrc = SQLITE_ROW==sqlite3_step(pCheckStmt);
-        assert(lrc!=0);
-        countIrreversible = sqlite3_column_double(pCheckStmt, 0);
-        sqlite3_finalize(pStmt);
-        sqlite3_finalize(pCheckStmt);
-        if( countIrreversible>0 ){
-          int n = (int)(countIrreversible + 0.5);
-          utf8_printf(stderr,
-                      "Digest includes %d invalidly encoded text field%s.\n",
-                      n, (n>1)? "s": "");
+        if( SQLITE_OK==lrc ){
+         if( SQLITE_ROW==sqlite3_step(pCheckStmt) ){
+           double countIrreversible = sqlite3_column_double(pCheckStmt, 0);
+           if( countIrreversible>0 ){
+             int n = (int)(countIrreversible + 0.5);
+             utf8_printf(stderr,
+                "Digest includes %d invalidly encoded text field%s.\n",
+                n, (n>1)? "s": "");
+           }
+         }
+         sqlite3_finalize(pCheckStmt);
         }
+        sqlite3_finalize(pStmt);
       }
-
+      sqlite3_free(zRevText);
     }
     sqlite3_free(zSql);
-    sqlite3_free(zRevText);
   }else
 
 #if !defined(SQLITE_NOHAVE_SYSTEM) && !defined(SQLITE_SHELL_FIDDLE)
@@ -11008,6 +11011,7 @@ static QuickScanState quickscan(char *zLine, QuickScanState qss){
           goto PlainScan;
         case '`': case '\'': case '"':
           if(*zLine==cWait){
+           /* Swallow doubled end-delimiter.*/
             ++zLine;
             continue;
           }