From: larrybr Date: Tue, 6 Dec 2022 18:48:37 +0000 (+0000) Subject: For CLI .sha3sum, emit warning to stderr for any invalidly encoded text fields. X-Git-Tag: version-3.41.0~299^2~1 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=09d953706c29d9189e7c618d6c3fd579d571b335;p=thirdparty%2Fsqlite.git For CLI .sha3sum, emit warning to stderr for any invalidly encoded text fields. FossilOrigin-Name: 8e833ecc81367658e81acd1d3a0242ab954a62d6f719af56f1d185656d0b73e6 --- diff --git a/manifest b/manifest index 34ffa10ecc..56f9cf4031 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Cause\sCLI\s.sha3sum\sto\swarn\sof\stext\sfields\sthat\sdo\snot\ssurvive\sCAST(CAST(t\sas\sBLOB)\sAS\sTEXT)\sdue\sto\sinvalid\sUTF\sencoding. -D 2022-12-03T16:09:32.554 +C For\sCLI\s.sha3sum,\semit\swarning\sto\sstderr\sfor\sany\sinvalidly\sencoded\stext\sfields. +D 2022-12-06T18:48:37.123 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -646,7 +646,7 @@ F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c efea4e5fbecfd6d0a9071b0be0d952620991673391b6ffaaf4c277b0bb674633 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 F src/select.c 6983de0e6b8b68c97f82f9fca27ffb8f17161cff4d0d48fdf9eb40b213da0cc6 -F src/shell.c.in aa3e4c78ae25c24562b0c3497a88875baa61425714437143170b2c462842035d +F src/shell.c.in cf32c16e1b4ff8e434b51fe38753bfddccd75d4a3ecb1fb838a6f96a70e2ff3a F src/sqlite.h.in 3439711b72cf1a541716da3671ac40f8d5957cdecfc192d47d32f7aed94207c2 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h c4b9fa7a7e2bcdf850cfeb4b8a91d5ec47b7a00033bc996fd2ee96cbf2741f5f @@ -2065,11 +2065,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P d9807656f8a7c2a893d3f68ee5592f44826b8e999ae66f7d9000674b5c1b0207 -R 9f83c1fb14dddfc272778480afabb0a3 -T *branch * sha3sum_text_validation -T *sym-sha3sum_text_validation * -T -sym-trunk * +P 123f2a0785790addf9c60a0fd09077dda9cb84d33a2594901a2b22bb555be491 +R 1a108a993a6e5c4e1463804542f95600 U larrybr -Z 1ba97ab620561a8dccaf913734ec9094 +Z 23502144cb7b8e35b977c4658f2c6efa # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index b35a727cc3..b0fd6bc454 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -123f2a0785790addf9c60a0fd09077dda9cb84d33a2594901a2b22bb555be491 \ No newline at end of file +8e833ecc81367658e81acd1d3a0242ab954a62d6f719af56f1d185656d0b73e6 \ No newline at end of file diff --git a/src/shell.c.in b/src/shell.c.in index 1f0ac24c81..a74614eaac 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -10077,7 +10077,6 @@ static int do_meta_command(char *zLine, ShellState *p){ sqlite3_stmt *pStmt; /* For querying tables names */ char *zSql; /* SQL to be run */ char *zSep; /* Separator */ - char *zRevText; /* Query for reversible to-blob-to-text check */ ShellText sSql; /* Complete SQL for the query to run the hash */ ShellText sQuery; /* Set of queries used to read all content */ open_db(p, 0); @@ -10126,24 +10125,6 @@ static int do_meta_command(char *zLine, ShellState *p){ " ORDER BY 1 collate nocase"; } sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0); - zRevText = sqlite3_mprintf( - /* lower-case query is first run, producing upper-case query. */ - "with text_cols as materialized(" - "select tname, cname " - "from (" - " select ss.tname as tname, ti.name as cname, lower(type) as lca" - " from (%s) ss join pragma_table_info(tname) ti" - " where (instr(lca,'int')=0 and " - " instr(lca,'char')+instr(lca,'clob')+instr(lca,'text')>0)" - "))" - "select 'SELECT total(bad_text_count) AS bad_text_count" - " FROM ('||group_concat(query, ' UNION ALL ')||')' as btc_query" - " from (select 'SELECT COUNT(*) AS bad_text_count" - " FROM '||tname||' WHERE '" - " ||group_concat('CAST(CAST('||cname||' AS BLOB) AS TEXT)<>'||cname," - " ' OR ') as query, tname from text_cols group by tname)", - zSql); - shell_check_oom(zRevText); initText(&sQuery); initText(&sSql); appendText(&sSql, "WITH [sha3sum$query](a,b) AS(",0); @@ -10200,34 +10181,56 @@ static int do_meta_command(char *zLine, ShellState *p){ shell_exec(p, zSql, 0); } { - int lrc = sqlite3_prepare_v2(p->db, zRevText, -1, &pStmt, 0); + int lrc; + char *zRevText = /* Query for reversible to-blob-to-text check */ + "SELECT lower(name) as tname FROM sqlite_schema\n" + "WHERE type='table' AND coalesce(rootpage,0)>1\n" + "AND name NOT LIKE 'sqlite_%%'%s\n" + "ORDER BY 1 collate nocase"; + zRevText = sqlite3_mprintf(zRevText, zLike? " AND name LIKE $tspec" : ""); + zRevText = sqlite3_mprintf( + /* lower-case query is first run, producing upper-case query. */ + "with tabcols as materialized(\n" + "select tname, cname\n" + "from (" + " select ss.tname as tname, ti.name as cname\n" + " from (%z) ss\n inner join pragma_table_info(tname) ti))\n" + "select 'SELECT total(bad_text_count) AS bad_text_count\n" + "FROM ('||group_concat(query, ' UNION ALL ')||')' as btc_query\n" + " from (select 'SELECT COUNT(*) AS bad_text_count\n" + "FROM '||tname||' WHERE '\n" + "||group_concat('CAST(CAST('||cname||' AS BLOB) AS TEXT)<>'||cname\n" + "|| ' AND typeof('||cname||')=''text'' ',\n" + "' OR ') as query, tname from tabcols group by tname)" + , zRevText); + shell_check_oom(zRevText); if( bDebug ) utf8_printf(p->out, "%s\n", zRevText); + lrc = sqlite3_prepare_v2(p->db, zRevText, -1, &pStmt, 0); assert(lrc==SQLITE_OK); + if( zLike ) sqlite3_bind_text(pStmt,1,zLike,-1,SQLITE_STATIC); lrc = SQLITE_ROW==sqlite3_step(pStmt); - assert(lrc!=0); if( lrc ){ const char *zGenQuery = sqlite3_column_text(pStmt,0); sqlite3_stmt *pCheckStmt; - double countIrreversible; lrc = sqlite3_prepare_v2(p->db, zGenQuery, -1, &pCheckStmt, 0); if( bDebug ) utf8_printf(p->out, "%s\n", zGenQuery); - assert(lrc==SQLITE_OK); - lrc = SQLITE_ROW==sqlite3_step(pCheckStmt); - assert(lrc!=0); - countIrreversible = sqlite3_column_double(pCheckStmt, 0); - sqlite3_finalize(pStmt); - sqlite3_finalize(pCheckStmt); - if( countIrreversible>0 ){ - int n = (int)(countIrreversible + 0.5); - utf8_printf(stderr, - "Digest includes %d invalidly encoded text field%s.\n", - n, (n>1)? "s": ""); + if( SQLITE_OK==lrc ){ + if( SQLITE_ROW==sqlite3_step(pCheckStmt) ){ + double countIrreversible = sqlite3_column_double(pCheckStmt, 0); + if( countIrreversible>0 ){ + int n = (int)(countIrreversible + 0.5); + utf8_printf(stderr, + "Digest includes %d invalidly encoded text field%s.\n", + n, (n>1)? "s": ""); + } + } + sqlite3_finalize(pCheckStmt); } + sqlite3_finalize(pStmt); } - + sqlite3_free(zRevText); } sqlite3_free(zSql); - sqlite3_free(zRevText); }else #if !defined(SQLITE_NOHAVE_SYSTEM) && !defined(SQLITE_SHELL_FIDDLE) @@ -11008,6 +11011,7 @@ static QuickScanState quickscan(char *zLine, QuickScanState qss){ goto PlainScan; case '`': case '\'': case '"': if(*zLine==cWait){ + /* Swallow doubled end-delimiter.*/ ++zLine; continue; }