From: larrybr Date: Mon, 17 Apr 2023 21:18:53 +0000 (+0000) Subject: When CLI -utf8 active, do no translation to MBCS for output to Windows console. X-Git-Tag: version-3.42.0~134^2~1 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=690ebc690828fca38db442ff64ade286ebb604d8;p=thirdparty%2Fsqlite.git When CLI -utf8 active, do no translation to MBCS for output to Windows console. FossilOrigin-Name: cc1d4296d71ee6e2ba133edc581385e201c89657911c2ed39108843c776633a4 --- 690ebc690828fca38db442ff64ade286ebb604d8 diff --cc manifest index 454e8ac7f9,fd72ff319f,010a7b6132..17cad130ed --- a/manifest +++ b/manifest @@@@ -1,5 -1,5 -1,5 +1,5 @@@@ - C For\sCLI\s-utf8,\sset\soutput\scodepage\stoo.\sAdjust\sPP\svars\sso\sthat\sthe\scode\scan\sbe\sentirely\somitted\sfor\stargets\spretending\sto\sbe\s"WIN32"\sbut\snot\squite\sdoing\sso. - D 2023-04-15T16:12:46.806 - C Add\sthe\s'secure-delete'\soption\sto\sthe\sfts5\sextension.\sFor\sconfiguring\sfts5\sto\sdelete\sold\sentries\sdirectly\sfrom\sthe\sfull-text\sindex\sinstead\sof\susing\sdelete\skeys. - D 2023-04-15T17:47:32.556 -C Fix\ssome\sproblems\swith\susing\sfts5\soptions\s'secure-delete'\sand\sdetail=none\stogether. -D 2023-04-17T18:32:22.329 +++C When\sCLI\s-utf8\sactive,\sdo\sno\stranslation\sto\sMBCS\sfor\soutput\sto\sWindows\sconsole. +++D 2023-04-17T21:18:53.516 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@@@ -86,14 -86,14 -86,14 +86,14 @@@@ F ext/fts3/unicode/mkunicode.tcl d5aebf F ext/fts3/unicode/parseunicode.tcl a981bd6466d12dd17967515801c3ff23f74a281be1a03cf1e6f52a6959fc77eb F ext/fts5/extract_api_docs.tcl a36e54ec777172ddd3f9a88daf593b00848368e0 F ext/fts5/fts5.h c132a9323f22a972c4c93a8d5a3d901113a6e612faf30ca8e695788438c5ca2a - F ext/fts5/fts5Int.h c0d46e399e345e35985b72a1c1af025973bfaa5b1e3563b0ce3bb0ce144a7ca3 + F ext/fts5/fts5Int.h f473de2bdae0977af0d6c8cce96e3666821b85efba5f6006c7732662c3aabcb3 F ext/fts5/fts5_aux.c 572d5ec92ba7301df2fea3258576332f2f4d2dfd66d8263afd157d9deceac480 F ext/fts5/fts5_buffer.c 3001fbabb585d6de52947b44b455235072b741038391f830d6b729225eeaf6a5 - F ext/fts5/fts5_config.c 501e7d3566bc92766b0e11c0109a7c5a6146bc41144195459af5422f6c2078aa + F ext/fts5/fts5_config.c 46af0b3c3c3f00bfc5bdd307434d7c5f0fa0678a034dd48345cd83b20068efbd F ext/fts5/fts5_expr.c 48e8e45261c6030cf5c77f606217a22722b1a4d0b34e2ba6cbfc386581627989 F ext/fts5/fts5_hash.c d4fb70940359f2120ccd1de7ffe64cc3efe65de9e8995b822cd536ff64c96982 - F ext/fts5/fts5_index.c df5b29576a409f673e54b470723d817df9d5167cff208c48ab9a3773cba6fa89 - F ext/fts5/fts5_main.c fe67b6fb2ef134d9dbfa3941c63f777d755b075449be1863cb913a7f8754cb69 - F ext/fts5/fts5_index.c 2e7829ffae1185961a87cec13052d9781e74eaf0d8831ea6743ee88f286f17ce ++ F ext/fts5/fts5_index.c 2a59602a7ea830ae40b742277f4d89cae60a646bb2fb9c2ce895aea5e7ca89ce + F ext/fts5/fts5_main.c b4dba04a36aaf9b8e8cef0100b6dbb422cc74753eacc11d6401cac7a87c0f38d F ext/fts5/fts5_storage.c 76c6085239eb44424004c022e9da17a5ecd5aaec859fba90ad47d3b08f4c8082 F ext/fts5/fts5_tcl.c b1445cbe69908c411df8084a10b2485500ac70a9c747cdc8cda175a3da59d8ae F ext/fts5/fts5_test_mi.c 08c11ec968148d4cb4119d96d819f8c1f329812c568bac3684f5464be177d3ee @@@@ -187,6 -187,11 -187,12 +187,12 @@@@ F ext/fts5/test/fts5rebuild.test 55d6f1 F ext/fts5/test/fts5restart.test 835ecc8f449e3919f72509ab58056d0cedca40d1fe04108ccf8ac4c2ba41f415 F ext/fts5/test/fts5rowid.test b8790ec170a8dc1942a15aef3db926a5f3061b1ff171013003d8297203a20ad6 F ext/fts5/test/fts5savepoint.test fc02929f238d02a22df4172625704e029f7c1e0e92e332d654375690f8e6e43f + F ext/fts5/test/fts5secure.test 214a561519d1b1817f146efd1057e2a97cc896e75c2accc77157d874154bda64 + F ext/fts5/test/fts5secure2.test 2e961d7eef939f294c56b5d895cac7f1c3a60b934ee2cfd5e5e620bdf1ba6bbc + F ext/fts5/test/fts5secure3.test c7e1080a6912f2a3ac68f2e05b88b72a99de38543509b2bbf427cac5c9c1c610 - F ext/fts5/test/fts5secure4.test 17ac602024db3953401c3e51ad8e03de3b972c1770554585f63812ca4083550e - F ext/fts5/test/fts5securefault.test d18e3cab7a3c4a66f44812202e1b0ea0b4bd0f9ee766951749737cfbab100e7e ++ F ext/fts5/test/fts5secure4.test 0d10a80590c07891478700af7793b232962042677432b9846cf7fc8337b67c97 ++ F ext/fts5/test/fts5secure5.test c07a68ced5951567ac116c22f2d2aafae497e47fe9fcb6a335c22f9c7a4f2c3a ++ F ext/fts5/test/fts5securefault.test 7208daed4171de8a54a293ef800411e3cec1ffc6e9d34209b53e56a70ee554da F ext/fts5/test/fts5simple.test a298670508c1458b88ce6030440f26a30673931884eb5f4094ac1773b3ba217b F ext/fts5/test/fts5simple2.test 258a1b0c590409bfa5271e872c79572b319d2a56554d0585f68f146a0da603f0 F ext/fts5/test/fts5simple3.test d5c74a9d3ca71bd5dd5cacb7c55b86ea12cdddfc8b1910e3de2995206898380f @@@@ -625,8 -630,8 -631,8 +631,8 @@@@ F src/printf.c 7eac1a9896a80697e03e0896 F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c 3e53e02ce87c9582bd7e7d22f13f4094a271678d9dc72820fa257a2abb5e4032 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 - F src/select.c 93bb02212256b49a90589a4664031896e2b2991198153dff1a33a72f437dad94 - F src/shell.c.in ec8f1e4f87882670885e27037b8c236f569932a823feecbefc25a910c851d4e8 + F src/select.c ad6f48e0953fb0d9dd434a7fcd8bc6bde05beb82500a59fea7a7b02dc0ecfb3b - F src/shell.c.in 80f7c4c1bdabc9dab416b59287458044c01df97035d626ea57ffeabbb289bdc6 -F src/shell.c.in ec8f1e4f87882670885e27037b8c236f569932a823feecbefc25a910c851d4e8 +++F src/shell.c.in 6269bf25a222099205c25d4e3917493ef076586064fff0c7b2c33395377b3740 F src/sqlite.h.in 4fff9c6cc5d4cbba9532a668112efb6dc469c425e1a2196664d7c07d508363ef F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h da473ce2b3d0ae407a6300c4a164589b9a6bfdbec9462688a8593ff16f3bb6e4 @@@@ -2052,8 -2057,9 -2058,8 +2058,8 @@@@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a9 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 - P fa3ce7f013d40147f012e26d0ede75b134174f144d314336310c9ac585e23fb7 - R bf7c6e9f3d2c14f31428bf0dcd17722f - P 430ec1a61507d8afd8adc161f577f32856dba21506db9717fae3d14570786439 c5a47063b7971868ebbb3f0718434dd6bd22d01c43697f48941df9aae3885abc - R d90ff03c96717145cedda138162c4605 - T +closed c5a47063b7971868ebbb3f0718434dd6bd22d01c43697f48941df9aae3885abc -P 2f63d4a8bac6dc72197df32efd84c148adbc4794943088dff08da57b15dd7900 -R 94f7d6e08b83dc7ea2e9a235119af696 --U dan - Z f96a9a0e867832099372b7f8b2a1eddf -Z 7915ab356b23565aa915e2ae0ad516a8 +++P 543594a7277b12d1c58af8e3cba3841667cf924172995536489ce7da704a11d7 394980e4fe12125460ab14da41edae9089a4da332a46f3124bf0b9778793791f 4d3f27ba90aa59837e49ceaae9f36cad426af3a33707d85d1bb848fc1bda6fed +++R a35a825488de1096cbecfb4e55ef07c0 ++U larrybr - Z e087e76624523fc274ea2606bdd42051 +++Z cc0b8fd4da0e661d2adcb75f4dd54fc5 # Remove this line to create a well-formed Fossil manifest. diff --cc manifest.uuid index ccd8ec1c4a,a5c4ea1878,f88d210269..7079f75b36 --- a/manifest.uuid +++ b/manifest.uuid @@@@ -1,1 -1,1 -1,1 +1,1 @@@@ - 543594a7277b12d1c58af8e3cba3841667cf924172995536489ce7da704a11d7 - 394980e4fe12125460ab14da41edae9089a4da332a46f3124bf0b9778793791f -4d3f27ba90aa59837e49ceaae9f36cad426af3a33707d85d1bb848fc1bda6fed +++cc1d4296d71ee6e2ba133edc581385e201c89657911c2ed39108843c776633a4 diff --cc src/shell.c.in index 06648f494a,220e91fd1d,06648f494a..0afc63d6b6 --- a/src/shell.c.in +++ b/src/shell.c.in @@@@ -594,134 -579,16 -594,134 +594,143 @@@@ static char *dynamicContinuePrompt(void } #endif /* !defined(SQLITE_OMIT_DYNAPROMPT) */ + #if SHELL_WIN_UTF8_OPT + /* Following struct is used for -utf8 operation. */ + static struct ConsoleState { + int stdinEof; /* EOF has been seen on console input */ + int infsMode; /* Input file stream mode upon shell start */ + UINT inCodePage; /* Input code page upon shell start */ + UINT outCodePage; /* Output code page upon shell start */ + HANDLE hConsoleIn; /* Console input handle */ + DWORD consoleMode; /* Console mode upon shell start */ + } conState = { 0, 0, 0, 0, INVALID_HANDLE_VALUE, 0 }; + + /* + ** Prepare console, (if known to be a WIN32 console), for UTF-8 + ** input (from either typing or suitable paste operations) and for + ** UTF-8 rendering. This may "fail" with a message to stderr, where + ** the preparation is not done and common "code page" issues occur. + */ + static void console_prepare(void){ - - conState.hConsoleIn = GetStdHandle(STD_INPUT_HANDLE); - - if( isatty(0) && GetFileType(conState.hConsoleIn)==FILE_TYPE_CHAR ){ +++ HANDLE hCI = GetStdHandle(STD_INPUT_HANDLE); +++ if( isatty(0) && GetFileType(hCI)==FILE_TYPE_CHAR +++ && GetConsoleMode( hCI, &conState.consoleMode) ){ + if( !IsValidCodePage(CP_UTF8) ){ + fprintf(stderr, "Cannot use UTF-8 code page.\n"); + console_utf8 = 0; + return; + } +++ conState.hConsoleIn = hCI; + conState.inCodePage = GetConsoleCP(); + conState.outCodePage = GetConsoleOutputCP(); + SetConsoleCP(CP_UTF8); + SetConsoleOutputCP(CP_UTF8); - - GetConsoleMode( conState.hConsoleIn, &conState.consoleMode); + SetConsoleMode( conState.hConsoleIn, + conState.consoleMode | ENABLE_LINE_INPUT ); + conState.infsMode = _setmode(_fileno(stdin), _O_U16TEXT); + console_utf8 = 1; + }else{ + console_utf8 = 0; + } + } + + /* + ** Undo the effects of console_prepare(), if any. + */ + static void SQLITE_CDECL console_restore(void){ + if( console_utf8 && conState.inCodePage!=0 + && conState.hConsoleIn != INVALID_HANDLE_VALUE ){ + _setmode(_fileno(stdin), conState.infsMode); + SetConsoleCP(conState.inCodePage); + SetConsoleOutputCP(conState.outCodePage); + SetConsoleMode( conState.hConsoleIn, conState.consoleMode ); - - console_utf8 = 0; /* Avoid multiple calls. */ +++ /* Avoid multiple calls. */ +++ conState.hConsoleIn = INVALID_HANDLE_VALUE; +++ console_utf8 = 0; + } + } + + /* + ** Collect input like fgets(...) with special provisions for input + ** from the Windows console to get around its strange coding issues. + ** Defers to plain fgets() when input is not interactive or when the + ** startup option, -utf8, has not been provided or taken effect. + */ + static char* utf8_fgets(char *buf, int ncmax, FILE *fin){ + if( fin==0 ) fin = stdin; + if( fin==stdin && stdin_is_interactive && console_utf8 ){ + # define SQLITE_IALIM 150 + wchar_t wbuf[SQLITE_IALIM]; + int lend = 0; + int noc = 0; + if( ncmax == 0 || conState.stdinEof ) return 0; + buf[0] = 0; + while( noc < ncmax-7-1 && !lend ){ + /* There is room for at least 2 more characters and a 0-terminator. */ + int na = (ncmax > SQLITE_IALIM*4+1 + noc) + ? SQLITE_IALIM : (ncmax-1 - noc)/4; + # undef SQLITE_IALIM + DWORD nbr = 0; + BOOL bRC = ReadConsoleW(conState.hConsoleIn, wbuf, na, &nbr, 0); + if( !bRC || (noc==0 && nbr==0) ) return 0; + if( nbr > 0 ){ + int nmb = WideCharToMultiByte(CP_UTF8, WC_COMPOSITECHECK|WC_DEFAULTCHAR, + wbuf,nbr, 0,0, 0, 0); + if( nmb !=0 && noc+nmb <= ncmax ){ + int iseg = noc; + nmb = WideCharToMultiByte(CP_UTF8, WC_COMPOSITECHECK|WC_DEFAULTCHAR, + wbuf,nbr, buf+noc,nmb, 0,0); + noc += nmb; + /* Fixup line-ends as coded by Windows for CR (or "Enter".)*/ + if( noc > 0 ){ + if( buf[noc-1]=='\n' ){ + lend = 1; + if( noc > 1 && buf[noc-2]=='\r' ){ + buf[noc-2] = '\n'; + --noc; + } + } + } + /* Check for ^Z (anywhere in line) too. */ + while( iseg < noc ){ + if( buf[iseg]==0x1a ){ + conState.stdinEof = 1; + noc = iseg; /* Chop ^Z and anything following. */ + break; + } + ++iseg; + } + }else break; /* Drop apparent garbage in. (Could assert.) */ + }else break; + } + /* If got nothing, (after ^Z chop), must be at end-of-file. */ + if( noc == 0 ) return 0; + buf[noc] = 0; + return buf; + }else{ + return fgets(buf, ncmax, fin); + } + } + + # define fgets(b,n,f) utf8_fgets(b,n,f) + #endif /* SHELL_WIN_UTF8_OPT */ + /* ** Render output like fprintf(). Except, if the output is going to the ---** console and if this is running on a Windows machine, translate the ---** output from UTF-8 into MBCS. +++** console and if this is running on a Windows machine, and if the -utf8 +++** option is unavailable or (available and inactive), translate the +++** output from UTF-8 into MBCS for output through 8-bit stdout stream. +++** (With -utf8 active, no translation is needed and must not be done.) */ #if defined(_WIN32) || defined(WIN32) void utf8_printf(FILE *out, const char *zFormat, ...){ va_list ap; va_start(ap, zFormat); --- if( stdout_is_console && (out==stdout || out==stderr) ){ +++ if( stdout_is_console && (out==stdout || out==stderr) +++# if SHELL_WIN_UTF8_OPT +++ && !console_utf8 +++# endif +++ ){ char *z1 = sqlite3_vmprintf(zFormat, ap); char *z2 = sqlite3_win32_utf8_to_mbcs_v2(z1, 0); sqlite3_free(z1);