From: larrybr Date: Wed, 25 Oct 2023 20:27:18 +0000 (+0000) Subject: In CLI for Windows builds, do MBCS/UTF-8 translation independently for input and... X-Git-Tag: version-3.44.0~24^2~3 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=63aff62e697b46438004ac05fd6588e574e00f2f;p=thirdparty%2Fsqlite.git In CLI for Windows builds, do MBCS/UTF-8 translation independently for input and output. (WIP) FossilOrigin-Name: 47b13e66e07be3a26cf7fa0b7b65ad8576c3b4be50fc422be8fe93be93d681ac --- diff --git a/manifest b/manifest index 8ad88c1414..dd27939d14 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\san\sALWAYS\son\sbranch\sthat\swas\smade\salways-true\sby\sthe\sDISTINCT\sORDER\sBY\nfix. -D 2023-10-25T15:30:11.779 +C In\sCLI\sfor\sWindows\sbuilds,\sdo\sMBCS/UTF-8\stranslation\sindependently\sfor\sinput\sand\soutput.\s(WIP) +D 2023-10-25T20:27:18.335 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -722,7 +722,7 @@ F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c 31229276a8eb5b5de1428cd2d80f6f1cf8ffc5248be25e47cf575df12f1b8f23 F src/rowset.c 8432130e6c344b3401a8874c3cb49fefe6873fec593294de077afea2dce5ec97 F src/select.c 64c9bc7494f3d220a27498137551762c25458282388ea9ac0a710dd6d5dc1510 -F src/shell.c.in acc452c414fddd10289d165be3c89a7a2c36c919def04c93fb7dd11ac022e6ed +F src/shell.c.in 6b7662064d78025f0c68567971fe834840da8683972ac1ac7a02d6fbc9c166ef F src/sqlite.h.in 81c70644aeef9c974f72c9cadeb505ebb9441d2f6db594c018604ae935a12e6e F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 2f30b2671f4c03cd27a43f039e11251391066c97d11385f5f963bb40b03038ac @@ -2138,8 +2138,11 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P d2dbbdf7194bab4e5e3b74d3dbffb012a335829824c775c72dd7347c013d2125 -R 69224f49fd227efd4871d1e294714787 -U drh -Z 09c12c04ae80362f6524540f0fd3a848 +P c29ebcb0be8e969c359c077bd4ce752a9c952d2659062868f12efdd2cc77186d +R 023b0809f9198b763f6cfba361f37800 +T *branch * win-utf8-io-split +T *sym-win-utf8-io-split * +T -sym-trunk * +U larrybr +Z 1eb3a8144b68660f3c58b794f51200b9 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index d996fb461a..0e8e6f875b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c29ebcb0be8e969c359c077bd4ce752a9c952d2659062868f12efdd2cc77186d \ No newline at end of file +47b13e66e07be3a26cf7fa0b7b65ad8576c3b4be50fc422be8fe93be93d681ac \ No newline at end of file diff --git a/src/shell.c.in b/src/shell.c.in index b78759c002..b970ce4799 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -449,12 +449,20 @@ static int bail_on_error = 0; */ static int stdin_is_interactive = 1; +/* +** If build is for Windows, without 3rd-party line editing, Console +** input and output may be done in a UTF-8 compatible way. This is +** determined by invocation option and OS installed capability. +*/ #if (defined(_WIN32) || defined(WIN32)) && SHELL_USE_LOCAL_GETLINE \ && !defined(SHELL_OMIT_WIN_UTF8) # define SHELL_WIN_UTF8_OPT 1 - static int console_utf8 = sizeof(char*)/4 - 1; + static int console_utf8_in = 0; + static int console_utf8_out = 0; + static int mbcs_opted = 0; #else -# define console_utf8 0 +# define console_utf8_in 0 +# define console_utf8_out 0 # define SHELL_WIN_UTF8_OPT 0 #endif @@ -597,7 +605,7 @@ static struct ConsoleState { int infsMode; /* Input file stream mode upon shell start */ UINT inCodePage; /* Input code page upon shell start */ UINT outCodePage; /* Output code page upon shell start */ - HANDLE hConsoleIn; /* Console input handle */ + HANDLE hConsole; /* Console input or output handle */ DWORD consoleMode; /* Console mode upon shell start */ } conState = { 0, 0, 0, 0, INVALID_HANDLE_VALUE, 0 }; @@ -607,49 +615,59 @@ static struct ConsoleState { /* ** Prepare console, (if known to be a WIN32 console), for UTF-8 -** input (from either typing or suitable paste operations) and for +** input (from either typing or suitable paste operations) and/or for ** UTF-8 rendering. This may "fail" with a message to stderr, where ** the preparation is not done and common "code page" issues occur. */ -static void console_prepare(void){ +static void console_prepare_utf8(void){ HANDLE hCI = GetStdHandle(STD_INPUT_HANDLE); + HANDLE hCO = GetStdHandle(STD_OUTPUT_HANDLE); + HANDLE hCC = INVALID_HANDLE_VALUE; DWORD consoleMode = 0; - if( isatty(0) && GetFileType(hCI)==FILE_TYPE_CHAR - && GetConsoleMode( hCI, &consoleMode) ){ - if( !IsValidCodePage(CP_UTF8) ){ - fprintf(stderr, "Cannot use UTF-8 code page.\n"); - console_utf8 = 0; - return; - } - conState.hConsoleIn = hCI; - conState.consoleMode = consoleMode; - conState.inCodePage = GetConsoleCP(); - conState.outCodePage = GetConsoleOutputCP(); + u8 conIO = 0; + + console_utf8_in = console_utf8_out = 0; + if( isatty(0) && GetFileType(hCI)==FILE_TYPE_CHAR ) conIO = 1; + if( isatty(1) && GetFileType(hCO)==FILE_TYPE_CHAR ) conIO |= 2; + if( conIO==0 || mbcs_opted ) return; + if( conIO & 1 ) hCC = hCI; + else hCC = hCO; + if( !IsValidCodePage(CP_UTF8) || !GetConsoleMode( hCC, &consoleMode) ){ + fprintf(stderr, "Cannot use UTF-8 code page.\n"); + return; + } + conState.hConsole = hCC; + conState.consoleMode = consoleMode; + conState.inCodePage = GetConsoleCP(); + conState.outCodePage = GetConsoleOutputCP(); + if( conIO & 1 ){ + console_utf8_in = 1; SetConsoleCP(CP_UTF8); - SetConsoleOutputCP(CP_UTF8); consoleMode |= ENABLE_LINE_INPUT | ENABLE_PROCESSED_INPUT; - SetConsoleMode(conState.hConsoleIn, consoleMode); + SetConsoleMode(conState.hConsole, consoleMode); conState.infsMode = _setmode(_fileno(stdin), _O_U16TEXT); - console_utf8 = 1; - }else{ - console_utf8 = 0; + } + if( conIO & 2 ){ + console_utf8_out = 1; + SetConsoleOutputCP(CP_UTF8); } } /* -** Undo the effects of console_prepare(), if any. +** Undo the effects of console_prepare_utf8(), if any. */ static void SQLITE_CDECL console_restore(void){ - if( console_utf8 && conState.inCodePage!=0 - && conState.hConsoleIn!=INVALID_HANDLE_VALUE ){ + if( (console_utf8_in||console_utf8_out) && conState.inCodePage!=0 + && conState.hConsole!=INVALID_HANDLE_VALUE ){ _setmode(_fileno(stdin), conState.infsMode); SetConsoleCP(conState.inCodePage); SetConsoleOutputCP(conState.outCodePage); - SetConsoleMode(conState.hConsoleIn, conState.consoleMode); + SetConsoleMode(conState.hConsole, conState.consoleMode); /* Avoid multiple calls. */ - conState.hConsoleIn = INVALID_HANDLE_VALUE; + conState.hConsole = INVALID_HANDLE_VALUE; conState.consoleMode = 0; - console_utf8 = 0; + console_utf8_in = 0; + console_utf8_out = 0; } } @@ -657,11 +675,11 @@ static void SQLITE_CDECL console_restore(void){ ** Collect input like fgets(...) with special provisions for input ** from the Windows console to get around its strange coding issues. ** Defers to plain fgets() when input is not interactive or when the -** startup option, -utf8, has not been provided or taken effect. +** UTF-8 input is unavailable or opted out. */ static char* utf8_fgets(char *buf, int ncmax, FILE *fin){ if( fin==0 ) fin = stdin; - if( fin==stdin && stdin_is_interactive && console_utf8 ){ + if( fin==stdin && stdin_is_interactive && console_utf8_in ){ # define SQLITE_IALIM 150 wchar_t wbuf[SQLITE_IALIM]; int lend = 0; @@ -674,7 +692,7 @@ static char* utf8_fgets(char *buf, int ncmax, FILE *fin){ ? SQLITE_IALIM : (ncmax-1 - noc)/4; # undef SQLITE_IALIM DWORD nbr = 0; - BOOL bRC = ReadConsoleW(conState.hConsoleIn, wbuf, na, &nbr, 0); + BOOL bRC = ReadConsoleW(conState.hConsole, wbuf, na, &nbr, 0); if( !bRC || (noc==0 && nbr==0) ) return 0; if( nbr > 0 ){ int nmb = WideCharToMultiByte(CP_UTF8,WC_COMPOSITECHECK|WC_DEFAULTCHAR, @@ -720,8 +738,8 @@ static char* utf8_fgets(char *buf, int ncmax, FILE *fin){ /* ** Render output like fprintf(). Except, if the output is going to the -** console and if this is running on a Windows machine, and if the -utf8 -** option is unavailable or (available and inactive), translate the +** console and if this is running on a Windows machine, and if UTF-8 +** output unavailable (or available but opted out), translate the ** output from UTF-8 into MBCS for output through 8-bit stdout stream. ** (With -utf8 active, no translation is needed and must not be done.) */ @@ -729,7 +747,7 @@ static char* utf8_fgets(char *buf, int ncmax, FILE *fin){ void utf8_printf(FILE *out, const char *zFormat, ...){ va_list ap; va_start(ap, zFormat); - if( stdout_is_console && (out==stdout || out==stderr) && !console_utf8 ){ + if( stdout_is_console && (out==stdout || out==stderr) && !console_utf8_out ){ char *z1 = sqlite3_vmprintf(zFormat, ap); char *z2 = sqlite3_win32_utf8_to_mbcs_v2(z1, 0); sqlite3_free(z1); @@ -942,8 +960,8 @@ static char *local_getline(char *zLine, FILE *in){ #if defined(_WIN32) || defined(WIN32) /* For interactive input on Windows systems, without -utf8, ** translate the multi-byte characterset characters into UTF-8. - ** This is the translation that predates the -utf8 option. */ - if( stdin_is_interactive && in==stdin && !console_utf8 ){ + ** This is the translation that predates console UTF-8 input. */ + if( stdin_is_interactive && in==stdin && !console_utf8_in ){ char *zTrans = sqlite3_win32_mbcs_to_utf8_v2(zLine, 0); if( zTrans ){ i64 nTrans = strlen(zTrans)+1; @@ -12503,11 +12521,11 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ stdin_is_interactive = 0; }else if( cli_strcmp(z,"-utf8")==0 ){ #if SHELL_WIN_UTF8_OPT - console_utf8 = 1; + mbcs_opted = 0; #endif /* SHELL_WIN_UTF8_OPT */ }else if( cli_strcmp(z,"-no-utf8")==0 ){ #if SHELL_WIN_UTF8_OPT - console_utf8 = 0; + mbcs_opted = 1; #endif /* SHELL_WIN_UTF8_OPT */ }else if( cli_strcmp(z,"-heap")==0 ){ i++; @@ -12591,11 +12609,10 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ data.cMode = data.mode; } #if SHELL_WIN_UTF8_OPT - if( console_utf8 && stdin_is_interactive ){ - console_prepare(); + if( stdin_is_interactive ){ + console_prepare_utf8(); }else{ setBinaryMode(stdin, 0); - console_utf8 = 0; } #endif @@ -12636,7 +12653,7 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ const char *zCharset = ""; int nHistory; #if SHELL_WIN_UTF8_OPT - if( console_utf8 ) zCharset = " (utf8)"; + if( console_utf8_in ) zCharset = " (utf8)"; #endif printf( "SQLite version %s %.19s%s\n" /*extra-version-info*/