From: larrybr <larrybr@noemail.net>
Date: Wed, 12 Apr 2023 17:54:52 +0000 (+0000)
Subject: WIP: CLI option to take control of console on Windows and make it support UTF-8 input... 
X-Git-Tag: version-3.42.0~141^2~5
X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=ad5a7da489f3657c42fe6770309a5d0fcccf92d5;p=thirdparty%2Fsqlite.git

WIP: CLI option to take control of console on Windows and make it support UTF-8 input pasting (or typing). Needs work to become robust per "ToDo:".

FossilOrigin-Name: 824382393d92d9eb6df8701de7c263280150569a708759c4a539acc6d8d38821
---

diff --git a/manifest b/manifest
index bc517744b1..ef57a3a675 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C New\s#ifdef\sto\senable\sbuilding\swith\s-DSQLITE_OMIT_WINDOWFUNC.
-D 2023-04-11T19:38:47.332
+C WIP:\sCLI\soption\sto\stake\scontrol\sof\sconsole\son\sWindows\sand\smake\sit\ssupport\sUTF-8\sinput\spasting\s(or\styping).\sNeeds\swork\sto\sbecome\srobust\sper\s"ToDo:".
+D 2023-04-12T17:54:52.445
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -626,7 +626,7 @@ F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c
 F src/resolve.c 3e53e02ce87c9582bd7e7d22f13f4094a271678d9dc72820fa257a2abb5e4032
 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92
 F src/select.c 93bb02212256b49a90589a4664031896e2b2991198153dff1a33a72f437dad94
-F src/shell.c.in 80f7c4c1bdabc9dab416b59287458044c01df97035d626ea57ffeabbb289bdc6
+F src/shell.c.in 156279044afe3acdd48997531617c6bd1b5a37b5761b67900cc4dbfcc8af7130
 F src/sqlite.h.in 4fff9c6cc5d4cbba9532a668112efb6dc469c425e1a2196664d7c07d508363ef
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h da473ce2b3d0ae407a6300c4a164589b9a6bfdbec9462688a8593ff16f3bb6e4
@@ -2052,8 +2052,11 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P c8fb143d64d8e823684cd26799080da4b42bef121ca3c6315b1803a593490926
-R 23a3c5b92c8f88b7b4f263831382c52f
-U drh
-Z 51e0cafc6ddabd2f402c0ebee1449b15
+P e1ff83fa2565334b28bd0d6582088c4ae0d2d9a590d973615a4a598683fe419c
+R 42a09bb2160cadfa8aa798fa30b96ed8
+T *branch * cli-utf8
+T *sym-cli-utf8 *
+T -sym-trunk *
+U larrybr
+Z e14b183810e19b5d66903057d8becd3e
 # Remove this line to create a well-formed Fossil manifest.
diff --git a/manifest.uuid b/manifest.uuid
index 497bd4caa0..230ab719ef 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-e1ff83fa2565334b28bd0d6582088c4ae0d2d9a590d973615a4a598683fe419c
\ No newline at end of file
+824382393d92d9eb6df8701de7c263280150569a708759c4a539acc6d8d38821
\ No newline at end of file
diff --git a/src/shell.c.in b/src/shell.c.in
index 220e91fd1d..c7c0d537ec 100644
--- a/src/shell.c.in
+++ b/src/shell.c.in
@@ -228,6 +228,7 @@ typedef unsigned char u8;
 #if SQLITE_OS_WINRT
 #include <intrin.h>
 #endif
+#define WIN32_LEAN_AND_MEAN
 #include <windows.h>
 
 /* string conversion routines only needed on Win32 */
@@ -442,11 +443,18 @@ static void endTimer(void){
 static int bail_on_error = 0;
 
 /*
-** Threat stdin as an interactive input if the following variable
+** Treat stdin as an interactive input if the following variable
 ** is true.  Otherwise, assume stdin is connected to a file or pipe.
 */
 static int stdin_is_interactive = 1;
 
+#if (defined(_WIN32) || defined(WIN32)) && !defined(SQLITE_SHELL_FIDDLE)
+/*
+** Setup console for UTF-8 input/output when following variable true.
+*/
+static int console_utf8 = 0;
+#endif
+
 /*
 ** On Windows systems we have to know if standard output is a console
 ** in order to translate UTF-8 into MBCS.  The following variable is
@@ -579,6 +587,80 @@ static char *dynamicContinuePrompt(void){
 }
 #endif /* !defined(SQLITE_OMIT_DYNAPROMPT) */
 
+#if (defined(_WIN32) || defined(WIN32)) && !defined(SQLITE_SHELL_FIDDLE)
+static int infsMode;
+static UINT codePage;
+static HANDLE hConsoleIn;
+static DWORD consoleMode;
+/*
+** Prepare input stream, (if known to be a WIN32 console), for UTF-8
+** input (from either typing or suitable paste operations) and for
+** UTF-8 rendering. This may "fail" with a message to stderr, where
+** the preparation is not done and common "code page" issues occur.
+*/
+static void instream_prepare(){
+  if( isatty(0) ){
+    if( !IsValidCodePage(CP_UTF8) ){
+      fprintf(stderr, "Cannot use UTF-8 code page.\n");
+      console_utf8 = 0;
+      return;
+    }
+    codePage = GetConsoleCP();
+    SetConsoleCP(CP_UTF8);
+    hConsoleIn = GetStdHandle(STD_INPUT_HANDLE);
+    GetConsoleMode( hConsoleIn, &consoleMode);
+    SetConsoleMode( hConsoleIn, consoleMode | ENABLE_LINE_INPUT );
+    infsMode = _setmode(_fileno(stdin), _O_U16TEXT);
+    console_utf8 = 1;
+  }else{
+    console_utf8 = 0;
+  }
+}
+
+/*
+** Undo the effects of instream_prepare(), if any.
+*/
+static void instream_restore(){
+  if( console_utf8 ){
+    _setmode(_fileno(stdin), infsMode);
+    SetConsoleCP(codePage);
+    SetConsoleMode( hConsoleIn, consoleMode );
+  }
+}
+
+/*
+** Collect input like fgets(...) with special provisions for input
+** from the Windows console to get around its strange coding issues.
+** Defers to plain fgets() when input is not interactive or when the
+** startup option, -utf8, has not been provided or taken effect.
+** WIP: This routine needs work to consume only as many characters
+** as can be fit into the provided input buffer. ToDo: That work.
+** Avoiding this issue will require some refactoring of CLI input.
+*/
+static char* utf8_fgets(char *buf, int ncmax, FILE *fin){
+  if( fin==stdin && stdin_is_interactive && console_utf8 ){
+    wchar_t * wbuf = (wchar_t*) malloc(ncmax * sizeof(wchar_t));
+    wchar_t * wz;
+    int nmb;
+    if( wbuf==0 ) return 0;
+    if( 0 == (wz = fgetws(wbuf, ncmax, stdin))
+        || 0 == (nmb = WideCharToMultiByte(CP_UTF8, 0, wz, -1, 0, 0, 0, 0))
+        || nmb > ncmax ){
+      buf = 0;
+      goto bail;
+    }
+    WideCharToMultiByte(CP_UTF8, 0, wz, -1, buf, nmb, 0, 0);
+  bail:
+    free(wbuf);
+    return buf;
+  }else{
+    return fgets(buf, ncmax, fin);
+  }
+}
+
+# define fgets(b,n,f) utf8_fgets(b,n,f)
+#endif /* (defined(_WIN32)||defined(WIN32)) && !defined(SQLITE_SHELL_FIDDLE) */
+
 /*
 ** Render output like fprintf().  Except, if the output is going to the
 ** console and if this is running on a Windows machine, translate the
@@ -802,7 +884,11 @@ static char *local_getline(char *zLine, FILE *in){
   /* For interactive input on Windows systems, translate the
   ** multi-byte characterset characters into UTF-8. */
   if( stdin_is_interactive && in==stdin ){
+    // FILE *cil = fopen("cin.log", "wa");
+    // i64 nzl = strlen(zLine);
     char *zTrans = sqlite3_win32_mbcs_to_utf8_v2(zLine, 0);
+    // fwrite(zLine, nzl, 1, cil);
+    // fwrite("\n", 1,1, cil);
     if( zTrans ){
       i64 nTrans = strlen(zTrans)+1;
       if( nTrans>nLine ){
@@ -810,8 +896,11 @@ static char *local_getline(char *zLine, FILE *in){
         shell_check_oom(zLine);
       }
       memcpy(zLine, zTrans, nTrans);
+      // fwrite(zLine, nTrans-1, 1, cil);
+      // fwrite("\n", 1,1, cil);
       sqlite3_free(zTrans);
     }
+    // fclose(cil);
   }
 #endif /* defined(_WIN32) || defined(WIN32) */
   return zLine;
@@ -11585,6 +11674,9 @@ static const char zOptions[] =
   "   -stats               print memory stats before each finalize\n"
   "   -table               set output mode to 'table'\n"
   "   -tabs                set output mode to 'tabs'\n"
+#if (defined(_WIN32) || defined(WIN32)) && !defined(SQLITE_SHELL_FIDDLE)
+  "   -utf8                setup interactive console code page for UTF-8\n"
+#endif
   "   -version             show SQLite version\n"
   "   -vfs NAME            use NAME as the default VFS\n"
 #ifdef SQLITE_ENABLE_VFSTRACE
@@ -12116,6 +12208,10 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){
       stdin_is_interactive = 1;
     }else if( cli_strcmp(z,"-batch")==0 ){
       stdin_is_interactive = 0;
+    }else if( cli_strcmp(z,"-utf8")==0 ){
+#if (defined(_WIN32) || defined(WIN32)) && !defined(SQLITE_SHELL_FIDDLE)
+      console_utf8 = 1;
+#endif
     }else if( cli_strcmp(z,"-heap")==0 ){
       i++;
     }else if( cli_strcmp(z,"-pagecache")==0 ){
@@ -12193,6 +12289,13 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){
     }
     data.cMode = data.mode;
   }
+#if (defined(_WIN32) || defined(WIN32)) && !defined(SQLITE_SHELL_FIDDLE)
+  if( console_utf8 && stdin_is_interactive ){
+    instream_prepare();
+  }else{
+    console_utf8 = 0;
+  }
+#endif
 
   if( !readStdin ){
     /* Run all arguments that do not begin with '-' as if they were separate
@@ -12267,6 +12370,11 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){
       rc = process_input(&data);
     }
   }
+#if (defined(_WIN32) || defined(WIN32)) && !defined(SQLITE_SHELL_FIDDLE)
+  if( console_utf8 ){
+    instream_restore();
+  }
+#endif
 #ifndef SQLITE_SHELL_FIDDLE
   /* In WASM mode we have to leave the db state in place so that
   ** client code can "push" SQL into it after this call returns. */