commit bash-20161115 snapshot

diff --git a/CWRU/CWRU.chlog b/CWRU/CWRU.chlog
index 16835c92b25797a534a1eb04d88c6db213e0f3c0..782a778b411f69284f5150ccdff2c9825a6d9394 100644 (file)
--- a/CWRU/CWRU.chlog
+++ b/CWRU/CWRU.chlog
@@ -12127,7 +12127,7 @@ doc/{bash.1,bashref.texi}
 lib/glob/sm_loop.c
        - parse_collsym: make sure to not return an out-of-bounds read if a
          collating symbol is unterminated.  Fixes OOB read reported by
-         op7ic \x00 <op7ica@gmail.com>
+         Jerzy Kramarz <op7ica@gmail.com>
        - brackmatch: after incrementing p, before checking whether it's a
          character range, check whether *p was NULL before the increment
          and short-circuit the bracket expression if it is
@@ -12282,12 +12282,12 @@ subst.c
        - parameter_brace_{patsub,remove_pattern,transform,casemod}: save and
          restore this_command_name while temporarily setting it for use in
          error messages.  Fixes use-after-free error reported by
-         op7ic \x00 <op7ica@gmail.com>
+         Jerzy Kramarz <op7ica@gmail.com>
        - string_extract_verbatim: make sure when we increment i by 2 due to
          a CTLESC or CTLESC-CTLNUL that we don't read past the end of the
          string. This can happen if the string ends with an odd number of
          CTLESC chars. Fixes oob-read error reported by
-         op7ic \x00 <op7ica@gmail.com>
+         Jerzy Kramarz <op7ica@gmail.com>
 
                                   11/11
                                   -----
@@ -12355,3 +12355,37 @@ lib/sh/unicode.c
        - u32cconv: even if we don't have iconv, if locale_utf8locale is non-
          zero, return u32toutf8
 
+                                  11/15
+                                  -----
+lib/readline/nls.c
+       - _rl_init_locale: new function, split off code that determines current
+         locale from _rl_init_eightbit, use it to set _rl_utf8locale
+
+lib/readiline/rlprivate.h
+       - _rl_init_locale: new extern declaration
+
+lib/readline/readline.c
+       - rl_initialize: if not initializing everything the first time, call
+         _rl_init_locale to check current locale and set _rl_utf8locale
+
+lib/readline/text.c
+       - _rl_insert_char: optimize cases where we are inserting a single-byte
+         character in a locale with multibyte characters
+       - _rl_insert_char: check whether character is single byte if we know
+         we are in a UTF-8 locale, optimize single-byte case to avoid calls
+         to mbrtowc and memmove/memcpy
+
+lib/readline/mbutil.c
+       - _rl_char_value: if we are in a UTF-8 locale (_rl_utf8locale) and the
+         current character does not have the eighth bit set ((c & 0x80) == 0),
+         return that char without bothering to call mbrtowc
+       - _rl_adjust_point: don't bother calling mbrlen if we are in a UTF-8
+         locale and the current character is not a multibyte character
+       - _rl_find_next_mbchar_internal: if we are in a UTF-8 locale, use that
+         to avoid calls to mbrtowc
+
+lib/readline/display.c
+       - _rl_col_width: if in a UTF-8 locale, take advantage of that to avoid
+         calls to mbrlen and mbrtowc
+       - rl_redisplay: if in a UTF-8 locale, take advantage of that to avoid
+         calls to mbrtowc

diff --git a/RUN-ONE-TEST b/RUN-ONE-TEST
new file mode 100755 (executable)
index 0000000..d29259a
--- /dev/null
+++ b/RUN-ONE-TEST
@@ -0,0 +1,10 @@
+BUILD_DIR=/usr/local/build/chet/bash/bash-current
+THIS_SH=$BUILD_DIR/bash
+PATH=$PATH:$BUILD_DIR
+
+export THIS_SH PATH
+
+export BASH_TSTOUT=/tmp/xx
+rm -f ${BASH_TSTOUT}
+
+/bin/sh "$@"

diff --git a/include/shmbutil.h b/include/shmbutil.h
index dbaaba86f8aefe0274f2ae5a0195dfff3b811927..264cae33fbc6564239fb97fcfe6623fb7fc26451 100644 (file)
--- a/include/shmbutil.h
+++ b/include/shmbutil.h
@@ -37,6 +37,7 @@ extern size_t mbstrlen __P((const char *));
 extern char *xstrchr __P((const char *, int));
 
 extern int locale_mb_cur_max;  /* XXX */
+extern int locale_utf8locale;  /* XXX */
 
 #ifndef MB_INVALIDCH
 #define MB_INVALIDCH(x)                ((x) == (size_t)-1 || (x) == (size_t)-2)
@@ -109,6 +110,8 @@ extern int locale_mb_cur_max;       /* XXX */
            _f = is_basic ((_str)[_i]); \
            if (_f) \
              mblength = 1; \
+           else if (locale_utf8locale && (((_str)[_i] & 0x80) == 0)) \
+             mblength = 1; \
            else \
              { \
                state_bak = state; \
@@ -267,6 +270,8 @@ extern int locale_mb_cur_max;       /* XXX */
            _k = is_basic (*(_src)); \
            if (_k) \
              mblength = 1; \
+           else if (locale_utf8locale && ((*(_src) & 0x80) == 0)) \
+             mblength = 1; \
            else \
              { \
                state_bak = state; \

diff --git a/lib/readline/display.c b/lib/readline/display.c
index cd4f3fed7cc749cd65ab9eba423a1c56d8f2362c..0e0d44c3502417f4bfb9f9e2140d8175c195dbfa 100644 (file)
--- a/lib/readline/display.c
+++ b/lib/readline/display.c
@@ -124,7 +124,6 @@ static int _rl_col_width PARAMS((const char *, int, int, int));
 #define PROMPT_ENDING_INDEX \
   ((MB_CUR_MAX > 1 && rl_byte_oriented == 0) ? prompt_physical_chars : prompt_last_invisible+1)
   
-
 /* **************************************************************** */
 /*                                                                 */
 /*                     Display stuff                               */
@@ -844,8 +843,13 @@ rl_redisplay ()
   if (mb_cur_max > 1 && rl_byte_oriented == 0)
     {
       memset (&ps, 0, sizeof (mbstate_t));
-      /* XXX - what if wc_bytes ends up <= 0? check for MB_INVALIDCH */
-      wc_bytes = mbrtowc (&wc, rl_line_buffer, rl_end, &ps);
+      if (_rl_utf8locale && UTF8_SINGLEBYTE(rl_line_buffer[0]))
+       {
+         wc = (wchar_t)rl_line_buffer[0];
+         wc_bytes = 1;
+       }
+      else
+       wc_bytes = mbrtowc (&wc, rl_line_buffer, rl_end, &ps);
     }
   else
     wc_bytes = 1;
@@ -1005,13 +1009,18 @@ rl_redisplay ()
       if (mb_cur_max > 1 && rl_byte_oriented == 0)
        {
          in += wc_bytes;
-         /* XXX - what if wc_bytes ends up <= 0? check for MB_INVALIDCH */
-         wc_bytes = mbrtowc (&wc, rl_line_buffer + in, rl_end - in, &ps);
+         if (_rl_utf8locale && UTF8_SINGLEBYTE(rl_line_buffer[in]))
+           {
+             wc = (wchar_t)rl_line_buffer[in];
+             wc_bytes = 1;
+             memset (&ps, 0, sizeof (mbstate_t));      /* re-init state */
+           }
+         else
+           wc_bytes = mbrtowc (&wc, rl_line_buffer + in, rl_end - in, &ps);
        }
       else
         in++;
 #endif
-
     }
   line[out] = '\0';
   if (cpos_buffer_position < 0)
@@ -2961,7 +2970,13 @@ _rl_col_width (str, start, end, flags)
 
   while (point < start)
     {
-      tmp = mbrlen (str + point, max, &ps);
+      if (_rl_utf8locale && UTF8_SINGLEBYTE(str[point]))
+       {
+         memset (&ps, 0, sizeof (mbstate_t));
+         tmp = 1;
+       }
+      else
+       tmp = mbrlen (str + point, max, &ps);
       if (MB_INVALIDCH ((size_t)tmp))
        {
          /* In this case, the bytes are invalid or too short to compose a
@@ -2990,7 +3005,13 @@ _rl_col_width (str, start, end, flags)
 
   while (point < end)
     {
-      tmp = mbrtowc (&wc, str + point, max, &ps);
+      if (_rl_utf8locale && UTF8_SINGLEBYTE(str[point]))
+       {
+         tmp = 1;
+         wc = (wchar_t) str[point];
+       }
+      else
+       tmp = mbrtowc (&wc, str + point, max, &ps);
       if (MB_INVALIDCH ((size_t)tmp))
        {
          /* In this case, the bytes are invalid or too short to compose a

diff --git a/lib/readline/mbutil.c b/lib/readline/mbutil.c
index 314a49b64bff04a4f0f1bc9450497577de48324f..df1c48439422e239d251b6fad09fc0fded47df6c 100644 (file)
--- a/lib/readline/mbutil.c
+++ b/lib/readline/mbutil.c
@@ -105,7 +105,14 @@ _rl_find_next_mbchar_internal (string, seed, count, find_non_zero)
       len = strlen (string + point);
       if (len == 0)
        break;
-      tmp = mbrtowc (&wc, string+point, len, &ps);
+      if (_rl_utf8locale && UTF8_SINGLEBYTE(string[point]))
+       {
+         tmp = 1;
+         wc = (wchar_t) string[point];
+         memset(&ps, 0, sizeof(mbstate_t));
+       }
+      else
+       tmp = mbrtowc (&wc, string+point, len, &ps);
       if (MB_INVALIDCH ((size_t)tmp))
        {
          /* invalid bytes. assume a byte represents a character */
@@ -169,7 +176,7 @@ _rl_find_prev_mbchar_internal (string, seed, find_non_zero)
       tmp = mbrtowc (&wc, string + point, length - point, &ps);
       if (MB_INVALIDCH ((size_t)tmp))
        {
-         /* in this case, bytes are invalid or shorted to compose
+         /* in this case, bytes are invalid or too short to compose
             multibyte char, so assume that the first byte represents
             a single character anyway. */
          tmp = 1;
@@ -283,10 +290,13 @@ _rl_adjust_point (string, point, ps)
   
   while (pos < point)
     {
-      tmp = mbrlen (string + pos, length - pos, ps);
+      if (_rl_utf8locale && UTF8_SINGLEBYTE(string[pos]))
+       tmp = 1;
+      else
+       tmp = mbrlen (string + pos, length - pos, ps);
       if (MB_INVALIDCH ((size_t)tmp))
        {
-         /* in this case, bytes are invalid or shorted to compose
+         /* in this case, bytes are invalid or too short to compose
             multibyte char, so assume that the first byte represents
             a single character anyway. */
          pos++;
@@ -334,6 +344,8 @@ _rl_char_value (buf, ind)
 
   if (MB_LEN_MAX == 1 || rl_byte_oriented)
     return ((wchar_t) buf[ind]);
+  if (_rl_utf8locale && UTF8_SINGLEBYTE(buf[ind]))
+    return ((wchar_t) buf[ind]);
   l = strlen (buf);
   if (ind >= l - 1)
     return ((wchar_t) buf[ind]);

diff --git a/lib/readline/nls.c b/lib/readline/nls.c
index d3b36344f103b20945158f2e8a3db7a8a993d5f7..7f3e2d8a4e24137c3cd6e13f8d3ed81053845eff 100644 (file)
--- a/lib/readline/nls.c
+++ b/lib/readline/nls.c
@@ -74,6 +74,7 @@ static char *legal_lang_values[] =
  "iso88599",
  "iso885910",
  "koi8r",
+ "utf8",
   0
 };
 
@@ -118,16 +119,12 @@ utf8locale (lspec)
 #endif
 }
 
-/* Check for LC_ALL, LC_CTYPE, and LANG and use the first with a value
-   to decide the defaults for 8-bit character input and output.  Returns
-   1 if we set eight-bit mode. */
-int
-_rl_init_eightbit ()
+/* Query the right environment variables and call setlocale() to initialize
+   the C library locale settings. */
+char *
+_rl_init_locale ()
 {
-/* If we have setlocale(3), just check the current LC_CTYPE category
-   value, and go into eight-bit mode if it's not C or POSIX. */
-#if defined (HAVE_SETLOCALE)
-  char *lspec, *t;
+  char *ret, *lspec;
 
   /* Set the LC_CTYPE locale category from environment variables. */
   lspec = _rl_get_locale_var ("LC_CTYPE");
@@ -140,10 +137,25 @@ _rl_init_eightbit ()
     lspec = setlocale (LC_CTYPE, (char *)NULL);
   if (lspec == 0)
     lspec = "";
-  t = setlocale (LC_CTYPE, lspec);
+  ret = setlocale (LC_CTYPE, lspec);   /* ok, since it does not change locale */
+
+  _rl_utf8locale = (ret && *ret) ? utf8locale (ret) : 0;
+
+  return ret;
+}
+
+/* Check for LC_ALL, LC_CTYPE, and LANG and use the first with a value
+   to decide the defaults for 8-bit character input and output.  Returns
+   1 if we set eight-bit mode. */
+int
+_rl_init_eightbit ()
+{
+/* If we have setlocale(3), just check the current LC_CTYPE category
+   value, and go into eight-bit mode if it's not C or POSIX. */
+#if defined (HAVE_SETLOCALE)
+  char *lspec, *t;
 
-  if (t && *t)
-    _rl_utf8locale = utf8locale (t);
+  t = _rl_init_locale ();      /* returns static pointer */
 
   if (t && *t && (t[0] != 'C' || t[1]) && (STREQ (t, "POSIX") == 0))
     {
@@ -174,9 +186,11 @@ _rl_init_eightbit ()
        _rl_output_meta_chars = 1;
        break;
       }
+
+  _rl_utf8locale = *t ? STREQ (t, "utf8") : 0;
+
   xfree (t);
   return (legal_lang_values[i] ? 1 : 0);
-
 #endif /* !HAVE_SETLOCALE */
 }
 

diff --git a/lib/readline/readline.c b/lib/readline/readline.c
index 4e88dc76eaf69a05415da76e0459fc97c54a7cd3..c022fb1d7454763cc8f0978f2a5a12e6d4a2e0eb 100644 (file)
--- a/lib/readline/readline.c
+++ b/lib/readline/readline.c
@@ -1104,7 +1104,7 @@ rl_initialize ()
 {
   /* If we have never been called before, initialize the
      terminal and data structures. */
-  if (!rl_initialized)
+  if (rl_initialized == 0)
     {
       RL_SETSTATE(RL_STATE_INITIALIZING);
       readline_initialize_everything ();
@@ -1112,6 +1112,8 @@ rl_initialize ()
       rl_initialized++;
       RL_SETSTATE(RL_STATE_INITIALIZED);
     }
+  else
+    (void)_rl_init_locale ();  /* check current locale */
 
   /* Initialize the current line information. */
   _rl_init_line_state ();

diff --git a/lib/readline/rlmbutil.h b/lib/readline/rlmbutil.h
index 4748a610b0b749e44606922bac4ad7bc7fb4e69a..2d083b9f7db11a7c16921bfb8e8e99bc4217ee3e 100644 (file)
--- a/lib/readline/rlmbutil.h
+++ b/lib/readline/rlmbutil.h
@@ -172,6 +172,8 @@ _rl_wcwidth (wc)
 #  define IS_COMBINING_CHAR(x) (WCWIDTH(x) == 0)
 #endif
 
+#define UTF8_SINGLEBYTE(c)     (((c) & 0x80) == 0)
+
 #else /* !HANDLE_MULTIBYTE */
 
 #undef MB_LEN_MAX
@@ -196,6 +198,8 @@ _rl_wcwidth (wc)
 #define MB_INVALIDCH(x)                (0)
 #define MB_NULLWCH(x)          (0)
 
+#define UTF8_SINGLEBYTE(c)     (1)
+
 #endif /* !HANDLE_MULTIBYTE */
 
 extern int rl_byte_oriented;

diff --git a/lib/readline/rlprivate.h b/lib/readline/rlprivate.h
index f2d661636b5725c29d9ecb812d5876f1a783a763..1269e5c46d915b6fcbe883f28e85f8559ca340a2 100644 (file)
--- a/lib/readline/rlprivate.h
+++ b/lib/readline/rlprivate.h
@@ -331,6 +331,7 @@ extern void _rl_set_insert_mode PARAMS((int, int));
 extern void _rl_revert_all_lines PARAMS((void));
 
 /* nls.c */
+extern char *_rl_init_locale PARAMS((void));
 extern int _rl_init_eightbit PARAMS((void));
 
 /* parens.c */

diff --git a/lib/readline/text.c b/lib/readline/text.c
index c353252bc9fbe8adee0bf206a994c2d85d9eee73..e57344ad71b6b9dbb02b07b3a9ef77ec125cd299 100644 (file)
--- a/lib/readline/text.c
+++ b/lib/readline/text.c
@@ -695,6 +695,12 @@ _rl_insert_char (count, c)
       incoming[1] = '\0';
       incoming_length = 1;
     }
+  else if (_rl_utf8locale && (c & 0x80) == 0)
+    {
+      incoming[0] = c;
+      incoming[1] = '\0';
+      incoming_length = 1;
+    }
   else
     {
       wchar_t wc;
@@ -739,6 +745,12 @@ _rl_insert_char (count, c)
             effect of mbstate is undefined. */
          memset (&ps, 0, sizeof (mbstate_t));
        }
+      else if (ret == 1)
+       {
+         incoming[0] = pending_bytes[0];
+         incoming[incoming_length = 1] = '\0';
+         pending_bytes_length = 0;
+       }
       else
        {
          /* We successfully read a single multibyte character. */
@@ -761,8 +773,13 @@ _rl_insert_char (count, c)
       i = 0;
       while (i < string_size)
        {
-         strncpy (string + i, incoming, incoming_length);
-         i += incoming_length;
+         if (incoming_length == 1)
+           string[i++] = *incoming;
+         else
+           {
+             strncpy (string + i, incoming, incoming_length);
+             i += incoming_length;
+           }
        }
       incoming_length = 0;
       stored_count = 0;
@@ -790,8 +807,13 @@ _rl_insert_char (count, c)
       i = 0;
       while (i < string_size)
        {
-         strncpy (string + i, incoming, incoming_length);
-         i += incoming_length;
+         if (incoming_length == 1)
+           string[i++] = *incoming;
+         else
+           {
+             strncpy (string + i, incoming, incoming_length);
+             i += incoming_length;
+           }
        }
 
       while (count)

diff --git a/locale.c b/locale.c
index 7aa150aa55878199001a0720e8f34d5415b117f7..c42046bb83d872e1ef77aef7e57a915e65c4eb04 100644 (file)
--- a/locale.c
+++ b/locale.c
@@ -45,6 +45,7 @@ extern int errno;
 
 int locale_utf8locale; /* set but unused for now */
 int locale_mb_cur_max; /* value of MB_CUR_MAX for current locale (LC_CTYPE) */
+int locale_shiftstates;
 
 extern int dump_translatable_strings, dump_po_strings;
 
@@ -85,6 +86,7 @@ set_default_locale ()
 
   locale_mb_cur_max = MB_CUR_MAX;
   locale_utf8locale = locale_isutf8 (default_locale);
+  locale_shiftstates = mblen ((char *)NULL, 0);
 }
 
 /* Set default values for LC_CTYPE, LC_COLLATE, LC_MESSAGES, LC_NUMERIC and
@@ -105,6 +107,7 @@ set_default_locale_vars ()
       locale_setblanks ();
       locale_mb_cur_max = MB_CUR_MAX;
       locale_utf8locale = locale_isutf8 (lc_all);
+      locale_shiftstates = mblen ((char *)NULL, 0);
       u32reset ();
     }
 #  endif
@@ -208,6 +211,7 @@ set_locale_var (var, value)
       /* if LC_ALL == "", reset_locale_vars has already called this */
       if (*lc_all && x)
        locale_utf8locale = locale_isutf8 (lc_all);
+      locale_shiftstates = mblen ((char *)NULL, 0);
       u32reset ();
       return r;
 #else
@@ -227,6 +231,7 @@ set_locale_var (var, value)
          /* if setlocale() returns NULL, the locale is not changed */
          if (x)
            locale_utf8locale = locale_isutf8 (x);
+         locale_shiftstates = mblen ((char *)NULL, 0);
          u32reset ();
        }
 #  endif
@@ -363,6 +368,7 @@ reset_locale_vars ()
   locale_mb_cur_max = MB_CUR_MAX;
   if (x)
     locale_utf8locale = locale_isutf8 (x);
+  locale_shiftstates = mblen ((char *)NULL, 0);
   u32reset ();
 #endif
   return 1;

diff --git a/patchlevel.h b/patchlevel.h
index 1cd7c96c7bf94ae7242e8af39fce6b1c82d94b20..1bc098b84f43c98059b287b9d16049ec84a76278 100644 (file)
--- a/patchlevel.h
+++ b/patchlevel.h
@@ -25,6 +25,6 @@
    regexp `^#define[   ]*PATCHLEVEL', since that's what support/mkversion.sh
    looks for to find the patch level (for the sccs version string). */
 
-#define PATCHLEVEL 0
+#define PATCHLEVEL 5
 
 #endif /* _PATCHLEVEL_H_ */

diff --git a/shell.h b/shell.h
index 6a6bdcdb03b41205e3437f4839cb58775dcb1c8d..a8ca731f1f6bb5bf4270314d1de3589f3c83e9a4 100644 (file)
--- a/shell.h
+++ b/shell.h
@@ -99,6 +99,7 @@ extern int subshell_environment;
 extern int shell_compatibility_level;
 
 extern int locale_mb_cur_max;
+extern int locale_utf8locale;
 
 /* Structure to pass around that holds a bitmap of file descriptors
    to close, and the size of that structure.  Used in execute_cmd.c. */

diff --git a/tests/RUN-ONE-TEST b/tests/RUN-ONE-TEST
index d29259a940a27237e8cbf3877622680101b329a1..a5ab273deeae63fdcdbc137f2b477bc07f5e91e1 100755 (executable)
--- a/tests/RUN-ONE-TEST
+++ b/tests/RUN-ONE-TEST
@@ -1,4 +1,4 @@
-BUILD_DIR=/usr/local/build/chet/bash/bash-current
+BUILD_DIR=/usr/local/build/bash/bash-current
 THIS_SH=$BUILD_DIR/bash
 PATH=$PATH:$BUILD_DIR
 

diff --git a/tests/glob.right b/tests/glob.right
index 68c0666f0689f9e872aac2f5c74e16d40472b528..330f65601cfe9add12d254fb3b64003426079522 100644 (file)
--- a/tests/glob.right
+++ b/tests/glob.right
@@ -9,7 +9,7 @@ argv[1] = <A
 argv[1] = <A>
 argv[1] = <B>
 argv[1] = <a£\b>
-0000000   141 243 134 142                                                
+0000000 141 243 134 142
 0000004
 ok 6
 ok 7

diff --git a/tests/glob2.sub b/tests/glob2.sub
index 2a74340949c6b2167530553f34da43ac0760962e..d33013b672ac4b8fb3e4b3122a846d99db17f310 100644 (file)
--- a/tests/glob2.sub
+++ b/tests/glob2.sub
@@ -1,3 +1,5 @@
+. ./test-glue-functions
+
 var='ab\'
 
 case $var in
@@ -34,7 +36,7 @@ recho "$b"
 unset a b v
 
 recho "a${alpha}b"
-printf "%s" "a${alpha}b" | LC_ALL=C od -b
+printf "%s" "a${alpha}b" | LC_ALL=C od -b | _intl_normalize_spaces
 
 a=$'\u3b1'
 [[ $a = $a ]] && echo ok 6