1 Submitted by: DJ Lucas (dj_AT_linuxfromscratch_DOT_org)
3 Initial Package Version: 8.27
4 Upstream Status: Rejected
5 Origin: Based on Fedora's i18n patches at
6 http://pkgs.fedoraproject.org/cgit/rpms/coreutils.git/tree/
7 Description: Fixes i18n issues with various Coreutils programs
9 diff -Naurp coreutils-8.27-orig/bootstrap.conf coreutils-8.27/bootstrap.conf
10 --- coreutils-8.27-orig/bootstrap.conf 2017-03-07 23:34:06.000000000 -0600
11 +++ coreutils-8.27/bootstrap.conf 2017-03-11 23:47:38.068058445 -0600
12 @@ -152,6 +152,7 @@ gnulib_modules="
20 diff -Naurp coreutils-8.27-orig/configure.ac coreutils-8.27/configure.ac
21 --- coreutils-8.27-orig/configure.ac 2017-02-26 08:52:29.000000000 -0600
22 +++ coreutils-8.27/configure.ac 2017-03-11 23:47:38.068058445 -0600
23 @@ -429,6 +429,8 @@ fi
24 # I'm leaving it here for now. This whole thing needs to be modernized...
29 gl_HEADER_TIOCGWINSZ_IN_TERMIOS_H
31 if test $gl_cv_sys_tiocgwinsz_needs_termios_h = no && \
32 diff -Naurp coreutils-8.27-orig/lib/linebuffer.h coreutils-8.27/lib/linebuffer.h
33 --- coreutils-8.27-orig/lib/linebuffer.h 2017-01-01 16:35:38.000000000 -0600
34 +++ coreutils-8.27/lib/linebuffer.h 2017-03-11 23:47:13.089286391 -0600
44 /* A 'struct linebuffer' holds a line of text. */
47 @@ -28,6 +33,9 @@ struct linebuffer
48 size_t size; /* Allocated. */
49 size_t length; /* Used. */
56 /* Initialize linebuffer LINEBUFFER for use. */
57 diff -Naurp coreutils-8.27-orig/lib/mbfile.c coreutils-8.27/lib/mbfile.c
58 --- coreutils-8.27-orig/lib/mbfile.c 1969-12-31 18:00:00.000000000 -0600
59 +++ coreutils-8.27/lib/mbfile.c 2017-03-11 23:47:38.069058397 -0600
62 +#define MBFILE_INLINE _GL_EXTERN_INLINE
64 diff -Naurp coreutils-8.27-orig/lib/mbfile.h coreutils-8.27/lib/mbfile.h
65 --- coreutils-8.27-orig/lib/mbfile.h 1969-12-31 18:00:00.000000000 -0600
66 +++ coreutils-8.27/lib/mbfile.h 2017-03-11 23:47:38.069058397 -0600
68 +/* Multibyte character I/O: macros for multi-byte encodings.
69 + Copyright (C) 2001, 2005, 2009-2017 Free Software Foundation, Inc.
71 + This program is free software: you can redistribute it and/or modify
72 + it under the terms of the GNU General Public License as published by
73 + the Free Software Foundation; either version 3 of the License, or
74 + (at your option) any later version.
76 + This program is distributed in the hope that it will be useful,
77 + but WITHOUT ANY WARRANTY; without even the implied warranty of
78 + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
79 + GNU General Public License for more details.
81 + You should have received a copy of the GNU General Public License
82 + along with this program. If not, see <http://www.gnu.org/licenses/>. */
84 +/* Written by Mitsuru Chinen <mchinen@yamato.ibm.com>
85 + and Bruno Haible <bruno@clisp.org>. */
87 +/* The macros in this file implement multi-byte character input from a
91 + is the type for multibyte character input stream, usable for variable
95 + is the type for multibyte character or EOF, usable for variable
98 + mbf_init (mbf, stream)
99 + initializes the MB_FILE for reading from stream.
101 + mbf_getc (mbc, mbf)
102 + reads the next multibyte character from mbf and stores it in mbc.
105 + returns true if mbc represents the EOF value.
107 + Here are the function prototypes of the macros.
109 + extern void mbf_init (mb_file_t mbf, FILE *stream);
110 + extern void mbf_getc (mbf_char_t mbc, mb_file_t mbf);
111 + extern bool mb_iseof (const mbf_char_t mbc);
118 +#include <stdbool.h>
122 +/* Tru64 with Desktop Toolkit C has a bug: <stdio.h> must be included before
124 + BSD/OS 4.1 has a bug: <stdio.h> and <time.h> must be included before
132 +#ifndef _GL_INLINE_HEADER_BEGIN
133 + #error "Please include config.h first."
135 +_GL_INLINE_HEADER_BEGIN
136 +#ifndef MBFILE_INLINE
137 +# define MBFILE_INLINE _GL_INLINE
140 +struct mbfile_multi {
143 + bool have_pushback;
145 + unsigned int bufcount;
146 + char buf[MBCHAR_BUF_SIZE];
147 + struct mbchar pushback;
151 +mbfile_multi_getc (struct mbchar *mbc, struct mbfile_multi *mbf)
155 + /* If EOF has already been seen, don't use getc. This matters if
156 + mbf->fp is connected to an interactive tty. */
160 + /* Return character pushed back, if there is one. */
161 + if (mbf->have_pushback)
163 + mb_copy (mbc, &mbf->pushback);
164 + mbf->have_pushback = false;
168 + /* Before using mbrtowc, we need at least one byte. */
169 + if (mbf->bufcount == 0)
171 + int c = getc (mbf->fp);
174 + mbf->eof_seen = true;
177 + mbf->buf[0] = (unsigned char) c;
181 + /* Handle most ASCII characters quickly, without calling mbrtowc(). */
182 + if (mbf->bufcount == 1 && mbsinit (&mbf->state) && is_basic (mbf->buf[0]))
184 + /* These characters are part of the basic character set. ISO C 99
185 + guarantees that their wide character code is identical to their
187 + mbc->wc = mbc->buf[0] = mbf->buf[0];
188 + mbc->wc_valid = true;
189 + mbc->ptr = &mbc->buf[0];
195 + /* Use mbrtowc on an increasing number of bytes. Read only as many bytes
196 + from mbf->fp as needed. This is needed to give reasonable interactive
197 + behaviour when mbf->fp is connected to an interactive tty. */
200 + /* We don't know whether the 'mbrtowc' function updates the state when
201 + it returns -2, - this is the ISO C 99 and glibc-2.2 behaviour - or
202 + not - amended ANSI C, glibc-2.1 and Solaris 2.7 behaviour. We
203 + don't have an autoconf test for this, yet.
204 + The new behaviour would allow us to feed the bytes one by one into
205 + mbrtowc. But the old behaviour forces us to feed all bytes since
206 + the end of the last character into mbrtowc. Since we want to retry
207 + with more bytes when mbrtowc returns -2, we must backup the state
208 + before calling mbrtowc, because implementations with the new
209 + behaviour will clobber it. */
210 + mbstate_t backup_state = mbf->state;
212 + bytes = mbrtowc (&mbc->wc, &mbf->buf[0], mbf->bufcount, &mbf->state);
214 + if (bytes == (size_t) -1)
216 + /* An invalid multibyte sequence was encountered. */
217 + /* Return a single byte. */
219 + mbc->wc_valid = false;
222 + else if (bytes == (size_t) -2)
224 + /* An incomplete multibyte character. */
225 + mbf->state = backup_state;
226 + if (mbf->bufcount == MBCHAR_BUF_SIZE)
228 + /* An overlong incomplete multibyte sequence was encountered. */
229 + /* Return a single byte. */
231 + mbc->wc_valid = false;
236 + /* Read one more byte and retry mbrtowc. */
237 + int c = getc (mbf->fp);
240 + /* An incomplete multibyte character at the end. */
241 + mbf->eof_seen = true;
242 + bytes = mbf->bufcount;
243 + mbc->wc_valid = false;
246 + mbf->buf[mbf->bufcount] = (unsigned char) c;
254 + /* A null wide character was encountered. */
256 + assert (mbf->buf[0] == '\0');
257 + assert (mbc->wc == 0);
259 + mbc->wc_valid = true;
264 + /* Return the multibyte sequence mbf->buf[0..bytes-1]. */
265 + mbc->ptr = &mbc->buf[0];
266 + memcpy (&mbc->buf[0], &mbf->buf[0], bytes);
267 + mbc->bytes = bytes;
269 + mbf->bufcount -= bytes;
270 + if (mbf->bufcount > 0)
272 + /* It's not worth calling memmove() for so few bytes. */
273 + unsigned int count = mbf->bufcount;
274 + char *p = &mbf->buf[0];
281 + while (--count > 0);
286 + /* An mbchar_t with bytes == 0 is used to indicate EOF. */
289 + mbc->wc_valid = false;
294 +mbfile_multi_ungetc (const struct mbchar *mbc, struct mbfile_multi *mbf)
296 + mb_copy (&mbf->pushback, mbc);
297 + mbf->have_pushback = true;
300 +typedef struct mbfile_multi mb_file_t;
302 +typedef mbchar_t mbf_char_t;
304 +#define mbf_init(mbf, stream) \
305 + ((mbf).fp = (stream), \
306 + (mbf).eof_seen = false, \
307 + (mbf).have_pushback = false, \
308 + memset (&(mbf).state, '\0', sizeof (mbstate_t)), \
309 + (mbf).bufcount = 0)
311 +#define mbf_getc(mbc, mbf) mbfile_multi_getc (&(mbc), &(mbf))
313 +#define mbf_ungetc(mbc, mbf) mbfile_multi_ungetc (&(mbc), &(mbf))
315 +#define mb_iseof(mbc) ((mbc).bytes == 0)
317 +#ifndef _GL_INLINE_HEADER_BEGIN
318 + #error "Please include config.h first."
320 +_GL_INLINE_HEADER_BEGIN
322 +#endif /* _MBFILE_H */
323 diff -Naurp coreutils-8.27-orig/m4/mbfile.m4 coreutils-8.27/m4/mbfile.m4
324 --- coreutils-8.27-orig/m4/mbfile.m4 1969-12-31 18:00:00.000000000 -0600
325 +++ coreutils-8.27/m4/mbfile.m4 2017-03-11 23:47:38.070058349 -0600
327 +# mbfile.m4 serial 7
328 +dnl Copyright (C) 2005, 2008-2017 Free Software Foundation, Inc.
329 +dnl This file is free software; the Free Software Foundation
330 +dnl gives unlimited permission to copy and/or distribute it,
331 +dnl with or without modifications, as long as this notice is preserved.
333 +dnl autoconf tests required for use of mbfile.h
334 +dnl From Bruno Haible.
336 +AC_DEFUN([gl_MBFILE],
338 + AC_REQUIRE([AC_TYPE_MBSTATE_T])
341 diff -Naurp coreutils-8.27-orig/src/cut.c coreutils-8.27/src/cut.c
342 --- coreutils-8.27-orig/src/cut.c 2017-01-01 16:34:24.000000000 -0600
343 +++ coreutils-8.27/src/cut.c 2017-03-11 23:47:59.526048471 -0600
347 #include <sys/types.h>
349 +/* Get mbstate_t, mbrtowc(). */
358 #include "set-fields.h"
360 +/* MB_LEN_MAX is incorrectly defined to be 1 in at least one GCC
361 + installation; work around this configuration error. */
362 +#if !defined MB_LEN_MAX || MB_LEN_MAX < 2
364 +# define MB_LEN_MAX 16
367 +/* Some systems, like BeOS, have multibyte encodings but lack mbstate_t. */
368 +#if HAVE_MBRTOWC && defined mbstate_t
369 +# define mbrtowc(pwc, s, n, ps) (mbrtowc) (pwc, s, n, 0)
372 /* The official name of this program (e.g., no 'g' prefix). */
373 #define PROGRAM_NAME "cut"
379 +/* Refill the buffer BUF to get a multibyte character. */
380 +#define REFILL_BUFFER(BUF, BUFPOS, BUFLEN, STREAM) \
383 + if (BUFLEN < MB_LEN_MAX && !feof (STREAM) && !ferror (STREAM)) \
385 + memmove (BUF, BUFPOS, BUFLEN); \
386 + BUFLEN += fread (BUF + BUFLEN, sizeof(char), BUFSIZ, STREAM); \
392 +/* Get wide character on BUFPOS. BUFPOS is not included after that.
393 + If byte sequence is not valid as a character, CONVFAIL is true. Otherwise false. */
394 +#define GET_NEXT_WC_FROM_BUFFER(WC, BUFPOS, BUFLEN, MBLENGTH, STATE, CONVFAIL) \
397 + mbstate_t state_bak; \
405 + /* Get a wide character. */ \
406 + CONVFAIL = false; \
407 + state_bak = STATE; \
408 + MBLENGTH = mbrtowc ((wchar_t *)&WC, BUFPOS, BUFLEN, &STATE); \
410 + switch (MBLENGTH) \
415 + STATE = state_bak; \
416 + /* Fall througn. */ \
426 /* Pointer inside RP. When checking if a byte or field is selected
427 by a finite range, we check if it is between CURRENT_RP.LO
429 CURRENT_RP.HI then we make CURRENT_RP to point to the next range pair. */
430 static struct field_range_pair *current_rp;
432 +/* Length of the delimiter given as argument to -d. */
435 /* This buffer is used to support the semantics of the -s option
436 (or lack of same) when the specified field list includes (does
437 not include) the first field. In both of those cases, the entire
438 @@ -77,15 +143,25 @@ enum operating_mode
442 - /* Output characters that are in the given bytes. */
443 + /* Output bytes that are at the given positions. */
446 + /* Output characters that are at the given positions. */
449 /* Output the given delimiter-separated fields. */
453 static enum operating_mode operating_mode;
455 +/* If nonzero, when in byte mode, don't split multibyte characters. */
456 +static int byte_mode_character_aware;
458 +/* If nonzero, the function for single byte locale is work
459 + if this program runs on multibyte locale. */
460 +static int force_singlebyte_mode;
462 /* If true do not output lines containing no delimiter characters.
463 Otherwise, all such lines are printed. This option is valid only
465 @@ -97,6 +173,9 @@ static bool complement;
467 /* The delimiter character for field mode. */
468 static unsigned char delim;
470 +static wchar_t wcdelim;
473 /* The delimiter for each line/record. */
474 static unsigned char line_delim = '\n';
475 @@ -164,7 +243,7 @@ Print selected parts of lines from each
476 -f, --fields=LIST select only these fields; also print any line\n\
477 that contains no delimiter character, unless\n\
478 the -s option is specified\n\
480 + -n with -b: don't split multibyte characters\n\
483 --complement complement the set of selected bytes, characters\n\
484 @@ -280,6 +359,82 @@ cut_bytes (FILE *stream)
489 +/* This function is in use for the following case.
491 + 1. Read from the stream STREAM, printing to standard output any selected
494 + 2. Read from stream STREAM, printing to standard output any selected bytes,
495 + without splitting multibyte characters. */
498 +cut_characters_or_cut_bytes_no_split (FILE *stream)
500 + size_t idx; /* number of bytes or characters in the line so far. */
501 + char buf[MB_LEN_MAX + BUFSIZ]; /* For spooling a read byte sequence. */
502 + char *bufpos; /* Next read position of BUF. */
503 + size_t buflen; /* The length of the byte sequence in buf. */
504 + wint_t wc; /* A gotten wide character. */
505 + size_t mblength; /* The byte size of a multibyte character which shows
506 + as same character as WC. */
507 + mbstate_t state; /* State of the stream. */
508 + bool convfail = false; /* true, when conversion failed. Otherwise false. */
509 + /* Whether to begin printing delimiters between ranges for the current line.
510 + Set after we've begun printing data corresponding to the first range. */
511 + bool print_delimiter = false;
516 + memset (&state, '\0', sizeof(mbstate_t));
522 + REFILL_BUFFER (buf, bufpos, buflen, stream);
524 + GET_NEXT_WC_FROM_BUFFER (wc, bufpos, buflen, mblength, state, convfail);
525 + (void) convfail; /* ignore unused */
530 + putchar (line_delim);
533 + else if (wc == line_delim)
535 + putchar (line_delim);
537 + print_delimiter = false;
543 + if (print_kth (idx))
545 + if (output_delimiter_specified)
547 + if (print_delimiter && is_range_start_index (idx))
549 + fwrite (output_delimiter_string, sizeof (char),
550 + output_delimiter_length, stdout);
552 + print_delimiter = true;
554 + fwrite (bufpos, mblength, sizeof(char), stdout);
558 + buflen -= mblength;
559 + bufpos += mblength;
564 /* Read from stream STREAM, printing to standard output any selected fields. */
567 @@ -425,13 +580,211 @@ cut_fields (FILE *stream)
573 +cut_fields_mb (FILE *stream)
577 + int found_any_selected_field;
578 + int buffer_first_field;
580 + char buf[MB_LEN_MAX + BUFSIZ]; /* For spooling a read byte sequence. */
581 + char *bufpos; /* Next read position of BUF. */
582 + size_t buflen; /* The length of the byte sequence in buf. */
583 + wint_t wc = 0; /* A gotten wide character. */
584 + size_t mblength; /* The byte size of a multibyte character which shows
585 + as same character as WC. */
586 + mbstate_t state; /* State of the stream. */
587 + bool convfail = false; /* true, when conversion failed. Otherwise false. */
591 + found_any_selected_field = 0;
595 + memset (&state, '\0', sizeof(mbstate_t));
598 + empty_input = (c == EOF);
601 + ungetc (c, stream);
607 + /* To support the semantics of the -s flag, we may have to buffer
608 + all of the first field to determine whether it is `delimited.'
609 + But that is unnecessary if all non-delimited lines must be printed
610 + and the first field has been selected, or if non-delimited lines
611 + must be suppressed and the first field has *not* been selected.
612 + That is because a non-delimited line has exactly one field. */
613 + buffer_first_field = (suppress_non_delimited ^ !print_kth (1));
617 + if (field_idx == 1 && buffer_first_field)
623 + REFILL_BUFFER (buf, bufpos, buflen, stream);
625 + GET_NEXT_WC_FROM_BUFFER
626 + (wc, bufpos, buflen, mblength, state, convfail);
631 + field_1_buffer = xrealloc (field_1_buffer, len + mblength);
632 + memcpy (field_1_buffer + len, bufpos, mblength);
634 + buflen -= mblength;
635 + bufpos += mblength;
637 + if (!convfail && (wc == line_delim || wc == wcdelim))
641 + if (len <= 0 && wc == WEOF)
644 + /* If the first field extends to the end of line (it is not
645 + delimited) and we are printing all non-delimited lines,
647 + if (convfail || (!convfail && wc != wcdelim))
649 + if (suppress_non_delimited)
655 + fwrite (field_1_buffer, sizeof (char), len, stdout);
656 + /* Make sure the output line is newline terminated. */
657 + if (convfail || (!convfail && wc != line_delim))
658 + putchar (line_delim);
665 + /* Print the field, but not the trailing delimiter. */
666 + fwrite (field_1_buffer, sizeof (char), len - 1, stdout);
667 + found_any_selected_field = 1;
669 + next_item (&field_idx);
674 + if (print_kth (field_idx))
676 + if (found_any_selected_field)
678 + fwrite (output_delimiter_string, sizeof (char),
679 + output_delimiter_length, stdout);
681 + found_any_selected_field = 1;
686 + REFILL_BUFFER (buf, bufpos, buflen, stream);
688 + GET_NEXT_WC_FROM_BUFFER
689 + (wc, bufpos, buflen, mblength, state, convfail);
693 + else if (!convfail && (wc == wcdelim || wc == line_delim))
695 + buflen -= mblength;
696 + bufpos += mblength;
700 + if (print_kth (field_idx))
701 + fwrite (bufpos, mblength, sizeof(char), stdout);
703 + buflen -= mblength;
704 + bufpos += mblength;
708 + if ((!convfail || wc == line_delim) && buflen < 1)
711 + if (!convfail && wc == wcdelim)
712 + next_item (&field_idx);
713 + else if (wc == WEOF || (!convfail && wc == line_delim))
715 + if (found_any_selected_field
716 + || (!empty_input && !(suppress_non_delimited && field_idx == 1)))
717 + putchar (line_delim);
722 + found_any_selected_field = 0;
729 cut_stream (FILE *stream)
731 - if (operating_mode == byte_mode)
732 - cut_bytes (stream);
734 + if (MB_CUR_MAX > 1 && !force_singlebyte_mode)
736 + switch (operating_mode)
739 + if (byte_mode_character_aware)
740 + cut_characters_or_cut_bytes_no_split (stream);
742 + cut_bytes (stream);
745 + case character_mode:
746 + cut_characters_or_cut_bytes_no_split (stream);
752 + /* Check if we have utf8 multibyte locale, so we can use this
753 + optimization because of uniqueness of characters, which is
754 + not true for e.g. SJIS */
755 + char * loc = setlocale(LC_CTYPE, NULL);
756 + if (loc && (strstr (loc, "UTF-8") || strstr (loc, "utf-8") ||
757 + strstr (loc, "UTF8") || strstr (loc, "utf8")))
759 + cut_fields (stream);
763 + cut_fields_mb (stream);
771 - cut_fields (stream);
774 + if (operating_mode == field_mode)
775 + cut_fields (stream);
777 + cut_bytes (stream);
781 /* Process file FILE to standard output.
782 @@ -483,6 +836,7 @@ main (int argc, char **argv)
784 bool delim_specified = false;
785 char *spec_list_string IF_LINT ( = NULL);
786 + char mbdelim[MB_LEN_MAX + 1];
788 initialize_main (&argc, &argv);
789 set_program_name (argv[0]);
790 @@ -505,7 +859,6 @@ main (int argc, char **argv)
795 /* Build the byte list. */
796 if (operating_mode != undefined_mode)
797 FATAL_ERROR (_("only one type of list may be specified"));
798 @@ -513,6 +866,14 @@ main (int argc, char **argv)
799 spec_list_string = optarg;
803 + /* Build the character list. */
804 + if (operating_mode != undefined_mode)
805 + FATAL_ERROR (_("only one type of list may be specified"));
806 + operating_mode = character_mode;
807 + spec_list_string = optarg;
811 /* Build the field list. */
812 if (operating_mode != undefined_mode)
813 @@ -524,10 +885,38 @@ main (int argc, char **argv)
816 /* Interpret -d '' to mean 'use the NUL byte as the delimiter.' */
817 - if (optarg[0] != '\0' && optarg[1] != '\0')
818 - FATAL_ERROR (_("the delimiter must be a single character"));
820 - delim_specified = true;
827 + memset (&state, '\0', sizeof(mbstate_t));
828 + delimlen = mbrtowc (&wcdelim, optarg, strnlen(optarg, MB_LEN_MAX), &state);
830 + if (delimlen == (size_t)-1 || delimlen == (size_t)-2)
831 + ++force_singlebyte_mode;
834 + delimlen = (delimlen < 1) ? 1 : delimlen;
835 + if (wcdelim != L'\0' && *(optarg + delimlen) != '\0')
836 + FATAL_ERROR (_("the delimiter must be a single character"));
837 + memcpy (mbdelim, optarg, delimlen);
838 + mbdelim[delimlen] = '\0';
844 + if (MB_CUR_MAX <= 1 || force_singlebyte_mode)
847 + if (optarg[0] != '\0' && optarg[1] != '\0')
848 + FATAL_ERROR (_("the delimiter must be a single character"));
849 + delim = (unsigned char) optarg[0];
851 + delim_specified = true;
855 case OUTPUT_DELIMITER_OPTION:
856 @@ -540,6 +929,7 @@ main (int argc, char **argv)
860 + byte_mode_character_aware = 1;
864 @@ -579,15 +969,34 @@ main (int argc, char **argv)
865 | (complement ? SETFLD_COMPLEMENT : 0) );
867 if (!delim_specified)
879 if (output_delimiter_string == NULL)
881 - static char dummy[2];
884 - output_delimiter_string = dummy;
885 - output_delimiter_length = 1;
887 + if (MB_CUR_MAX > 1 && !force_singlebyte_mode)
889 + output_delimiter_string = xstrdup(mbdelim);
890 + output_delimiter_length = delimlen;
893 + if (MB_CUR_MAX <= 1 || force_singlebyte_mode)
896 + static char dummy[2];
899 + output_delimiter_string = dummy;
900 + output_delimiter_length = 1;
905 diff -Naurp coreutils-8.27-orig/src/expand.c coreutils-8.27/src/expand.c
906 --- coreutils-8.27-orig/src/expand.c 2017-02-26 15:42:25.000000000 -0600
907 +++ coreutils-8.27/src/expand.c 2017-03-11 23:49:06.758133530 -0600
911 #include <sys/types.h>
917 #include "xstrndup.h"
918 @@ -100,19 +103,41 @@ expand (void)
921 FILE *fp = next_file (NULL);
924 + /* True if the starting locale is utf8. */
925 + bool using_utf_locale;
927 + /* True if the first file contains BOM header. */
929 + using_utf_locale=check_utf_locale();
933 + mbf_init (mbf, fp);
934 + found_bom=check_bom(fp,&mbf);
937 + if (using_utf_locale == false && found_bom == true)
939 + /*try using some predefined locale */
941 + if (set_utf_locale () != 0)
943 - /* Input character, or EOF. */
945 + error (EXIT_FAILURE, errno, _("cannot set UTF-8 locale"));
950 + if (found_bom == true)
957 /* If true, perform translations. */
961 /* The following variables have valid values only when CONVERT
964 @@ -122,17 +147,48 @@ expand (void)
965 /* Index in TAB_LIST of next tab stop to examine. */
966 size_t tab_index = 0;
969 /* Convert a line of text. */
973 - while ((c = getc (fp)) < 0 && (fp = next_file (fp)))
977 + if ((mb_iseof (c)) && (fp = next_file (fp)))
979 + mbf_init (mbf, fp);
982 + if (check_bom(fp,&mbf)==true)
984 + /*Not the first file - check BOM header*/
985 + if (using_utf_locale==false && found_bom==false)
987 + /*BOM header in subsequent file but not in the first one. */
988 + error (EXIT_FAILURE, errno, _("combination of files with and without BOM header"));
993 + if(using_utf_locale==false && found_bom==true)
995 + /*First file conatined BOM header - locale was switched to UTF
996 + /*all subsequent files should contain BOM. */
997 + error (EXIT_FAILURE, errno, _("combination of files with and without BOM header"));
1013 + if (mb_iseq (c, '\t'))
1015 /* Column the next input tab stop is on. */
1016 uintmax_t next_tab_column;
1017 @@ -151,32 +207,34 @@ expand (void)
1018 if (putchar (' ') < 0)
1019 die (EXIT_FAILURE, errno, _("write error"));
1022 + mb_setascii (&c, ' ');
1024 - else if (c == '\b')
1025 + else if (mb_iseq (c, '\b'))
1027 /* Go back one column, and force recalculation of the
1030 tab_index -= !!tab_index;
1033 + /* A leading control character could make us trip over. */
1034 + else if (!mb_iscntrl (c))
1037 + column += mb_width (c);
1039 die (EXIT_FAILURE, 0, _("input line is too long"));
1042 - convert &= convert_entire_line || !! isblank (c);
1043 + convert &= convert_entire_line || mb_isblank (c);
1050 - if (putchar (c) < 0)
1051 + mb_putc (c, stdout);
1052 + if (ferror (stdout))
1053 die (EXIT_FAILURE, errno, _("write error"));
1055 - while (c != '\n');
1056 + while (!mb_iseq (c, '\n'));
1060 diff -Naurp coreutils-8.27-orig/src/expand-common.c coreutils-8.27/src/expand-common.c
1061 --- coreutils-8.27-orig/src/expand-common.c 2017-03-01 11:22:55.000000000 -0600
1062 +++ coreutils-8.27/src/expand-common.c 2017-03-11 23:49:06.757133570 -0600
1066 #include <sys/types.h>
1067 +#include <mbfile.h>
1071 @@ -105,6 +106,119 @@ set_extend_size (uintmax_t tabval)
1076 +set_utf_locale (void)
1078 + /*try using some predefined locale */
1079 + const char* predef_locales[] = {"C.UTF8","en_US.UTF8","en_GB.UTF8"};
1081 + const int predef_locales_count=3;
1082 + for (int i=0;i<predef_locales_count;i++)
1084 + if (setlocale(LC_ALL,predef_locales[i])!=NULL)
1088 + else if (i==predef_locales_count-1)
1091 + error (EXIT_FAILURE, errno, _("cannot set UTF-8 locale"));
1098 +check_utf_locale(void)
1100 + char* locale = setlocale (LC_CTYPE , NULL);
1101 + if (locale == NULL)
1105 + else if (strcasestr(locale, "utf8") == NULL && strcasestr(locale, "utf-8") == NULL)
1113 +check_bom(FILE* fp, mb_file_t *mbf)
1120 + /*test BOM header of the first file */
1143 + mbf->buf[0]=(unsigned char) 0xEF;
1163 + mbf->buf[0]=(unsigned char) 0xEF;
1164 + mbf->buf[1]=(unsigned char) 0xBB;
1171 + mbf->buf[0]=(unsigned char) 0xEF;
1183 + putc (0xEF, stdout);
1184 + putc (0xBB, stdout);
1185 + putc (0xBF, stdout);
1188 /* Add the comma or blank separated list of tab stops STOPS
1189 to the list of tab stops. */
1191 diff -Naurp coreutils-8.27-orig/src/expand-common.h coreutils-8.27/src/expand-common.h
1192 --- coreutils-8.27-orig/src/expand-common.h 2017-01-01 16:34:24.000000000 -0600
1193 +++ coreutils-8.27/src/expand-common.h 2017-03-11 23:49:06.758133530 -0600
1194 @@ -34,6 +34,18 @@ extern size_t max_column_width;
1195 /* The desired exit status. */
1196 extern int exit_status;
1199 +set_utf_locale (void);
1202 +check_utf_locale(void);
1205 +check_bom(FILE* fp, mb_file_t *mbf);
1210 /* Add tab stop TABVAL to the end of 'tab_list'. */
1212 add_tab_stop (uintmax_t tabval);
1213 diff -Naurp coreutils-8.27-orig/src/fold.c coreutils-8.27/src/fold.c
1214 --- coreutils-8.27-orig/src/fold.c 2017-01-01 16:34:24.000000000 -0600
1215 +++ coreutils-8.27/src/fold.c 2017-03-11 23:49:30.982169404 -0600
1218 #include <sys/types.h>
1220 +/* Get mbstate_t, mbrtowc(), wcwidth(). */
1222 +# include <wchar.h>
1225 +/* Get iswprint(), iswblank(), wcwidth(). */
1227 +# include <wctype.h>
1233 #include "fadvise.h"
1234 #include "xdectoint.h"
1236 +/* MB_LEN_MAX is incorrectly defined to be 1 in at least one GCC
1237 + installation; work around this configuration error. */
1238 +#if !defined MB_LEN_MAX || MB_LEN_MAX < 2
1240 +# define MB_LEN_MAX 16
1243 +/* Some systems, like BeOS, have multibyte encodings but lack mbstate_t. */
1244 +#if HAVE_MBRTOWC && defined mbstate_t
1245 +# define mbrtowc(pwc, s, n, ps) (mbrtowc) (pwc, s, n, 0)
1250 /* The official name of this program (e.g., no 'g' prefix). */
1253 #define AUTHORS proper_name ("David MacKenzie")
1255 +#define FATAL_ERROR(Message) \
1258 + error (0, 0, (Message)); \
1263 +enum operating_mode
1265 + /* Fold texts by columns that are at the given positions. */
1268 + /* Fold texts by bytes that are at the given positions. */
1271 + /* Fold texts by characters that are at the given positions. */
1275 +/* The argument shows current mode. (Default: column_mode) */
1276 +static enum operating_mode operating_mode;
1278 /* If nonzero, try to break on whitespace. */
1279 static bool break_spaces;
1281 -/* If nonzero, count bytes, not column positions. */
1282 -static bool count_bytes;
1284 /* If nonzero, at least one of the files we read was standard input. */
1285 static bool have_read_stdin;
1287 -static char const shortopts[] = "bsw:0::1::2::3::4::5::6::7::8::9::";
1288 +static char const shortopts[] = "bcsw:0::1::2::3::4::5::6::7::8::9::";
1290 static struct option const longopts[] =
1292 {"bytes", no_argument, NULL, 'b'},
1293 + {"characters", no_argument, NULL, 'c'},
1294 {"spaces", no_argument, NULL, 's'},
1295 {"width", required_argument, NULL, 'w'},
1296 {GETOPT_HELP_OPTION_DECL},
1297 @@ -76,6 +119,7 @@ Wrap input lines in each FILE, writing t
1300 -b, --bytes count bytes rather than columns\n\
1301 + -c, --characters count characters rather than columns\n\
1302 -s, --spaces break at spaces\n\
1303 -w, --width=WIDTH use WIDTH columns instead of 80\n\
1305 @@ -93,7 +137,7 @@ Wrap input lines in each FILE, writing t
1307 adjust_column (size_t column, char c)
1310 + if (operating_mode != byte_mode)
1314 @@ -116,30 +160,14 @@ adjust_column (size_t column, char c)
1315 to stdout, with maximum line length WIDTH.
1316 Return true if successful. */
1319 -fold_file (char const *filename, size_t width)
1321 +fold_text (FILE *istream, size_t width, int *saved_errno)
1325 size_t column = 0; /* Screen column where next char will go. */
1326 size_t offset_out = 0; /* Index in 'line_out' for next char. */
1327 static char *line_out = NULL;
1328 static size_t allocated_out = 0;
1331 - if (STREQ (filename, "-"))
1334 - have_read_stdin = true;
1337 - istream = fopen (filename, "r");
1339 - if (istream == NULL)
1341 - error (0, errno, "%s", quotef (filename));
1345 fadvise (istream, FADVISE_SEQUENTIAL);
1347 @@ -169,6 +197,15 @@ fold_file (char const *filename, size_t
1348 bool found_blank = false;
1349 size_t logical_end = offset_out;
1351 + /* If LINE_OUT has no wide character,
1352 + put a new wide character in LINE_OUT
1353 + if column is bigger than width. */
1354 + if (offset_out == 0)
1356 + line_out[offset_out++] = c;
1360 /* Look for the last blank. */
1363 @@ -215,11 +252,220 @@ fold_file (char const *filename, size_t
1364 line_out[offset_out++] = c;
1367 - saved_errno = errno;
1368 + *saved_errno = errno;
1371 + fwrite (line_out, sizeof (char), (size_t) offset_out, stdout);
1377 +fold_multibyte_text (FILE *istream, size_t width, int *saved_errno)
1379 + char buf[MB_LEN_MAX + BUFSIZ]; /* For spooling a read byte sequence. */
1380 + size_t buflen = 0; /* The length of the byte sequence in buf. */
1381 + char *bufpos = buf; /* Next read position of BUF. */
1382 + wint_t wc; /* A gotten wide character. */
1383 + size_t mblength; /* The byte size of a multibyte character which shows
1384 + as same character as WC. */
1385 + mbstate_t state, state_bak; /* State of the stream. */
1386 + int convfail = 0; /* 1, when conversion is failed. Otherwise 0. */
1388 + static char *line_out = NULL;
1389 + size_t offset_out = 0; /* Index in `line_out' for next char. */
1390 + static size_t allocated_out = 0;
1393 + size_t column = 0;
1395 + size_t last_blank_pos;
1396 + size_t last_blank_column;
1397 + int is_blank_seen;
1398 + int last_blank_increment = 0;
1399 + int is_bs_following_last_blank;
1400 + size_t bs_following_last_blank_num;
1401 + int is_cr_after_last_blank;
1403 +#define CLEAR_FLAGS \
1406 + last_blank_pos = 0; \
1407 + last_blank_column = 0; \
1408 + is_blank_seen = 0; \
1409 + is_bs_following_last_blank = 0; \
1410 + bs_following_last_blank_num = 0; \
1411 + is_cr_after_last_blank = 0; \
1415 +#define START_NEW_LINE \
1426 + memset (&state, '\0', sizeof(mbstate_t));
1428 + for (;; bufpos += mblength, buflen -= mblength)
1430 + if (buflen < MB_LEN_MAX && !feof (istream) && !ferror (istream))
1432 + memmove (buf, bufpos, buflen);
1433 + buflen += fread (buf + buflen, sizeof(char), BUFSIZ, istream);
1440 + /* Get a wide character. */
1441 + state_bak = state;
1442 + mblength = mbrtowc ((wchar_t *)&wc, bufpos, buflen, &state);
1449 + state = state_bak;
1450 + /* Fall through. */
1460 + else if (wc == L'\n')
1462 + /* preserve newline */
1463 + fwrite (line_out, sizeof(char), offset_out, stdout);
1467 + else if (operating_mode == byte_mode) /* byte mode */
1468 + increment = mblength;
1469 + else if (operating_mode == character_mode) /* character mode */
1471 + else /* column mode */
1476 + increment = (column > 0) ? -1 : 0;
1480 + increment = -1 * column;
1484 + increment = 8 - column % 8;
1488 + increment = wcwidth (wc);
1489 + increment = (increment < 0) ? 0 : increment;
1493 + if (column + increment > width && break_spaces && last_blank_pos)
1495 + fwrite (line_out, sizeof(char), last_blank_pos, stdout);
1498 + offset_out = offset_out - last_blank_pos;
1499 + column = column - last_blank_column + ((is_cr_after_last_blank)
1500 + ? last_blank_increment : bs_following_last_blank_num);
1501 + memmove (line_out, line_out + last_blank_pos, offset_out);
1506 + if (column + increment > width && column != 0)
1508 + fwrite (line_out, sizeof(char), offset_out, stdout);
1513 + if (allocated_out < offset_out + mblength)
1515 + line_out = X2REALLOC (line_out, &allocated_out);
1518 + memcpy (line_out + offset_out, bufpos, mblength);
1519 + offset_out += mblength;
1520 + column += increment;
1522 + if (is_blank_seen && !convfail && wc == L'\r')
1523 + is_cr_after_last_blank = 1;
1525 + if (is_bs_following_last_blank && !convfail && wc == L'\b')
1526 + ++bs_following_last_blank_num;
1528 + is_bs_following_last_blank = 0;
1530 + if (break_spaces && !convfail && iswblank (wc))
1532 + last_blank_pos = offset_out;
1533 + last_blank_column = column;
1534 + is_blank_seen = 1;
1535 + last_blank_increment = increment;
1536 + is_bs_following_last_blank = 1;
1537 + bs_following_last_blank_num = 0;
1538 + is_cr_after_last_blank = 0;
1542 + *saved_errno = errno;
1545 fwrite (line_out, sizeof (char), (size_t) offset_out, stdout);
1550 +/* Fold file FILENAME, or standard input if FILENAME is "-",
1551 + to stdout, with maximum line length WIDTH.
1552 + Return 0 if successful, 1 if an error occurs. */
1555 +fold_file (char const *filename, size_t width)
1560 + if (STREQ (filename, "-"))
1563 + have_read_stdin = 1;
1566 + istream = fopen (filename, "r");
1568 + if (istream == NULL)
1570 + error (0, errno, "%s", filename);
1574 + /* Define how ISTREAM is being folded. */
1576 + if (MB_CUR_MAX > 1)
1577 + fold_multibyte_text (istream, width, &saved_errno);
1580 + fold_text (istream, width, &saved_errno);
1582 if (ferror (istream))
1584 error (0, saved_errno, "%s", quotef (filename));
1585 @@ -252,7 +498,8 @@ main (int argc, char **argv)
1587 atexit (close_stdout);
1589 - break_spaces = count_bytes = have_read_stdin = false;
1590 + operating_mode = column_mode;
1591 + break_spaces = have_read_stdin = false;
1593 while ((optc = getopt_long (argc, argv, shortopts, longopts, NULL)) != -1)
1595 @@ -261,7 +508,15 @@ main (int argc, char **argv)
1598 case 'b': /* Count bytes rather than columns. */
1599 - count_bytes = true;
1600 + if (operating_mode != column_mode)
1601 + FATAL_ERROR (_("only one way of folding may be specified"));
1602 + operating_mode = byte_mode;
1606 + if (operating_mode != column_mode)
1607 + FATAL_ERROR (_("only one way of folding may be specified"));
1608 + operating_mode = character_mode;
1611 case 's': /* Break at word boundaries. */
1612 diff -Naurp coreutils-8.27-orig/src/join.c coreutils-8.27/src/join.c
1613 --- coreutils-8.27-orig/src/join.c 2017-01-01 16:34:24.000000000 -0600
1614 +++ coreutils-8.27/src/join.c 2017-03-11 23:47:13.091286290 -0600
1616 #include <sys/types.h>
1619 +/* Get mbstate_t, mbrtowc(), mbrtowc(), wcwidth(). */
1621 +# include <wchar.h>
1624 +/* Get iswblank(), towupper. */
1626 +# include <wctype.h>
1632 #include "fadvise.h"
1633 #include "hard-locale.h"
1634 #include "linebuffer.h"
1635 -#include "memcasecmp.h"
1637 #include "stdio--.h"
1638 #include "xmemcoll.h"
1639 #include "xstrtol.h"
1640 #include "argmatch.h"
1642 +/* Some systems, like BeOS, have multibyte encodings but lack mbstate_t. */
1643 +#if HAVE_MBRTOWC && defined mbstate_t
1644 +# define mbrtowc(pwc, s, n, ps) (mbrtowc) (pwc, s, n, 0)
1647 /* The official name of this program (e.g., no 'g' prefix). */
1648 #define PROGRAM_NAME "join"
1650 @@ -136,10 +150,12 @@ static struct outlist outlist_head;
1651 /* Last element in 'outlist', where a new element can be added. */
1652 static struct outlist *outlist_end = &outlist_head;
1654 -/* Tab character separating fields. If negative, fields are separated
1655 - by any nonempty string of blanks, otherwise by exactly one
1656 - tab character whose value (when cast to unsigned char) equals TAB. */
1657 -static int tab = -1;
1658 +/* Tab character separating fields. If NULL, fields are separated
1659 + by any nonempty string of blanks. */
1660 +static char *tab = NULL;
1662 +/* The number of bytes used for tab. */
1663 +static size_t tablen = 0;
1665 /* If nonzero, check that the input is correctly ordered. */
1667 @@ -276,13 +292,14 @@ xfields (struct line *line)
1671 - if (0 <= tab && tab != '\n')
1674 + unsigned char t = tab[0];
1676 - for (; (sep = memchr (ptr, tab, lim - ptr)) != NULL; ptr = sep + 1)
1677 + for (; (sep = memchr (ptr, t, lim - ptr)) != NULL; ptr = sep + 1)
1678 extract_field (line, ptr, sep - ptr);
1683 /* Skip leading blanks before the first field. */
1684 while (field_sep (*ptr))
1685 @@ -306,6 +323,147 @@ xfields (struct line *line)
1686 extract_field (line, ptr, lim - ptr);
1691 +xfields_multibyte (struct line *line)
1693 + char *ptr = line->buf.buffer;
1694 + char const *lim = ptr + line->buf.length - 1;
1696 + size_t mblength = 1;
1697 + mbstate_t state, state_bak;
1699 + memset (&state, 0, sizeof (mbstate_t));
1707 + for (; ptr < lim; ptr = sep + mblength)
1712 + state_bak = state;
1713 + mblength = mbrtowc (&wc, sep, lim - sep + 1, &state);
1715 + if (mblength == (size_t)-1 || mblength == (size_t)-2)
1718 + state = state_bak;
1720 + mblength = (mblength < 1) ? 1 : mblength;
1722 + if (mblength == tablen && !memcmp (sep, tab, mblength))
1734 + extract_field (line, ptr, sep - ptr);
1739 + /* Skip leading blanks before the first field. */
1742 + state_bak = state;
1743 + mblength = mbrtowc (&wc, ptr, lim - ptr + 1, &state);
1745 + if (mblength == (size_t)-1 || mblength == (size_t)-2)
1748 + state = state_bak;
1751 + mblength = (mblength < 1) ? 1 : mblength;
1753 + if (!iswblank(wc) && wc != '\n')
1761 + state_bak = state;
1762 + mblength = mbrtowc (&wc, ptr, lim - ptr + 1, &state);
1763 + if (mblength == (size_t)-1 || mblength == (size_t)-2)
1766 + state = state_bak;
1769 + mblength = (mblength < 1) ? 1 : mblength;
1771 + sep = ptr + mblength;
1774 + state_bak = state;
1775 + mblength = mbrtowc (&wc, sep, lim - sep + 1, &state);
1776 + if (mblength == (size_t)-1 || mblength == (size_t)-2)
1779 + state = state_bak;
1782 + mblength = (mblength < 1) ? 1 : mblength;
1784 + if (iswblank (wc) || wc == '\n')
1790 + extract_field (line, ptr, sep - ptr);
1794 + state_bak = state;
1795 + mblength = mbrtowc (&wc, sep, lim - sep + 1, &state);
1796 + if (mblength == (size_t)-1 || mblength == (size_t)-2)
1799 + state = state_bak;
1802 + mblength = (mblength < 1) ? 1 : mblength;
1804 + ptr = sep + mblength;
1807 + state_bak = state;
1808 + mblength = mbrtowc (&wc, ptr, lim - ptr + 1, &state);
1809 + if (mblength == (size_t)-1 || mblength == (size_t)-2)
1812 + state = state_bak;
1815 + mblength = (mblength < 1) ? 1 : mblength;
1817 + if (!iswblank (wc) && wc != '\n')
1823 + while (ptr < lim);
1826 + extract_field (line, ptr, lim - ptr);
1831 freeline (struct line *line)
1833 @@ -327,56 +485,133 @@ keycmp (struct line const *line1, struct
1834 size_t jf_1, size_t jf_2)
1836 /* Start of field to compare in each file. */
1841 - size_t len2; /* Length of fields to compare. */
1844 + size_t len[2]; /* Length of fields to compare. */
1849 if (jf_1 < line1->nfields)
1851 - beg1 = line1->fields[jf_1].beg;
1852 - len1 = line1->fields[jf_1].len;
1853 + beg[0] = line1->fields[jf_1].beg;
1854 + len[0] = line1->fields[jf_1].len;
1864 if (jf_2 < line2->nfields)
1866 - beg2 = line2->fields[jf_2].beg;
1867 - len2 = line2->fields[jf_2].len;
1868 + beg[1] = line2->fields[jf_2].beg;
1869 + len[1] = line2->fields[jf_2].len;
1880 - return len2 == 0 ? 0 : -1;
1883 + return len[1] == 0 ? 0 : -1;
1889 - /* FIXME: ignore_case does not work with NLS (in particular,
1890 - with multibyte chars). */
1891 - diff = memcasecmp (beg1, beg2, MIN (len1, len2));
1892 +#ifdef HAVE_MBRTOWC
1893 + if (MB_CUR_MAX > 1)
1897 + mbstate_t state, state_bak;
1899 + memset (&state, '\0', sizeof (mbstate_t));
1901 + for (i = 0; i < 2; i++)
1904 + copy[i] = xmalloc (len[i] + 1);
1905 + memset (copy[i], '\0',len[i] + 1);
1907 + for (j = 0; j < MIN (len[0], len[1]);)
1909 + state_bak = state;
1910 + mblength = mbrtowc (&wc, beg[i] + j, len[i] - j, &state);
1916 + state = state_bak;
1917 + /* Fall through */
1923 + uwc = towupper (wc);
1927 + mbstate_t state_wc;
1930 + memset (&state_wc, '\0', sizeof (mbstate_t));
1931 + mblen = wcrtomb (copy[i] + j, uwc, &state_wc);
1932 + assert (mblen != (size_t)-1);
1935 + memcpy (copy[i] + j, beg[i] + j, mblength);
1939 + copy[i][j] = '\0';
1945 + for (i = 0; i < 2; i++)
1948 + copy[i] = xmalloc (len[i] + 1);
1950 + for (j = 0; j < MIN (len[0], len[1]); j++)
1951 + copy[i][j] = toupper (beg[i][j]);
1953 + copy[i][j] = '\0';
1959 - if (hard_LC_COLLATE)
1960 - return xmemcoll (beg1, len1, beg2, len2);
1961 - diff = memcmp (beg1, beg2, MIN (len1, len2));
1966 + if (hard_LC_COLLATE)
1968 + diff = xmemcoll ((char *) copy[0], len[0], (char *) copy[1], len[1]);
1971 + for (i = 0; i < 2; i++)
1976 + diff = memcmp (copy[0], copy[1], MIN (len[0], len[1]));
1979 + for (i = 0; i < 2; i++)
1985 - return len1 < len2 ? -1 : len1 != len2;
1986 + return len[0] - len[1];
1989 /* Check that successive input lines PREV and CURRENT from input file
1990 @@ -468,6 +703,11 @@ get_line (FILE *fp, struct line **linep,
1992 ++line_no[which - 1];
1995 + if (MB_CUR_MAX > 1)
1996 + xfields_multibyte (line);
2001 if (prevline[which - 1])
2002 @@ -567,21 +807,28 @@ prfield (size_t n, struct line const *li
2004 /* Output all the fields in line, other than the join field. */
2006 +#define PUT_TAB_CHAR \
2010 + fwrite(tab, sizeof(char), tablen, stdout) : putchar (' '); \
2015 prfields (struct line const *line, size_t join_field, size_t autocount)
2018 size_t nfields = autoformat ? autocount : line->nfields;
2019 - char output_separator = tab < 0 ? ' ' : tab;
2021 for (i = 0; i < join_field && i < nfields; ++i)
2023 - putchar (output_separator);
2027 for (i = join_field + 1; i < nfields; ++i)
2029 - putchar (output_separator);
2034 @@ -592,7 +839,6 @@ static void
2035 prjoin (struct line const *line1, struct line const *line2)
2037 const struct outlist *outlist;
2038 - char output_separator = tab < 0 ? ' ' : tab;
2040 struct line const *line;
2042 @@ -626,7 +872,7 @@ prjoin (struct line const *line1, struct
2046 - putchar (output_separator);
2051 @@ -1104,20 +1350,43 @@ main (int argc, char **argv)
2055 - unsigned char newtab = optarg[0];
2056 + char *newtab = NULL;
2058 + newtab = xstrdup (optarg);
2060 + if (MB_CUR_MAX > 1)
2064 + memset (&state, 0, sizeof (mbstate_t));
2065 + newtablen = mbrtowc (NULL, newtab,
2066 + strnlen (newtab, MB_LEN_MAX),
2068 + if (newtablen == (size_t) 0
2069 + || newtablen == (size_t) -1
2070 + || newtablen == (size_t) -2)
2077 - newtab = '\n'; /* '' => process the whole line. */
2078 + newtab = (char*)"\n"; /* '' => process the whole line. */
2081 - if (STREQ (optarg, "\\0"))
2084 - die (EXIT_FAILURE, 0, _("multi-character tab %s"),
2086 + if (newtablen == 1 && newtab[1])
2088 + if (STREQ (newtab, "\\0"))
2092 + if (tab != NULL && strcmp (tab, newtab))
2095 + die (EXIT_FAILURE, 0, _("incompatible tabs"));
2097 - if (0 <= tab && tab != newtab)
2098 - die (EXIT_FAILURE, 0, _("incompatible tabs"));
2100 + tablen = newtablen;
2104 diff -Naurp coreutils-8.27-orig/src/pr.c coreutils-8.27/src/pr.c
2105 --- coreutils-8.27-orig/src/pr.c 2017-01-01 16:34:24.000000000 -0600
2106 +++ coreutils-8.27/src/pr.c 2017-03-11 23:47:13.094286139 -0600
2107 @@ -311,6 +311,24 @@
2110 #include <sys/types.h>
2112 +/* Get MB_LEN_MAX. */
2113 +#include <limits.h>
2114 +/* MB_LEN_MAX is incorrectly defined to be 1 in at least one GCC
2115 + installation; work around this configuration error. */
2116 +#if !defined MB_LEN_MAX || MB_LEN_MAX == 1
2117 +# define MB_LEN_MAX 16
2120 +/* Get MB_CUR_MAX. */
2121 +#include <stdlib.h>
2123 +/* Solaris 2.5 has a bug: <wchar.h> must be included before <wctype.h>. */
2124 +/* Get mbstate_t, mbrtowc(), wcwidth(). */
2126 +# include <wchar.h>
2132 @@ -324,6 +342,18 @@
2133 #include "xstrtol.h"
2134 #include "xdectoint.h"
2136 +/* Some systems, like BeOS, have multibyte encodings but lack mbstate_t. */
2137 +#if HAVE_MBRTOWC && defined mbstate_t
2138 +# define mbrtowc(pwc, s, n, ps) (mbrtowc) (pwc, s, n, 0)
2141 +#ifndef HAVE_DECL_WCWIDTH
2142 +"this configure-time declaration test was not run"
2144 +#if !HAVE_DECL_WCWIDTH
2145 +extern int wcwidth ();
2148 /* The official name of this program (e.g., no 'g' prefix). */
2149 #define PROGRAM_NAME "pr"
2151 @@ -416,7 +446,20 @@ struct COLUMN
2153 typedef struct COLUMN COLUMN;
2155 -static int char_to_clump (char c);
2156 +/* Funtion pointers to switch functions for single byte locale or for
2157 + multibyte locale. If multibyte functions do not exist in your sysytem,
2158 + these pointers always point the function for single byte locale. */
2159 +static void (*print_char) (char c);
2160 +static int (*char_to_clump) (char c);
2162 +/* Functions for single byte locale. */
2163 +static void print_char_single (char c);
2164 +static int char_to_clump_single (char c);
2166 +/* Functions for multibyte locale. */
2167 +static void print_char_multi (char c);
2168 +static int char_to_clump_multi (char c);
2170 static bool read_line (COLUMN *p);
2171 static bool print_page (void);
2172 static bool print_stored (COLUMN *p);
2173 @@ -428,6 +471,7 @@ static void add_line_number (COLUMN *p);
2174 static void getoptnum (const char *n_str, int min, int *num,
2175 const char *errfmt);
2176 static void getoptarg (char *arg, char switch_char, char *character,
2177 + int *character_length, int *character_width,
2179 static void print_files (int number_of_files, char **av);
2180 static void init_parameters (int number_of_files);
2181 @@ -441,7 +485,6 @@ static void store_char (char c);
2182 static void pad_down (unsigned int lines);
2183 static void read_rest_of_line (COLUMN *p);
2184 static void skip_read (COLUMN *p, int column_number);
2185 -static void print_char (char c);
2186 static void cleanup (void);
2187 static void print_sep_string (void);
2188 static void separator_string (const char *optarg_S);
2189 @@ -453,7 +496,7 @@ static COLUMN *column_vector;
2190 we store the leftmost columns contiguously in buff.
2191 To print a line from buff, get the index of the first character
2192 from line_vector[i], and print up to line_vector[i + 1]. */
2194 +static unsigned char *buff;
2196 /* Index of the position in buff where the next character
2198 @@ -557,7 +600,7 @@ static int chars_per_column;
2199 static bool untabify_input = false;
2201 /* (-e) The input tab character. */
2202 -static char input_tab_char = '\t';
2203 +static char input_tab_char[MB_LEN_MAX] = "\t";
2205 /* (-e) Tabstops are at chars_per_tab, 2*chars_per_tab, 3*chars_per_tab, ...
2206 where the leftmost column is 1. */
2207 @@ -567,7 +610,10 @@ static int chars_per_input_tab = 8;
2208 static bool tabify_output = false;
2210 /* (-i) The output tab character. */
2211 -static char output_tab_char = '\t';
2212 +static char output_tab_char[MB_LEN_MAX] = "\t";
2214 +/* (-i) The byte length of output tab character. */
2215 +static int output_tab_char_length = 1;
2217 /* (-i) The width of the output tab. */
2218 static int chars_per_output_tab = 8;
2219 @@ -637,7 +683,13 @@ static int line_number;
2220 static bool numbered_lines = false;
2222 /* (-n) Character which follows each line number. */
2223 -static char number_separator = '\t';
2224 +static char number_separator[MB_LEN_MAX] = "\t";
2226 +/* (-n) The byte length of the character which follows each line number. */
2227 +static int number_separator_length = 1;
2229 +/* (-n) The character width of the character which follows each line number. */
2230 +static int number_separator_width = 0;
2232 /* (-n) line counting starts with 1st line of input file (not with 1st
2233 line of 1st page printed). */
2234 @@ -690,6 +742,7 @@ static bool use_col_separator = false;
2235 -a|COLUMN|-m is a 'space' and with the -J option a 'tab'. */
2236 static char const *col_sep_string = "";
2237 static int col_sep_length = 0;
2238 +static int col_sep_width = 0;
2239 static char *column_separator = (char *) " ";
2240 static char *line_separator = (char *) "\t";
2242 @@ -851,6 +904,13 @@ separator_string (const char *optarg_S)
2243 integer_overflow ();
2244 col_sep_length = len;
2245 col_sep_string = optarg_S;
2248 + if (MB_CUR_MAX > 1)
2249 + col_sep_width = mbswidth (col_sep_string, 0);
2252 + col_sep_width = col_sep_length;
2256 @@ -875,6 +935,21 @@ main (int argc, char **argv)
2258 atexit (close_stdout);
2260 +/* Define which functions are used, the ones for single byte locale or the ones
2261 + for multibyte locale. */
2263 + if (MB_CUR_MAX > 1)
2265 + print_char = print_char_multi;
2266 + char_to_clump = char_to_clump_multi;
2271 + print_char = print_char_single;
2272 + char_to_clump = char_to_clump_single;
2276 file_names = (argc > 1
2277 ? xnmalloc (argc - 1, sizeof (char *))
2278 @@ -951,8 +1026,12 @@ main (int argc, char **argv)
2282 - getoptarg (optarg, 'e', &input_tab_char,
2283 - &chars_per_input_tab);
2285 + int dummy_length, dummy_width;
2287 + getoptarg (optarg, 'e', input_tab_char, &dummy_length,
2288 + &dummy_width, &chars_per_input_tab);
2290 /* Could check tab width > 0. */
2291 untabify_input = true;
2293 @@ -965,8 +1044,12 @@ main (int argc, char **argv)
2297 - getoptarg (optarg, 'i', &output_tab_char,
2298 - &chars_per_output_tab);
2302 + getoptarg (optarg, 'i', output_tab_char, &output_tab_char_length,
2303 + &dummy_width, &chars_per_output_tab);
2305 /* Could check tab width > 0. */
2306 tabify_output = true;
2308 @@ -984,8 +1067,8 @@ main (int argc, char **argv)
2310 numbered_lines = true;
2312 - getoptarg (optarg, 'n', &number_separator,
2313 - &chars_per_number);
2314 + getoptarg (optarg, 'n', number_separator, &number_separator_length,
2315 + &number_separator_width, &chars_per_number);
2319 @@ -1010,6 +1093,7 @@ main (int argc, char **argv)
2320 /* Reset an additional input of -s, -S dominates -s */
2321 col_sep_string = "";
2323 + col_sep_width = 0;
2324 use_col_separator = true;
2326 separator_string (optarg);
2327 @@ -1166,10 +1250,45 @@ getoptnum (const char *n_str, int min, i
2331 -getoptarg (char *arg, char switch_char, char *character, int *number)
2332 +getoptarg (char *arg, char switch_char, char *character, int *character_length,
2333 + int *character_width, int *number)
2335 if (!ISDIGIT (*arg))
2336 - *character = *arg++;
2338 +#ifdef HAVE_MBRTOWC
2339 + if (MB_CUR_MAX > 1) /* for multibyte locale. */
2344 + mbstate_t state = {'\0'};
2346 + mblength = mbrtowc (&wc, arg, strnlen(arg, MB_LEN_MAX), &state);
2348 + if (mblength == (size_t)-1 || mblength == (size_t)-2)
2350 + *character_length = 1;
2351 + *character_width = 1;
2355 + *character_length = (mblength < 1) ? 1 : mblength;
2356 + width = wcwidth (wc);
2357 + *character_width = (width < 0) ? 0 : width;
2360 + strncpy (character, arg, *character_length);
2361 + arg += *character_length;
2363 + else /* for single byte locale. */
2366 + *character = *arg++;
2367 + *character_length = 1;
2368 + *character_width = 1;
2375 @@ -1191,6 +1310,11 @@ static void
2376 init_parameters (int number_of_files)
2378 int chars_used_by_number = 0;
2381 + if (MB_CUR_MAX > 1)
2382 + mb_len = MB_LEN_MAX;
2385 lines_per_body = lines_per_page - lines_per_header - lines_per_footer;
2386 if (lines_per_body <= 0)
2387 @@ -1228,7 +1352,7 @@ init_parameters (int number_of_files)
2389 col_sep_string = column_separator;
2391 - col_sep_length = 1;
2392 + col_sep_length = col_sep_width = 1;
2393 use_col_separator = true;
2395 /* It's rather pointless to define a TAB separator with column
2396 @@ -1258,11 +1382,11 @@ init_parameters (int number_of_files)
2397 + TAB_WIDTH (chars_per_input_tab, chars_per_number); */
2399 /* Estimate chars_per_text without any margin and keep it constant. */
2400 - if (number_separator == '\t')
2401 + if (number_separator[0] == '\t')
2402 number_width = (chars_per_number
2403 + TAB_WIDTH (chars_per_default_tab, chars_per_number));
2405 - number_width = chars_per_number + 1;
2406 + number_width = chars_per_number + number_separator_width;
2408 /* The number is part of the column width unless we are
2409 printing files in parallel. */
2410 @@ -1271,7 +1395,7 @@ init_parameters (int number_of_files)
2413 int sep_chars, useful_chars;
2414 - if (INT_MULTIPLY_WRAPV (columns - 1, col_sep_length, &sep_chars))
2415 + if (INT_MULTIPLY_WRAPV (columns - 1, col_sep_width, &sep_chars))
2416 sep_chars = INT_MAX;
2417 if (INT_SUBTRACT_WRAPV (chars_per_line - chars_used_by_number, sep_chars,
2419 @@ -1294,7 +1418,7 @@ init_parameters (int number_of_files)
2420 We've to use 8 as the lower limit, if we use chars_per_default_tab = 8
2421 to expand a tab which is not an input_tab-char. */
2423 - clump_buff = xmalloc (MAX (8, chars_per_input_tab));
2424 + clump_buff = xmalloc (mb_len * MAX (8, chars_per_input_tab));
2427 /* Open the necessary files,
2428 @@ -1402,7 +1526,7 @@ init_funcs (void)
2430 /* Enlarge p->start_position of first column to use the same form of
2431 padding_not_printed with all columns. */
2432 - h = h + col_sep_length;
2433 + h = h + col_sep_width;
2435 /* This loop takes care of all but the rightmost column. */
2437 @@ -1436,7 +1560,7 @@ init_funcs (void)
2441 - h = h_next + col_sep_length;
2442 + h = h_next + col_sep_width;
2443 h_next = h + chars_per_column;
2446 @@ -1727,9 +1851,9 @@ static void
2447 align_column (COLUMN *p)
2449 padding_not_printed = p->start_position;
2450 - if (col_sep_length < padding_not_printed)
2451 + if (col_sep_width < padding_not_printed)
2453 - pad_across_to (padding_not_printed - col_sep_length);
2454 + pad_across_to (padding_not_printed - col_sep_width);
2455 padding_not_printed = ANYWHERE;
2458 @@ -2004,13 +2128,13 @@ store_char (char c)
2459 /* May be too generous. */
2460 buff = X2REALLOC (buff, &buff_allocated);
2462 - buff[buff_current++] = c;
2463 + buff[buff_current++] = (unsigned char) c;
2467 add_line_number (COLUMN *p)
2474 @@ -2027,22 +2151,24 @@ add_line_number (COLUMN *p)
2475 /* Tabification is assumed for multiple columns, also for n-separators,
2476 but 'default n-separator = TAB' hasn't been given priority over
2477 equal column_width also specified by POSIX. */
2478 - if (number_separator == '\t')
2479 + if (number_separator[0] == '\t')
2481 i = number_width - chars_per_number;
2483 (p->char_func) (' ');
2486 - (p->char_func) (number_separator);
2487 + for (j = 0; j < number_separator_length; j++)
2488 + (p->char_func) (number_separator[j]);
2491 /* To comply with POSIX, we avoid any expansion of default TAB
2492 separator with a single column output. No column_width requirement
2493 has to be considered. */
2495 - (p->char_func) (number_separator);
2496 - if (number_separator == '\t')
2497 + for (j = 0; j < number_separator_length; j++)
2498 + (p->char_func) (number_separator[j]);
2499 + if (number_separator[0] == '\t')
2500 output_position = POS_AFTER_TAB (chars_per_output_tab,
2503 @@ -2203,7 +2329,7 @@ print_white_space (void)
2504 while (goal - h_old > 1
2505 && (h_new = POS_AFTER_TAB (chars_per_output_tab, h_old)) <= goal)
2507 - putchar (output_tab_char);
2508 + fwrite (output_tab_char, sizeof(char), output_tab_char_length, stdout);
2511 while (++h_old <= goal)
2512 @@ -2223,6 +2349,7 @@ print_sep_string (void)
2514 char const *s = col_sep_string;
2515 int l = col_sep_length;
2516 + int not_space_flag;
2518 if (separators_not_printed <= 0)
2520 @@ -2234,6 +2361,7 @@ print_sep_string (void)
2522 for (; separators_not_printed > 0; --separators_not_printed)
2524 + not_space_flag = 0;
2527 /* 3 types of sep_strings: spaces only, spaces and chars,
2528 @@ -2247,12 +2375,15 @@ print_sep_string (void)
2532 + not_space_flag = 1;
2533 if (spaces_not_printed > 0)
2534 print_white_space ();
2536 - ++output_position;
2539 + if (not_space_flag)
2540 + output_position += col_sep_width;
2542 /* sep_string ends with some spaces */
2543 if (spaces_not_printed > 0)
2544 print_white_space ();
2545 @@ -2280,7 +2411,7 @@ print_clump (COLUMN *p, int n, char *clu
2546 required number of tabs and spaces. */
2549 -print_char (char c)
2550 +print_char_single (char c)
2554 @@ -2304,6 +2435,74 @@ print_char (char c)
2558 +#ifdef HAVE_MBRTOWC
2560 +print_char_multi (char c)
2562 + static size_t mbc_pos = 0;
2563 + static char mbc[MB_LEN_MAX] = {'\0'};
2564 + static mbstate_t state = {'\0'};
2565 + mbstate_t state_bak;
2570 + if (tabify_output)
2572 + state_bak = state;
2573 + mbc[mbc_pos++] = c;
2574 + mblength = mbrtowc (&wc, mbc, mbc_pos, &state);
2576 + while (mbc_pos > 0)
2581 + state = state_bak;
2585 + state = state_bak;
2586 + ++output_position;
2588 + memmove (mbc, mbc + 1, MB_CUR_MAX - 1);
2598 + memmove (mbc, mbc + mblength, MB_CUR_MAX - mblength);
2600 + ++spaces_not_printed;
2603 + else if (spaces_not_printed > 0)
2604 + print_white_space ();
2606 + /* Nonprintables are assumed to have width 0, except L'\b'. */
2607 + if ((width = wcwidth (wc)) < 1)
2610 + --output_position;
2613 + output_position += width;
2615 + fwrite (mbc, sizeof(char), mblength, stdout);
2616 + memmove (mbc, mbc + mblength, MB_CUR_MAX - mblength);
2617 + mbc_pos -= mblength;
2626 /* Skip to page PAGE before printing.
2627 PAGE may be larger than total number of pages. */
2629 @@ -2483,9 +2682,9 @@ read_line (COLUMN *p)
2630 align_empty_cols = false;
2633 - if (col_sep_length < padding_not_printed)
2634 + if (col_sep_width < padding_not_printed)
2636 - pad_across_to (padding_not_printed - col_sep_length);
2637 + pad_across_to (padding_not_printed - col_sep_width);
2638 padding_not_printed = ANYWHERE;
2641 @@ -2555,7 +2754,7 @@ print_stored (COLUMN *p)
2644 int line = p->current_line++;
2645 - char *first = &buff[line_vector[line]];
2646 + unsigned char *first = &buff[line_vector[line]];
2648 UMR: Uninitialized memory read:
2649 * This is occurring while in:
2650 @@ -2567,7 +2766,7 @@ print_stored (COLUMN *p)
2651 xmalloc [xmalloc.c:94]
2652 init_store_cols [pr.c:1648]
2654 - char *last = &buff[line_vector[line + 1]];
2655 + unsigned char *last = &buff[line_vector[line + 1]];
2657 pad_vertically = true;
2659 @@ -2586,9 +2785,9 @@ print_stored (COLUMN *p)
2663 - if (col_sep_length < padding_not_printed)
2664 + if (col_sep_width < padding_not_printed)
2666 - pad_across_to (padding_not_printed - col_sep_length);
2667 + pad_across_to (padding_not_printed - col_sep_width);
2668 padding_not_printed = ANYWHERE;
2671 @@ -2601,8 +2800,8 @@ print_stored (COLUMN *p)
2672 if (spaces_not_printed == 0)
2674 output_position = p->start_position + end_vector[line];
2675 - if (p->start_position - col_sep_length == chars_per_margin)
2676 - output_position -= col_sep_length;
2677 + if (p->start_position - col_sep_width == chars_per_margin)
2678 + output_position -= col_sep_width;
2682 @@ -2621,7 +2820,7 @@ print_stored (COLUMN *p)
2683 number of characters is 1.) */
2686 -char_to_clump (char c)
2687 +char_to_clump_single (char c)
2689 unsigned char uc = c;
2690 char *s = clump_buff;
2691 @@ -2631,10 +2830,10 @@ char_to_clump (char c)
2693 int chars_per_c = 8;
2695 - if (c == input_tab_char)
2696 + if (c == input_tab_char[0])
2697 chars_per_c = chars_per_input_tab;
2699 - if (c == input_tab_char || c == '\t')
2700 + if (c == input_tab_char[0] || c == '\t')
2702 width = TAB_WIDTH (chars_per_c, input_position);
2704 @@ -2715,6 +2914,164 @@ char_to_clump (char c)
2708 +#ifdef HAVE_MBRTOWC
2710 +char_to_clump_multi (char c)
2712 + static size_t mbc_pos = 0;
2713 + static char mbc[MB_LEN_MAX] = {'\0'};
2714 + static mbstate_t state = {'\0'};
2715 + mbstate_t state_bak;
2719 + register char *s = clump_buff;
2720 + register int i, j;
2724 + int chars_per_c = 8;
2726 + state_bak = state;
2727 + mbc[mbc_pos++] = c;
2728 + mblength = mbrtowc (&wc, mbc, mbc_pos, &state);
2732 + while (mbc_pos > 0)
2737 + state = state_bak;
2741 + state = state_bak;
2744 + if (use_esc_sequence || use_cntrl_prefix)
2749 + sprintf (esc_buff, "%03o", (unsigned char) mbc[0]);
2750 + for (i = 0; i <= 2; ++i)
2751 + *s++ = (int) esc_buff[i];
2763 + /* Fall through */
2766 + if (memcmp (mbc, input_tab_char, mblength) == 0)
2767 + chars_per_c = chars_per_input_tab;
2769 + if (memcmp (mbc, input_tab_char, mblength) == 0 || c == '\t')
2773 + width_inc = TAB_WIDTH (chars_per_c, input_position);
2774 + width += width_inc;
2776 + if (untabify_input)
2778 + for (i = width_inc; i; --i)
2780 + chars += width_inc;
2784 + for (i = 0; i < mblength; i++)
2786 + chars += mblength;
2789 + else if ((wc_width = wcwidth (wc)) < 1)
2791 + if (use_esc_sequence)
2793 + for (i = 0; i < mblength; i++)
2798 + sprintf (esc_buff, "%03o", (unsigned char) mbc[i]);
2799 + for (j = 0; j <= 2; ++j)
2800 + *s++ = (int) esc_buff[j];
2803 + else if (use_cntrl_prefix)
2814 + for (i = 0; i < mblength; i++)
2819 + sprintf (esc_buff, "%03o", (unsigned char) mbc[i]);
2820 + for (j = 0; j <= 2; ++j)
2821 + *s++ = (int) esc_buff[j];
2825 + else if (wc == L'\b')
2834 + chars += mblength;
2835 + for (i = 0; i < mblength; i++)
2841 + width += wc_width;
2842 + chars += mblength;
2843 + for (i = 0; i < mblength; i++)
2847 + memmove (mbc, mbc + mblength, MB_CUR_MAX - mblength);
2848 + mbc_pos -= mblength;
2851 + /* Too many backspaces must put us in position 0 -- never negative. */
2852 + if (width < 0 && input_position == 0)
2855 + input_position = 0;
2857 + else if (width < 0 && input_position <= -width)
2858 + input_position = 0;
2860 + input_position += width;
2866 /* We've just printed some files and need to clean up things before
2867 looking for more options and printing the next batch of files.
2869 diff -Naurp coreutils-8.27-orig/src/sort.c coreutils-8.27/src/sort.c
2870 --- coreutils-8.27-orig/src/sort.c 2017-01-01 16:34:24.000000000 -0600
2871 +++ coreutils-8.27/src/sort.c 2017-03-11 23:49:22.416505389 -0600
2873 #include <sys/wait.h>
2877 +# include <wchar.h>
2879 +/* Get isw* functions. */
2881 +# include <wctype.h>
2885 #include "argmatch.h"
2887 @@ -165,14 +173,39 @@ static int decimal_point;
2888 /* Thousands separator; if -1, then there isn't one. */
2889 static int thousands_sep;
2891 +/* True if -f is specified. */
2892 +static bool folding;
2894 /* Nonzero if the corresponding locales are hard. */
2895 static bool hard_LC_COLLATE;
2896 -#if HAVE_NL_LANGINFO
2897 +#if HAVE_LANGINFO_CODESET
2898 static bool hard_LC_TIME;
2901 #define NONZERO(x) ((x) != 0)
2903 +/* get a multibyte character's byte length. */
2904 +#define GET_BYTELEN_OF_CHAR(LIM, PTR, MBLENGTH, STATE) \
2908 + mbstate_t state_bak; \
2910 + state_bak = STATE; \
2911 + mblength = mbrtowc (&wc, PTR, LIM - PTR, &STATE); \
2913 + switch (MBLENGTH) \
2915 + case (size_t)-1: \
2916 + case (size_t)-2: \
2917 + STATE = state_bak; \
2918 + /* Fall through. */ \
2925 /* The kind of blanks for '-b' to skip in various options. */
2926 enum blanktype { bl_start, bl_end, bl_both };
2928 @@ -346,13 +379,11 @@ static bool reverse;
2929 they were read if all keys compare equal. */
2932 -/* If TAB has this value, blanks separate fields. */
2933 -enum { TAB_DEFAULT = CHAR_MAX + 1 };
2935 -/* Tab character separating fields. If TAB_DEFAULT, then fields are
2936 +/* Tab character separating fields. If tab_length is 0, then fields are
2937 separated by the empty string between a non-blank character and a blank
2939 -static int tab = TAB_DEFAULT;
2940 +static char tab[MB_LEN_MAX + 1];
2941 +static size_t tab_length = 0;
2943 /* Flag to remove consecutive duplicate lines from the output.
2944 Only the last of a sequence of equal lines will be output. */
2945 @@ -811,6 +842,46 @@ reap_all (void)
2949 +/* Function pointers. */
2951 +(*inittables) (void);
2953 +(*begfield) (const struct line*, const struct keyfield *);
2955 +(*limfield) (const struct line*, const struct keyfield *);
2957 +(*skipblanks) (char **ptr, char *lim);
2959 +(*getmonth) (char const *, size_t, char **);
2961 +(*keycompare) (const struct line *, const struct line *);
2963 +(*numcompare) (const char *, const char *);
2965 +/* Test for white space multibyte character.
2966 + Set LENGTH the byte length of investigated multibyte character. */
2969 +ismbblank (const char *str, size_t len, size_t *length)
2975 + memset (&state, '\0', sizeof(mbstate_t));
2976 + mblength = mbrtowc (&wc, str, len, &state);
2978 + if (mblength == (size_t)-1 || mblength == (size_t)-2)
2984 + *length = (mblength < 1) ? 1 : mblength;
2985 + return iswblank (wc) || wc == '\n';
2989 /* Clean up any remaining temporary files. */
2992 @@ -1255,7 +1326,7 @@ zaptemp (char const *name)
2996 -#if HAVE_NL_LANGINFO
2997 +#if HAVE_LANGINFO_CODESET
3000 struct_month_cmp (void const *m1, void const *m2)
3001 @@ -1270,7 +1341,7 @@ struct_month_cmp (void const *m1, void c
3002 /* Initialize the character class tables. */
3006 +inittables_uni (void)
3010 @@ -1282,7 +1353,7 @@ inittables (void)
3011 fold_toupper[i] = toupper (i);
3014 -#if HAVE_NL_LANGINFO
3015 +#if HAVE_LANGINFO_CODESET
3016 /* If we're not in the "C" locale, read different names for months. */
3019 @@ -1364,6 +1435,84 @@ specify_nmerge (int oi, char c, char con
3020 xstrtol_fatal (e, oi, c, long_options, s);
3025 +inittables_mb (void)
3028 + char *name, *s, *lc_time, *lc_ctype;
3029 + size_t s_len, mblength;
3030 + char mbc[MB_LEN_MAX];
3032 + mbstate_t state_mb, state_wc;
3034 + lc_time = setlocale (LC_TIME, "");
3036 + lc_time = xstrdup (lc_time);
3038 + lc_ctype = setlocale (LC_CTYPE, "");
3040 + lc_ctype = xstrdup (lc_ctype);
3042 + if (lc_time && lc_ctype)
3043 + /* temporarily set LC_CTYPE to match LC_TIME, so that we can convert
3044 + * the names of months to upper case */
3045 + setlocale (LC_CTYPE, lc_time);
3047 + for (i = 0; i < MONTHS_PER_YEAR; i++)
3049 + s = (char *) nl_langinfo (ABMON_1 + i);
3050 + s_len = strlen (s);
3051 + monthtab[i].name = name = (char *) xmalloc (s_len + 1);
3052 + monthtab[i].val = i + 1;
3054 + memset (&state_mb, '\0', sizeof (mbstate_t));
3055 + memset (&state_wc, '\0', sizeof (mbstate_t));
3057 + for (j = 0; j < s_len;)
3059 + if (!ismbblank (s + j, s_len - j, &mblength))
3064 + for (k = 0; j < s_len;)
3066 + mblength = mbrtowc (&wc, (s + j), (s_len - j), &state_mb);
3067 + assert (mblength != (size_t)-1 && mblength != (size_t)-2);
3068 + if (mblength == 0)
3071 + pwc = towupper (wc);
3074 + memcpy (mbc, s + j, mblength);
3080 + mblength = wcrtomb (mbc, pwc, &state_wc);
3081 + assert (mblength != (size_t)0 && mblength != (size_t)-1);
3084 + for (l = 0; l < mblength; l++)
3085 + name[k++] = mbc[l];
3089 + qsort ((void *) monthtab, MONTHS_PER_YEAR,
3090 + sizeof (struct month), struct_month_cmp);
3092 + if (lc_time && lc_ctype)
3093 + /* restore the original locales */
3094 + setlocale (LC_CTYPE, lc_ctype);
3101 /* Specify the amount of main memory to use when sorting. */
3103 specify_sort_size (int oi, char c, char const *s)
3104 @@ -1597,7 +1746,7 @@ buffer_linelim (struct buffer const *buf
3108 -begfield (struct line const *line, struct keyfield const *key)
3109 +begfield_uni (const struct line *line, const struct keyfield *key)
3111 char *ptr = line->text, *lim = ptr + line->length - 1;
3112 size_t sword = key->sword;
3113 @@ -1606,10 +1755,10 @@ begfield (struct line const *line, struc
3114 /* The leading field separator itself is included in a field when -t
3117 - if (tab != TAB_DEFAULT)
3119 while (ptr < lim && sword--)
3121 - while (ptr < lim && *ptr != tab)
3122 + while (ptr < lim && *ptr != tab[0])
3126 @@ -1635,11 +1784,70 @@ begfield (struct line const *line, struc
3132 +begfield_mb (const struct line *line, const struct keyfield *key)
3135 + char *ptr = line->text, *lim = ptr + line->length - 1;
3136 + size_t sword = key->sword;
3137 + size_t schar = key->schar;
3141 + memset (&state, '\0', sizeof(mbstate_t));
3144 + while (ptr < lim && sword--)
3146 + while (ptr < lim && memcmp (ptr, tab, tab_length) != 0)
3148 + GET_BYTELEN_OF_CHAR (lim, ptr, mblength, state);
3153 + GET_BYTELEN_OF_CHAR (lim, ptr, mblength, state);
3158 + while (ptr < lim && sword--)
3160 + while (ptr < lim && ismbblank (ptr, lim - ptr, &mblength))
3164 + GET_BYTELEN_OF_CHAR (lim, ptr, mblength, state);
3167 + while (ptr < lim && !ismbblank (ptr, lim - ptr, &mblength))
3171 + if (key->skipsblanks)
3172 + while (ptr < lim && ismbblank (ptr, lim - ptr, &mblength))
3175 + for (i = 0; i < schar; i++)
3177 + GET_BYTELEN_OF_CHAR (lim, ptr, mblength, state);
3179 + if (ptr + mblength > lim)
3189 /* Return the limit of (a pointer to the first character after) the field
3190 in LINE specified by KEY. */
3193 -limfield (struct line const *line, struct keyfield const *key)
3194 +limfield_uni (const struct line *line, const struct keyfield *key)
3196 char *ptr = line->text, *lim = ptr + line->length - 1;
3197 size_t eword = key->eword, echar = key->echar;
3198 @@ -1654,10 +1862,10 @@ limfield (struct line const *line, struc
3199 'beginning' is the first character following the delimiting TAB.
3200 Otherwise, leave PTR pointing at the first 'blank' character after
3201 the preceding field. */
3202 - if (tab != TAB_DEFAULT)
3204 while (ptr < lim && eword--)
3206 - while (ptr < lim && *ptr != tab)
3207 + while (ptr < lim && *ptr != tab[0])
3209 if (ptr < lim && (eword || echar))
3211 @@ -1703,10 +1911,10 @@ limfield (struct line const *line, struc
3214 /* Make LIM point to the end of (one byte past) the current field. */
3215 - if (tab != TAB_DEFAULT)
3219 - newlim = memchr (ptr, tab, lim - ptr);
3220 + newlim = memchr (ptr, tab[0], lim - ptr);
3224 @@ -1737,6 +1945,130 @@ limfield (struct line const *line, struc
3230 +limfield_mb (const struct line *line, const struct keyfield *key)
3232 + char *ptr = line->text, *lim = ptr + line->length - 1;
3233 + size_t eword = key->eword, echar = key->echar;
3239 + eword++; /* skip all of end field. */
3241 + memset (&state, '\0', sizeof(mbstate_t));
3244 + while (ptr < lim && eword--)
3246 + while (ptr < lim && memcmp (ptr, tab, tab_length) != 0)
3248 + GET_BYTELEN_OF_CHAR (lim, ptr, mblength, state);
3251 + if (ptr < lim && (eword | echar))
3253 + GET_BYTELEN_OF_CHAR (lim, ptr, mblength, state);
3258 + while (ptr < lim && eword--)
3260 + while (ptr < lim && ismbblank (ptr, lim - ptr, &mblength))
3264 + GET_BYTELEN_OF_CHAR (lim, ptr, mblength, state);
3267 + while (ptr < lim && !ismbblank (ptr, lim - ptr, &mblength))
3272 +# ifdef POSIX_UNSPECIFIED
3273 + /* Make LIM point to the end of (one byte past) the current field. */
3279 + for (p = ptr; p < lim;)
3281 + if (memcmp (p, tab, tab_length) == 0)
3287 + GET_BYTELEN_OF_CHAR (lim, ptr, mblength, state);
3296 + while (newlim < lim && ismbblank (newlim, lim - newlim, &mblength))
3297 + newlim += mblength;
3300 + GET_BYTELEN_OF_CHAR (lim, ptr, mblength, state);
3303 + while (newlim < lim && !ismbblank (newlim, lim - newlim, &mblength))
3304 + newlim += mblength;
3311 + /* If we're skipping leading blanks, don't start counting characters
3312 + * until after skipping past any leading blanks. */
3313 + if (key->skipeblanks)
3314 + while (ptr < lim && ismbblank (ptr, lim - ptr, &mblength))
3317 + memset (&state, '\0', sizeof(mbstate_t));
3319 + /* Advance PTR by ECHAR (if possible), but no further than LIM. */
3320 + for (i = 0; i < echar; i++)
3322 + GET_BYTELEN_OF_CHAR (lim, ptr, mblength, state);
3324 + if (ptr + mblength > lim)
3336 +skipblanks_uni (char **ptr, char *lim)
3338 + while (*ptr < lim && blanks[to_uchar (**ptr)])
3344 +skipblanks_mb (char **ptr, char *lim)
3347 + while (*ptr < lim && ismbblank (*ptr, lim - *ptr, &mblength))
3348 + (*ptr) += mblength;
3352 /* Fill BUF reading from FP, moving buf->left bytes from the end
3353 of buf->buf to the beginning first. If EOF is reached and the
3354 file wasn't terminated by a newline, supply one. Set up BUF's line
3355 @@ -1823,8 +2155,22 @@ fillbuf (struct buffer *buf, FILE *fp, c
3358 if (key->skipsblanks)
3359 - while (blanks[to_uchar (*line_start)])
3363 + if (MB_CUR_MAX > 1)
3366 + while (line_start < line->keylim &&
3367 + ismbblank (line_start,
3368 + line->keylim - line_start,
3370 + line_start += mblength;
3374 + while (blanks[to_uchar (*line_start)])
3377 line->keybeg = line_start;
3380 @@ -1958,12 +2304,10 @@ find_unit_order (char const *number)
3381 <none/unknown> < K/k < M < G < T < P < E < Z < Y */
3384 -human_numcompare (char const *a, char const *b)
3385 +human_numcompare (char *a, char *b)
3387 - while (blanks[to_uchar (*a)])
3389 - while (blanks[to_uchar (*b)])
3391 + skipblanks(&a, a + strlen(a));
3392 + skipblanks(&b, b + strlen(b));
3394 int diff = find_unit_order (a) - find_unit_order (b);
3395 return (diff ? diff : strnumcmp (a, b, decimal_point, thousands_sep));
3396 @@ -1974,7 +2318,7 @@ human_numcompare (char const *a, char co
3400 -numcompare (char const *a, char const *b)
3401 +numcompare_uni (const char *a, const char *b)
3403 while (blanks[to_uchar (*a)])
3405 @@ -1984,6 +2328,25 @@ numcompare (char const *a, char const *b
3406 return strnumcmp (a, b, decimal_point, thousands_sep);
3411 +numcompare_mb (const char *a, const char *b)
3413 + size_t mblength, len;
3414 + len = strlen (a); /* okay for UTF-8 */
3415 + while (*a && ismbblank (a, len > MB_CUR_MAX ? MB_CUR_MAX : len, &mblength))
3420 + len = strlen (b); /* okay for UTF-8 */
3421 + while (*b && ismbblank (b, len > MB_CUR_MAX ? MB_CUR_MAX : len, &mblength))
3424 + return strnumcmp (a, b, decimal_point, thousands_sep);
3426 +#endif /* HAV_EMBRTOWC */
3428 /* Work around a problem whereby the long double value returned by glibc's
3429 strtold ("NaN", ...) contains uninitialized bits: clear all bytes of
3430 A and B before calling strtold. FIXME: remove this function once
3431 @@ -2034,7 +2397,7 @@ general_numcompare (char const *sa, char
3432 Return 0 if the name in S is not recognized. */
3435 -getmonth (char const *month, char **ea)
3436 +getmonth_uni (char const *month, size_t len, char **ea)
3439 size_t hi = MONTHS_PER_YEAR;
3440 @@ -2310,15 +2673,14 @@ debug_key (struct line const *line, stru
3444 - while (blanks[to_uchar (*beg)])
3446 + skipblanks (&beg, lim);
3448 char *tighter_lim = beg;
3452 else if (key->month)
3453 - getmonth (beg, &tighter_lim);
3454 + getmonth (beg, lim-beg, &tighter_lim);
3455 else if (key->general_numeric)
3456 ignore_value (strtold (beg, &tighter_lim));
3457 else if (key->numeric || key->human_numeric)
3458 @@ -2452,7 +2814,7 @@ key_warnings (struct keyfield const *gke
3459 /* Warn about significant leading blanks. */
3460 bool implicit_skip = key_numeric (key) || key->month;
3461 bool line_offset = key->eword == 0 && key->echar != 0; /* -k1.x,1.y */
3462 - if (!zero_width && !gkey_only && tab == TAB_DEFAULT && !line_offset
3463 + if (!zero_width && !gkey_only && !tab_length && !line_offset
3464 && ((!key->skipsblanks && !implicit_skip)
3465 || (!key->skipsblanks && key->schar)
3466 || (!key->skipeblanks && key->echar)))
3467 @@ -2510,11 +2872,87 @@ key_warnings (struct keyfield const *gke
3468 error (0, 0, _("option '-r' only applies to last-resort comparison"));
3473 +getmonth_mb (const char *s, size_t len, char **ea)
3476 + register size_t i;
3477 + register int lo = 0, hi = MONTHS_PER_YEAR, result;
3479 + size_t wclength, mblength;
3481 + const wchar_t *wpp;
3482 + wchar_t *month_wcs;
3485 + while (len > 0 && ismbblank (s, len, &mblength))
3494 + if (SIZE_MAX - len < 1)
3497 + month = (char *) xnmalloc (len + 1, MB_CUR_MAX);
3499 + pp = tmp = (char *) xnmalloc (len + 1, MB_CUR_MAX);
3500 + memcpy (tmp, s, len);
3502 + wpp = month_wcs = (wchar_t *) xnmalloc (len + 1, sizeof (wchar_t));
3503 + memset (&state, '\0', sizeof (mbstate_t));
3505 + wclength = mbsrtowcs (month_wcs, &pp, len + 1, &state);
3506 + if (wclength == (size_t)-1 || pp != NULL)
3507 + error (SORT_FAILURE, 0, _("Invalid multibyte input %s."), quote(s));
3509 + for (i = 0; i < wclength; i++)
3511 + month_wcs[i] = towupper(month_wcs[i]);
3512 + if (iswblank (month_wcs[i]))
3514 + month_wcs[i] = L'\0';
3519 + mblength = wcsrtombs (month, &wpp, (len + 1) * MB_CUR_MAX, &state);
3520 + assert (mblength != (-1) && wpp == NULL);
3524 + int ix = (lo + hi) / 2;
3526 + if (strncmp (month, monthtab[ix].name, strlen (monthtab[ix].name)) < 0)
3531 + while (hi - lo > 1);
3533 + result = (!strncmp (month, monthtab[lo].name, strlen (monthtab[lo].name))
3534 + ? monthtab[lo].val : 0);
3537 + *ea = (char*) s + strlen (monthtab[lo].name);
3547 /* Compare two lines A and B trying every key in sequence until there
3548 are no more keys or a difference is found. */
3551 -keycompare (struct line const *a, struct line const *b)
3552 +keycompare_uni (const struct line *a, const struct line *b)
3554 struct keyfield *key = keylist;
3556 @@ -2599,7 +3037,7 @@ keycompare (struct line const *a, struct
3557 else if (key->human_numeric)
3558 diff = human_numcompare (ta, tb);
3559 else if (key->month)
3560 - diff = getmonth (ta, NULL) - getmonth (tb, NULL);
3561 + diff = getmonth (ta, tlena, NULL) - getmonth (tb, tlenb, NULL);
3562 else if (key->random)
3563 diff = compare_random (ta, tlena, tb, tlenb);
3564 else if (key->version)
3565 @@ -2715,6 +3153,211 @@ keycompare (struct line const *a, struct
3566 return key->reverse ? -diff : diff;
3571 +keycompare_mb (const struct line *a, const struct line *b)
3573 + struct keyfield *key = keylist;
3575 + /* For the first iteration only, the key positions have been
3576 + precomputed for us. */
3577 + char *texta = a->keybeg;
3578 + char *textb = b->keybeg;
3579 + char *lima = a->keylim;
3580 + char *limb = b->keylim;
3582 + size_t mblength_a, mblength_b;
3583 + wchar_t wc_a, wc_b;
3584 + mbstate_t state_a, state_b;
3588 + memset (&state_a, '\0', sizeof(mbstate_t));
3589 + memset (&state_b, '\0', sizeof(mbstate_t));
3590 + /* Ignore keys with start after end. */
3591 + if (a->keybeg - a->keylim > 0)
3595 + /* Ignore and/or translate chars before comparing. */
3596 +# define IGNORE_CHARS(NEW_LEN, LEN, TEXT, COPY, WC, MBLENGTH, STATE) \
3600 + char mbc[MB_LEN_MAX]; \
3601 + mbstate_t state_wc; \
3603 + for (NEW_LEN = i = 0; i < LEN;) \
3605 + mbstate_t state_bak; \
3607 + state_bak = STATE; \
3608 + MBLENGTH = mbrtowc (&WC, TEXT + i, LEN - i, &STATE); \
3610 + if (MBLENGTH == (size_t)-2 || MBLENGTH == (size_t)-1 \
3611 + || MBLENGTH == 0) \
3613 + if (MBLENGTH == (size_t)-2 || MBLENGTH == (size_t)-1) \
3614 + STATE = state_bak; \
3616 + COPY[NEW_LEN++] = TEXT[i]; \
3623 + if ((ignore == nonprinting && !iswprint (WC)) \
3624 + || (ignore == nondictionary \
3625 + && !iswalnum (WC) && !iswblank (WC))) \
3635 + uwc = towupper(WC); \
3638 + memcpy (mbc, TEXT + i, MBLENGTH); \
3645 + memset (&state_wc, '\0', sizeof (mbstate_t)); \
3647 + MBLENGTH = wcrtomb (mbc, WC, &state_wc); \
3648 + assert (MBLENGTH != (size_t)-1 && MBLENGTH != 0); \
3651 + for (j = 0; j < MBLENGTH; j++) \
3652 + COPY[NEW_LEN++] = mbc[j]; \
3655 + for (j = 0; j < MBLENGTH; j++) \
3656 + COPY[NEW_LEN++] = TEXT[i++]; \
3658 + COPY[NEW_LEN] = '\0'; \
3662 + /* Actually compare the fields. */
3666 + /* Find the lengths. */
3667 + size_t lena = lima <= texta ? 0 : lima - texta;
3668 + size_t lenb = limb <= textb ? 0 : limb - textb;
3670 + char enda IF_LINT (= 0);
3671 + char endb IF_LINT (= 0);
3673 + char const *translate = key->translate;
3674 + bool const *ignore = key->ignore;
3676 + if (ignore || translate)
3678 + if (SIZE_MAX - lenb - 2 < lena)
3680 + char *copy_a = (char *) xnmalloc (lena + lenb + 2, MB_CUR_MAX);
3681 + char *copy_b = copy_a + lena * MB_CUR_MAX + 1;
3682 + size_t new_len_a, new_len_b;
3685 + IGNORE_CHARS (new_len_a, lena, texta, copy_a,
3686 + wc_a, mblength_a, state_a);
3687 + IGNORE_CHARS (new_len_b, lenb, textb, copy_b,
3688 + wc_b, mblength_b, state_b);
3689 + texta = copy_a; textb = copy_b;
3690 + lena = new_len_a; lenb = new_len_b;
3694 + /* Use the keys in-place, temporarily null-terminated. */
3695 + enda = texta[lena]; texta[lena] = '\0';
3696 + endb = textb[lenb]; textb[lenb] = '\0';
3700 + diff = compare_random (texta, lena, textb, lenb);
3701 + else if (key->numeric | key->general_numeric | key->human_numeric)
3703 + char savea = *lima, saveb = *limb;
3705 + *lima = *limb = '\0';
3706 + diff = (key->numeric ? numcompare (texta, textb)
3707 + : key->general_numeric ? general_numcompare (texta, textb)
3708 + : human_numcompare (texta, textb));
3709 + *lima = savea, *limb = saveb;
3711 + else if (key->version)
3712 + diff = filevercmp (texta, textb);
3713 + else if (key->month)
3714 + diff = getmonth (texta, lena, NULL) - getmonth (textb, lenb, NULL);
3715 + else if (lena == 0)
3716 + diff = - NONZERO (lenb);
3717 + else if (lenb == 0)
3719 + else if (hard_LC_COLLATE && !folding)
3721 + diff = xmemcoll0 (texta, lena + 1, textb, lenb + 1);
3725 + diff = memcmp (texta, textb, MIN (lena, lenb));
3727 + diff = lena < lenb ? -1 : lena != lenb;
3730 + if (ignore || translate)
3734 + texta[lena] = enda;
3735 + textb[lenb] = endb;
3745 + /* Find the beginning and limit of the next field. */
3746 + if (key->eword != -1)
3747 + lima = limfield (a, key), limb = limfield (b, key);
3749 + lima = a->text + a->length - 1, limb = b->text + b->length - 1;
3751 + if (key->sword != -1)
3752 + texta = begfield (a, key), textb = begfield (b, key);
3755 + texta = a->text, textb = b->text;
3756 + if (key->skipsblanks)
3758 + while (texta < lima && ismbblank (texta, lima - texta, &mblength_a))
3759 + texta += mblength_a;
3760 + while (textb < limb && ismbblank (textb, limb - textb, &mblength_b))
3761 + textb += mblength_b;
3767 + if (key && key->reverse)
3774 /* Compare two lines A and B, returning negative, zero, or positive
3775 depending on whether A compares less than, equal to, or greater than B. */
3777 @@ -2742,7 +3385,7 @@ compare (struct line const *a, struct li
3778 diff = - NONZERO (blen);
3781 - else if (hard_LC_COLLATE)
3782 + else if (hard_LC_COLLATE && !folding)
3784 /* Note xmemcoll0 is a performance enhancement as
3785 it will not unconditionally write '\0' after the
3786 @@ -4139,6 +4782,7 @@ set_ordering (char const *s, struct keyf
3789 key->translate = fold_toupper;
3793 key->general_numeric = true;
3794 @@ -4218,7 +4862,7 @@ main (int argc, char **argv)
3795 initialize_exit_failure (SORT_FAILURE);
3797 hard_LC_COLLATE = hard_locale (LC_COLLATE);
3798 -#if HAVE_NL_LANGINFO
3799 +#if HAVE_LANGINFO_CODESET
3800 hard_LC_TIME = hard_locale (LC_TIME);
3803 @@ -4239,6 +4883,29 @@ main (int argc, char **argv)
3808 + if (MB_CUR_MAX > 1)
3810 + inittables = inittables_mb;
3811 + begfield = begfield_mb;
3812 + limfield = limfield_mb;
3813 + skipblanks = skipblanks_mb;
3814 + getmonth = getmonth_mb;
3815 + keycompare = keycompare_mb;
3816 + numcompare = numcompare_mb;
3821 + inittables = inittables_uni;
3822 + begfield = begfield_uni;
3823 + limfield = limfield_uni;
3824 + skipblanks = skipblanks_uni;
3825 + getmonth = getmonth_uni;
3826 + keycompare = keycompare_uni;
3827 + numcompare = numcompare_uni;
3830 have_read_stdin = false;
3833 @@ -4513,13 +5180,34 @@ main (int argc, char **argv)
3837 - char newtab = optarg[0];
3839 + char newtab[MB_LEN_MAX + 1];
3840 + size_t newtab_length = 1;
3841 + strncpy (newtab, optarg, MB_LEN_MAX);
3843 die (SORT_FAILURE, 0, _("empty tab"));
3846 + if (MB_CUR_MAX > 1)
3851 + memset (&state, '\0', sizeof (mbstate_t));
3852 + newtab_length = mbrtowc (&wc, newtab, strnlen (newtab,
3855 + switch (newtab_length)
3860 + newtab_length = 1;
3864 + if (newtab_length == 1 && optarg[1])
3866 if (STREQ (optarg, "\\0"))
3871 /* Provoke with 'sort -txx'. Complain about
3872 @@ -4530,9 +5218,11 @@ main (int argc, char **argv)
3876 - if (tab != TAB_DEFAULT && tab != newtab)
3877 + if (tab_length && (tab_length != newtab_length
3878 + || memcmp (tab, newtab, tab_length) != 0))
3879 die (SORT_FAILURE, 0, _("incompatible tabs"));
3881 + memcpy (tab, newtab, newtab_length);
3882 + tab_length = newtab_length;
3886 @@ -4770,12 +5460,10 @@ main (int argc, char **argv)
3887 sort (files, nfiles, outfile, nthreads);
3892 readtokens0_free (&tok);
3897 if (have_read_stdin && fclose (stdin) == EOF)
3898 sort_die (_("close failed"), "-");
3899 diff -Naurp coreutils-8.27-orig/src/unexpand.c coreutils-8.27/src/unexpand.c
3900 --- coreutils-8.27-orig/src/unexpand.c 2017-01-01 16:34:24.000000000 -0600
3901 +++ coreutils-8.27/src/unexpand.c 2017-03-11 23:49:06.758133530 -0600
3905 #include <sys/types.h>
3907 +#include <mbfile.h>
3911 #include "xstrndup.h"
3912 @@ -107,24 +110,47 @@ unexpand (void)
3915 FILE *fp = next_file (NULL);
3918 /* The array of pending blanks. In non-POSIX locales, blanks can
3919 include characters other than spaces, so the blanks must be
3920 stored, not merely counted. */
3921 - char *pending_blank;
3922 + mbf_char_t *pending_blank;
3923 + /* True if the starting locale is utf8. */
3924 + bool using_utf_locale;
3926 + /* True if the first file contains BOM header. */
3928 + using_utf_locale=check_utf_locale();
3932 + mbf_init (mbf, fp);
3933 + found_bom=check_bom(fp,&mbf);
3935 + if (using_utf_locale == false && found_bom == true)
3937 + /*try using some predefined locale */
3939 + if (set_utf_locale () != 0)
3941 + error (EXIT_FAILURE, errno, _("cannot set UTF-8 locale"));
3944 /* The worst case is a non-blank character, then one blank, then a
3945 tab stop, then MAX_COLUMN_WIDTH - 1 blanks, then a non-blank; so
3946 allocate MAX_COLUMN_WIDTH bytes to store the blanks. */
3947 - pending_blank = xmalloc (max_column_width);
3948 + pending_blank = xmalloc (max_column_width * sizeof (mbf_char_t));
3950 + if (found_bom == true)
3957 /* Input character, or EOF. */
3961 /* If true, perform translations. */
3962 bool convert = true;
3963 @@ -158,12 +184,44 @@ unexpand (void)
3967 - while ((c = getc (fp)) < 0 && (fp = next_file (fp)))
3970 + mbf_getc (c, mbf);
3971 + if ((mb_iseof (c)) && (fp = next_file (fp)))
3973 + mbf_init (mbf, fp);
3976 + if (check_bom(fp,&mbf)==true)
3978 + /*Not the first file - check BOM header*/
3979 + if (using_utf_locale==false && found_bom==false)
3981 + /*BOM header in subsequent file but not in the first one. */
3982 + error (EXIT_FAILURE, errno, _("combination of files with and without BOM header"));
3987 + if(using_utf_locale==false && found_bom==true)
3989 + /*First file conatined BOM header - locale was switched to UTF
3990 + /*all subsequent files should contain BOM. */
3991 + error (EXIT_FAILURE, errno, _("combination of files with and without BOM header"));
4006 - bool blank = !! isblank (c);
4007 + bool blank = mb_isblank (c);
4011 @@ -180,16 +238,16 @@ unexpand (void)
4012 if (next_tab_column < column)
4013 die (EXIT_FAILURE, 0, _("input line is too long"));
4016 + if (mb_iseq (c, '\t'))
4018 column = next_tab_column;
4021 - pending_blank[0] = '\t';
4022 + mb_setascii (&pending_blank[0], '\t');
4027 + column += mb_width (c);
4029 if (! (prev_blank && column == next_tab_column))
4031 @@ -197,13 +255,14 @@ unexpand (void)
4032 will be replaced by tabs. */
4033 if (column == next_tab_column)
4034 one_blank_before_tab_stop = true;
4035 - pending_blank[pending++] = c;
4036 + mb_copy (&pending_blank[pending++], &c);
4041 /* Replace the pending blanks by a tab or two. */
4042 - pending_blank[0] = c = '\t';
4043 + mb_setascii (&c, '\t');
4044 + mb_setascii (&pending_blank[0], '\t');
4047 /* Discard pending blanks, unless it was a single
4048 @@ -211,7 +270,7 @@ unexpand (void)
4049 pending = one_blank_before_tab_stop;
4052 - else if (c == '\b')
4053 + else if (mb_iseq (c, '\b'))
4055 /* Go back one column, and force recalculation of the
4057 @@ -219,9 +278,9 @@ unexpand (void)
4058 next_tab_column = column;
4059 tab_index -= !!tab_index;
4062 + else if (!mb_iseq (c, '\n'))
4065 + column += mb_width (c);
4067 die (EXIT_FAILURE, 0, _("input line is too long"));
4069 @@ -229,8 +288,11 @@ unexpand (void)
4072 if (pending > 1 && one_blank_before_tab_stop)
4073 - pending_blank[0] = '\t';
4074 - if (fwrite (pending_blank, 1, pending, stdout) != pending)
4075 + mb_setascii (&pending_blank[0], '\t');
4077 + for (int n = 0; n < pending; ++n)
4078 + mb_putc (pending_blank[n], stdout);
4079 + if (ferror (stdout))
4080 die (EXIT_FAILURE, errno, _("write error"));
4082 one_blank_before_tab_stop = false;
4083 @@ -240,16 +302,17 @@ unexpand (void)
4084 convert &= convert_entire_line || blank;
4090 free (pending_blank);
4094 - if (putchar (c) < 0)
4095 + mb_putc (c, stdout);
4096 + if (ferror (stdout))
4097 die (EXIT_FAILURE, errno, _("write error"));
4099 - while (c != '\n');
4100 + while (!mb_iseq (c, '\n'));
4104 diff -Naurp coreutils-8.27-orig/src/uniq.c coreutils-8.27/src/uniq.c
4105 --- coreutils-8.27-orig/src/uniq.c 2017-01-01 16:34:24.000000000 -0600
4106 +++ coreutils-8.27/src/uniq.c 2017-03-11 23:47:13.098285938 -0600
4109 #include <sys/types.h>
4111 +/* Get mbstate_t, mbrtowc(). */
4113 +# include <wchar.h>
4116 +/* Get isw* functions. */
4118 +# include <wctype.h>
4120 +#include <assert.h>
4123 #include "argmatch.h"
4124 #include "linebuffer.h"
4126 #include "stdio--.h"
4127 #include "xmemcoll.h"
4128 #include "xstrtol.h"
4129 -#include "memcasecmp.h"
4130 +#include "xmemcoll.h"
4133 +/* MB_LEN_MAX is incorrectly defined to be 1 in at least one GCC
4134 + installation; work around this configuration error. */
4135 +#if !defined MB_LEN_MAX || MB_LEN_MAX < 2
4136 +# define MB_LEN_MAX 16
4139 +/* Some systems, like BeOS, have multibyte encodings but lack mbstate_t. */
4140 +#if HAVE_MBRTOWC && defined mbstate_t
4141 +# define mbrtowc(pwc, s, n, ps) (mbrtowc) (pwc, s, n, 0)
4145 /* The official name of this program (e.g., no 'g' prefix). */
4146 #define PROGRAM_NAME "uniq"
4148 @@ -144,6 +167,10 @@ enum
4149 GROUP_OPTION = CHAR_MAX + 1
4152 +/* Function pointers. */
4154 +(*find_field) (struct linebuffer *line);
4156 static struct option const longopts[] =
4158 {"count", no_argument, NULL, 'c'},
4159 @@ -260,7 +287,7 @@ size_opt (char const *opt, char const *m
4160 return a pointer to the beginning of the line's field to be compared. */
4162 static char * _GL_ATTRIBUTE_PURE
4163 -find_field (struct linebuffer const *line)
4164 +find_field_uni (struct linebuffer *line)
4167 char const *lp = line->buffer;
4168 @@ -280,6 +307,83 @@ find_field (struct linebuffer const *lin
4169 return line->buffer + i;
4174 +# define MBCHAR_TO_WCHAR(WC, MBLENGTH, LP, POS, SIZE, STATEP, CONVFAIL) \
4177 + mbstate_t state_bak; \
4180 + state_bak = *STATEP; \
4182 + MBLENGTH = mbrtowc (&WC, LP + POS, SIZE - POS, STATEP); \
4184 + switch (MBLENGTH) \
4186 + case (size_t)-2: \
4187 + case (size_t)-1: \
4188 + *STATEP = state_bak; \
4190 + /* Fall through */ \
4198 +find_field_multi (struct linebuffer *line)
4201 + char *lp = line->buffer;
4202 + size_t size = line->length - 1;
4206 + mbstate_t *statep;
4210 + statep = &(line->state);
4212 + /* skip fields. */
4213 + for (count = 0; count < skip_fields && pos < size; count++)
4215 + while (pos < size)
4217 + MBCHAR_TO_WCHAR (wc, mblength, lp, pos, size, statep, convfail);
4219 + if (convfail || !(iswblank (wc) || wc == '\n'))
4227 + while (pos < size)
4229 + MBCHAR_TO_WCHAR (wc, mblength, lp, pos, size, statep, convfail);
4231 + if (!convfail && (iswblank (wc) || wc == '\n'))
4238 + /* skip fields. */
4239 + for (count = 0; count < skip_chars && pos < size; count++)
4241 + MBCHAR_TO_WCHAR (wc, mblength, lp, pos, size, statep, convfail);
4249 /* Return false if two strings OLD and NEW match, true if not.
4250 OLD and NEW point not to the beginnings of the lines
4251 but rather to the beginnings of the fields to compare.
4252 @@ -288,6 +392,8 @@ find_field (struct linebuffer const *lin
4254 different (char *old, char *new, size_t oldlen, size_t newlen)
4256 + char *copy_old, *copy_new;
4258 if (check_chars < oldlen)
4259 oldlen = check_chars;
4260 if (check_chars < newlen)
4261 @@ -295,14 +401,103 @@ different (char *old, char *new, size_t
4265 - /* FIXME: This should invoke strcoll somehow. */
4266 - return oldlen != newlen || memcasecmp (old, new, oldlen);
4269 + copy_old = xmalloc (oldlen + 1);
4270 + copy_new = xmalloc (oldlen + 1);
4272 + for (i = 0; i < oldlen; i++)
4274 + copy_old[i] = toupper (old[i]);
4275 + copy_new[i] = toupper (new[i]);
4277 + bool rc = xmemcoll (copy_old, oldlen, copy_new, newlen);
4282 - else if (hard_LC_COLLATE)
4283 - return xmemcoll (old, oldlen, new, newlen) != 0;
4285 - return oldlen != newlen || memcmp (old, new, oldlen);
4287 + copy_old = (char *)old;
4288 + copy_new = (char *)new;
4291 + return xmemcoll (copy_old, oldlen, copy_new, newlen);
4297 +different_multi (const char *old, const char *new, size_t oldlen, size_t newlen, mbstate_t oldstate, mbstate_t newstate)
4299 + size_t i, j, chars;
4300 + const char *str[2];
4303 + mbstate_t state[2];
4306 + mbstate_t state_bak;
4312 + state[0] = oldstate;
4313 + state[1] = newstate;
4315 + for (i = 0; i < 2; i++)
4317 + copy[i] = xmalloc (len[i] + 1);
4318 + memset (copy[i], '\0', len[i] + 1);
4320 + for (j = 0, chars = 0; j < len[i] && chars < check_chars; chars++)
4322 + state_bak = state[i];
4323 + mblength = mbrtowc (&wc, str[i] + j, len[i] - j, &(state[i]));
4329 + state[i] = state_bak;
4330 + /* Fall through */
4338 + uwc = towupper (wc);
4342 + mbstate_t state_wc;
4345 + memset (&state_wc, '\0', sizeof(mbstate_t));
4346 + mblen = wcrtomb (copy[i] + j, uwc, &state_wc);
4347 + assert (mblen != (size_t)-1);
4350 + memcpy (copy[i] + j, str[i] + j, mblength);
4353 + memcpy (copy[i] + j, str[i] + j, mblength);
4357 + copy[i][j] = '\0';
4360 + int rc = xmemcoll (copy[0], len[0], copy[1], len[1]);
4368 /* Output the line in linebuffer LINE to standard output
4369 provided that the switches say it should be output.
4370 @@ -367,19 +562,38 @@ check_file (const char *infile, const ch
4371 char *prevfield IF_LINT ( = NULL);
4372 size_t prevlen IF_LINT ( = 0);
4373 bool first_group_printed = false;
4375 + mbstate_t prevstate;
4377 + memset (&prevstate, '\0', sizeof (mbstate_t));
4380 while (!feof (stdin))
4386 + mbstate_t thisstate;
4389 if (readlinebuffer_delim (thisline, stdin, delimiter) == 0)
4392 thisfield = find_field (thisline);
4393 thislen = thisline->length - 1 - (thisfield - thisline->buffer);
4395 + if (MB_CUR_MAX > 1)
4397 + thisstate = thisline->state;
4399 + new_group = (prevline->length == 0
4400 + || different_multi (thisfield, prevfield,
4402 + thisstate, prevstate));
4406 new_group = (prevline->length == 0
4407 || different (thisfield, prevfield, thislen, prevlen));
4409 @@ -397,6 +611,10 @@ check_file (const char *infile, const ch
4410 SWAP_LINES (prevline, thisline);
4411 prevfield = thisfield;
4414 + if (MB_CUR_MAX > 1)
4415 + prevstate = thisstate;
4417 first_group_printed = true;
4420 @@ -409,17 +627,26 @@ check_file (const char *infile, const ch
4422 uintmax_t match_count = 0;
4423 bool first_delimiter = true;
4425 + mbstate_t prevstate;
4428 if (readlinebuffer_delim (prevline, stdin, delimiter) == 0)
4430 prevfield = find_field (prevline);
4431 prevlen = prevline->length - 1 - (prevfield - prevline->buffer);
4433 + prevstate = prevline->state;
4436 while (!feof (stdin))
4442 + mbstate_t thisstate = thisline->state;
4444 if (readlinebuffer_delim (thisline, stdin, delimiter) == 0)
4447 @@ -428,6 +655,14 @@ check_file (const char *infile, const ch
4449 thisfield = find_field (thisline);
4450 thislen = thisline->length - 1 - (thisfield - thisline->buffer);
4452 + if (MB_CUR_MAX > 1)
4454 + match = !different_multi (thisfield, prevfield,
4455 + thislen, prevlen, thisstate, prevstate);
4459 match = !different (thisfield, prevfield, thislen, prevlen);
4460 match_count += match;
4462 @@ -460,6 +695,9 @@ check_file (const char *infile, const ch
4463 SWAP_LINES (prevline, thisline);
4464 prevfield = thisfield;
4467 + prevstate = thisstate;
4472 @@ -506,6 +744,19 @@ main (int argc, char **argv)
4474 atexit (close_stdout);
4477 + if (MB_CUR_MAX > 1)
4479 + find_field = find_field_multi;
4484 + find_field = find_field_uni;
4491 check_chars = SIZE_MAX;
4492 diff -Naurp coreutils-8.27-orig/tests/expand/mb.sh coreutils-8.27/tests/expand/mb.sh
4493 --- coreutils-8.27-orig/tests/expand/mb.sh 1969-12-31 18:00:00.000000000 -0600
4494 +++ coreutils-8.27/tests/expand/mb.sh 2017-03-11 23:49:06.759133489 -0600
4498 +# Copyright (C) 2012-2017 Free Software Foundation, Inc.
4500 +# This program is free software: you can redistribute it and/or modify
4501 +# it under the terms of the GNU General Public License as published by
4502 +# the Free Software Foundation, either version 3 of the License, or
4503 +# (at your option) any later version.
4505 +# This program is distributed in the hope that it will be useful,
4506 +# but WITHOUT ANY WARRANTY; without even the implied warranty of
4507 +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
4508 +# GNU General Public License for more details.
4510 +# You should have received a copy of the GNU General Public License
4511 +# along with this program. If not, see <http://www.gnu.org/licenses/>.
4513 +. "${srcdir=.}/tests/init.sh"; path_prepend_ ./src
4516 +export LC_ALL=en_US.UTF-8
4518 +#input containing multibyte characters
4519 +cat <<\EOF > in || framework_failure_
4520 +1234567812345678123456781
4527 +env printf ' äöü\t. öüä. \tä xx\n' >> in || framework_failure_
4529 +cat <<\EOF > exp || framework_failure_
4530 +1234567812345678123456781
4539 +expand < in > out || fail=1
4540 +compare exp out > /dev/null 2>&1 || fail=1
4542 +#multiple files as an input
4543 +cat <<\EOF >> exp || framework_failure_
4544 +1234567812345678123456781
4553 +expand ./in ./in > out || fail=1
4554 +compare exp out > /dev/null 2>&1 || fail=1
4556 +#test characters with display widths != 1
4557 +env printf '12345678
4559 +\u00E9\t|composed(1)
4560 +e\u0301\t|decomposed(1)
4561 +\u3000\t|ideo-space(2)
4562 +\uFF0D\t|full-hypen(2)
4563 +' > in || framework_failure_
4565 +env printf '12345678
4567 +\u00E9 |composed(1)
4568 +e\u0301 |decomposed(1)
4569 +\u3000 |ideo-space(2)
4570 +\uFF0D |full-hypen(2)
4571 +' > exp || framework_failure_
4573 +expand < in > out || fail=1
4574 +compare exp out > /dev/null 2>&1 || fail=1
4576 +#shouldn't fail with "input line too long"
4577 +#when a line starts with a control character
4578 +env printf '\n' > in || framework_failure_
4580 +expand < in > out || fail=1
4581 +compare in out > /dev/null 2>&1 || fail=1
4583 +#non-Unicode characters interspersed between Unicode ones
4584 +env printf '12345678
4592 +' > in || framework_failure_
4594 +env printf '12345678
4602 +' > exp || framework_failure_
4604 +expand < in > out || fail=1
4605 +compare exp out > /dev/null 2>&1 || fail=1
4610 +printf "\xEF\xBB\xBF" > in; cat <<\EOF >> in || framework_failure_
4611 +1234567812345678123456781
4618 +env printf ' äöü\t. öüä. \tä xx\n' >> in || framework_failure_
4620 +printf "\xEF\xBB\xBF" > exp; cat <<\EOF >> exp || framework_failure_
4621 +1234567812345678123456781
4631 +expand < in > out || fail=1
4632 +compare exp out > /dev/null 2>&1 || fail=1
4634 +LANG=C expand < in > out || fail=1
4635 +compare exp out > /dev/null 2>&1 || fail=1
4637 +LC_ALL=C expand < in > out || fail=1
4638 +compare exp out > /dev/null 2>&1 || fail=1
4641 +printf '\xEF\xBB\xBF' > in1; cat <<\EOF >> in1 || framework_failure_
4642 +1234567812345678123456781
4649 +env printf ' äöü\t. öüä. \tä xx\n' >> in1 || framework_failure_
4652 +printf '\xEF\xBB\xBF' > exp; cat <<\EOF >> exp || framework_failure_
4653 +1234567812345678123456781
4660 +1234567812345678123456781
4669 +expand in1 in1 > out || fail=1
4670 +compare exp out > /dev/null 2>&1 || fail=1
4672 +LANG=C expand in1 in1 > out || fail=1
4673 +compare exp out > /dev/null 2>&1 || fail=1
4675 +LC_ALL=C expand in1 in1 > out || fail=1
4676 +compare exp out > /dev/null 2>&1 || fail=1
4679 diff -Naurp coreutils-8.27-orig/tests/i18n/sort.sh coreutils-8.27/tests/i18n/sort.sh
4680 --- coreutils-8.27-orig/tests/i18n/sort.sh 1969-12-31 18:00:00.000000000 -0600
4681 +++ coreutils-8.27/tests/i18n/sort.sh 2017-03-11 23:47:13.100285838 -0600
4684 +# Verify sort's multi-byte support.
4686 +. "${srcdir=.}/tests/init.sh"; path_prepend_ ./src
4689 +export LC_ALL=en_US.UTF-8
4690 +locale -k LC_CTYPE | grep -q "charmap.*UTF-8" \
4691 + || skip_ "No UTF-8 locale available"
4693 +# Enable heap consistency checkng on older systems
4694 +export MALLOC_CHECK_=2
4697 +# check buffer overflow issue due to
4698 +# expanding multi-byte representation due to case conversion
4699 +# https://bugzilla.suse.com/show_bug.cgi?id=928749
4704 +cat <<EOF | sort -f > out || fail=1
4708 +compare exp out || { fail=1; cat out; }
4712 diff -Naurp coreutils-8.27-orig/tests/local.mk coreutils-8.27/tests/local.mk
4713 --- coreutils-8.27-orig/tests/local.mk 2017-02-28 22:25:37.000000000 -0600
4714 +++ coreutils-8.27/tests/local.mk 2017-03-11 23:47:38.072058253 -0600
4715 @@ -352,6 +352,8 @@ all_tests = \
4716 tests/misc/sort-discrim.sh \
4717 tests/misc/sort-files0-from.pl \
4718 tests/misc/sort-float.sh \
4719 + tests/misc/sort-mb-tests.sh \
4720 + tests/i18n/sort.sh \
4721 tests/misc/sort-h-thousands-sep.sh \
4722 tests/misc/sort-merge.pl \
4723 tests/misc/sort-merge-fdlimit.sh \
4724 @@ -544,6 +546,7 @@ all_tests = \
4725 tests/du/threshold.sh \
4726 tests/du/trailing-slash.sh \
4727 tests/du/two-args.sh \
4728 + tests/expand/mb.sh \
4729 tests/id/gnu-zero-uids.sh \
4730 tests/id/no-context.sh \
4731 tests/id/context.sh \
4732 @@ -684,6 +687,7 @@ all_tests = \
4733 tests/touch/read-only.sh \
4734 tests/touch/relative.sh \
4735 tests/touch/trailing-slash.sh \
4736 + tests/unexpand/mb.sh \
4739 # See tests/factor/create-test.sh.
4740 diff -Naurp coreutils-8.27-orig/tests/misc/cut.pl coreutils-8.27/tests/misc/cut.pl
4741 --- coreutils-8.27-orig/tests/misc/cut.pl 2017-01-01 16:34:24.000000000 -0600
4742 +++ coreutils-8.27/tests/misc/cut.pl 2017-03-11 23:47:13.100285838 -0600
4743 @@ -23,9 +23,11 @@ use strict;
4744 # Turn off localization of executable's output.
4745 @ENV{qw(LANGUAGE LANG LC_ALL)} = ('C') x 3;
4747 -my $mb_locale = $ENV{LOCALE_FR_UTF8};
4749 +# uncommented enable multibyte paths
4750 +$mb_locale = $ENV{LOCALE_FR_UTF8};
4751 ! defined $mb_locale || $mb_locale eq 'none'
4752 - and $mb_locale = 'C';
4753 + and $mb_locale = 'C';
4756 my $try = "Try '$prog --help' for more information.\n";
4757 @@ -240,6 +242,7 @@ if ($mb_locale ne 'C')
4759 my $test_name = shift @new_t;
4761 + next if ($test_name =~ "newline-[12][0-9]");
4762 push @new, ["$test_name-mb", @new_t, {ENV => "LC_ALL=$mb_locale"}];
4765 diff -Naurp coreutils-8.27-orig/tests/misc/expand.pl coreutils-8.27/tests/misc/expand.pl
4766 --- coreutils-8.27-orig/tests/misc/expand.pl 2017-03-01 11:16:46.000000000 -0600
4767 +++ coreutils-8.27/tests/misc/expand.pl 2017-03-11 23:47:13.101285788 -0600
4768 @@ -27,6 +27,15 @@ my $prog = 'expand';
4769 # Turn off localization of executable's output.
4770 @ENV{qw(LANGUAGE LANG LC_ALL)} = ('C') x 3;
4772 +#comment out next line to disable multibyte tests
4773 +my $mb_locale = $ENV{LOCALE_FR_UTF8};
4774 +! defined $mb_locale || $mb_locale eq 'none'
4775 + and $mb_locale = 'C';
4777 +my $prog = 'expand';
4778 +my $try = "Try \`$prog --help' for more information.\n";
4779 +my $inval = "$prog: invalid byte, character or field list\n$try";
4783 ['t1', '--tabs=3', {IN=>"a\tb"}, {OUT=>"a b"}],
4784 @@ -152,6 +161,8 @@ my @Tests =
4785 ['trail9', '--tab=1,2 -t/5',{IN=>"\ta\tb\tc"}, {OUT=>" a b c"}],
4788 + # FIXME: The following tests contain ‘quoting’ specific to LC_MESSAGES
4789 + # So we force LC_MESSAGES=C to make them pass.
4790 ['e1', '--tabs="a"', {IN=>''}, {OUT=>''}, {EXIT=>1},
4791 {ERR => "$prog: tab size contains invalid character(s): 'a'\n"}],
4792 ['e2', "-t $UINTMAX_OFLOW", {IN=>''}, {OUT=>''}, {EXIT=>1},
4793 @@ -168,6 +179,37 @@ my @Tests =
4794 {ERR => "$prog: '/' specifier not at start of number: '/'\n"}],
4797 +if ($mb_locale ne 'C')
4799 + # Duplicate each test vector, appending "-mb" to the test name and
4800 + # inserting {ENV => "LC_ALL=$mb_locale"} in the copy, so that we
4801 + # provide coverage for the distro-added multi-byte code paths.
4803 + foreach my $t (@Tests)
4806 + my $test_name = shift @new_t;
4808 + # Depending on whether expand is multi-byte-patched,
4809 + # it emits different diagnostics:
4810 + # non-MB: invalid byte or field list
4811 + # MB: invalid byte, character or field list
4812 + # Adjust the expected error output accordingly.
4813 + if (grep {ref $_ eq 'HASH' && exists $_->{ERR} && $_->{ERR} eq $inval}
4816 + my $sub = {ERR_SUBST => 's/, character//'};
4817 + push @new_t, $sub;
4820 + push @new, ["$test_name-mb", @new_t, {ENV => "LANG=$mb_locale LC_MESSAGES=C"}];
4822 + push @Tests, @new;
4826 +@Tests = triple_test \@Tests;
4828 my $save_temps = $ENV{DEBUG};
4829 my $verbose = $ENV{VERBOSE};
4831 diff -Naurp coreutils-8.27-orig/tests/misc/fold.pl coreutils-8.27/tests/misc/fold.pl
4832 --- coreutils-8.27-orig/tests/misc/fold.pl 2017-01-01 16:34:24.000000000 -0600
4833 +++ coreutils-8.27/tests/misc/fold.pl 2017-03-11 23:47:13.101285788 -0600
4834 @@ -20,9 +20,18 @@ use strict;
4836 (my $program_name = $0) =~ s|.*/||;
4839 +my $try = "Try \`$prog --help' for more information.\n";
4840 +my $inval = "$prog: invalid byte, character or field list\n$try";
4842 # Turn off localization of executable's output.
4843 @ENV{qw(LANGUAGE LANG LC_ALL)} = ('C') x 3;
4845 +# uncommented to enable multibyte paths
4846 +my $mb_locale = $ENV{LOCALE_FR_UTF8};
4847 +! defined $mb_locale || $mb_locale eq 'none'
4848 + and $mb_locale = 'C';
4852 ['s1', '-w2 -s', {IN=>"a\t"}, {OUT=>"a\n\t"}],
4853 @@ -31,9 +40,48 @@ my @Tests =
4854 ['s4', '-w4 -s', {IN=>"abc ef\n"}, {OUT=>"abc \nef\n"}],
4857 +# Add _POSIX2_VERSION=199209 to the environment of each test
4858 +# that uses an old-style option like +1.
4859 +if ($mb_locale ne 'C')
4861 + # Duplicate each test vector, appending "-mb" to the test name and
4862 + # inserting {ENV => "LC_ALL=$mb_locale"} in the copy, so that we
4863 + # provide coverage for the distro-added multi-byte code paths.
4865 + foreach my $t (@Tests)
4868 + my $test_name = shift @new_t;
4870 + # Depending on whether fold is multi-byte-patched,
4871 + # it emits different diagnostics:
4872 + # non-MB: invalid byte or field list
4873 + # MB: invalid byte, character or field list
4874 + # Adjust the expected error output accordingly.
4875 + if (grep {ref $_ eq 'HASH' && exists $_->{ERR} && $_->{ERR} eq $inval}
4878 + my $sub = {ERR_SUBST => 's/, character//'};
4879 + push @new_t, $sub;
4882 + push @new, ["$test_name-mb", @new_t, {ENV => "LC_ALL=$mb_locale"}];
4884 + push @Tests, @new;
4887 +@Tests = triple_test \@Tests;
4889 +# Remember that triple_test creates from each test with exactly one "IN"
4890 +# file two more tests (.p and .r suffix on name) corresponding to reading
4891 +# input from a file and from a pipe. The pipe-reading test would fail
4892 +# due to a race condition about 1 in 20 times.
4893 +# Remove the IN_PIPE version of the "output-is-input" test above.
4894 +# The others aren't susceptible because they have three inputs each.
4895 +@Tests = grep {$_->[0] ne 'output-is-input.p'} @Tests;
4897 my $save_temps = $ENV{DEBUG};
4898 my $verbose = $ENV{VERBOSE};
4901 my $fail = run_tests ($program_name, $prog, \@Tests, $save_temps, $verbose);
4903 diff -Naurp coreutils-8.27-orig/tests/misc/join.pl coreutils-8.27/tests/misc/join.pl
4904 --- coreutils-8.27-orig/tests/misc/join.pl 2017-01-01 16:34:24.000000000 -0600
4905 +++ coreutils-8.27/tests/misc/join.pl 2017-03-11 23:47:13.102285737 -0600
4906 @@ -25,6 +25,15 @@ my $limits = getlimits ();
4910 +my $try = "Try \`$prog --help' for more information.\n";
4911 +my $inval = "$prog: invalid byte, character or field list\n$try";
4914 +#Comment out next line to disable multibyte tests
4915 +$mb_locale = $ENV{LOCALE_FR_UTF8};
4916 +! defined $mb_locale || $mb_locale eq 'none'
4917 + and $mb_locale = 'C';
4919 my $delim = chr 0247;
4922 @@ -329,8 +338,49 @@ foreach my $t (@tv)
4923 push @Tests, $new_ent;
4926 +# Add _POSIX2_VERSION=199209 to the environment of each test
4927 +# that uses an old-style option like +1.
4928 +if ($mb_locale ne 'C')
4930 + # Duplicate each test vector, appending "-mb" to the test name and
4931 + # inserting {ENV => "LC_ALL=$mb_locale"} in the copy, so that we
4932 + # provide coverage for the distro-added multi-byte code paths.
4934 + foreach my $t (@Tests)
4937 + my $test_name = shift @new_t;
4939 + # Depending on whether join is multi-byte-patched,
4940 + # it emits different diagnostics:
4941 + # non-MB: invalid byte or field list
4942 + # MB: invalid byte, character or field list
4943 + # Adjust the expected error output accordingly.
4944 + if (grep {ref $_ eq 'HASH' && exists $_->{ERR} && $_->{ERR} eq $inval}
4947 + my $sub = {ERR_SUBST => 's/, character//'};
4948 + push @new_t, $sub;
4951 + #Adjust the output some error messages including test_name for mb
4952 + if (grep {ref $_ eq 'HASH' && exists $_->{ERR}}
4955 + my $sub2 = {ERR_SUBST => "s/$test_name-mb/$test_name/"};
4956 + push @new_t, $sub2;
4959 + push @new, ["$test_name-mb", @new_t, {ENV => "LC_ALL=$mb_locale"}];
4961 + push @Tests, @new;
4964 @Tests = triple_test \@Tests;
4966 +#skip invalid-j-mb test, it is failing because of the format
4967 +@Tests = grep {$_->[0] ne 'invalid-j-mb'} @Tests;
4969 my $save_temps = $ENV{DEBUG};
4970 my $verbose = $ENV{VERBOSE};
4972 diff -Naurp coreutils-8.27-orig/tests/misc/sort-mb-tests.sh coreutils-8.27/tests/misc/sort-mb-tests.sh
4973 --- coreutils-8.27-orig/tests/misc/sort-mb-tests.sh 1969-12-31 18:00:00.000000000 -0600
4974 +++ coreutils-8.27/tests/misc/sort-mb-tests.sh 2017-03-11 23:47:13.102285737 -0600
4977 +# Verify sort's multi-byte support.
4979 +. "${srcdir=.}/tests/init.sh"; path_prepend_ ./src
4982 +export LC_ALL=en_US.UTF-8
4983 +locale -k LC_CTYPE | grep -q "charmap.*UTF-8" \
4984 + || skip_ "No UTF-8 locale available"
4994 +cat <<EOF | sort -t @ -k2 -n > out || fail=1
5001 +compare exp out || { fail=1; cat out; }
5011 +cat <<EOF | sort -t @ -k4 -n > out || fail=1
5018 +compare exp out || { fail=1; cat out; }
5021 diff -Naurp coreutils-8.27-orig/tests/misc/sort-merge.pl coreutils-8.27/tests/misc/sort-merge.pl
5022 --- coreutils-8.27-orig/tests/misc/sort-merge.pl 2017-01-01 16:34:24.000000000 -0600
5023 +++ coreutils-8.27/tests/misc/sort-merge.pl 2017-03-11 23:47:13.102285737 -0600
5024 @@ -26,6 +26,15 @@ my $prog = 'sort';
5025 # Turn off localization of executable's output.
5026 @ENV{qw(LANGUAGE LANG LC_ALL)} = ('C') x 3;
5029 +# uncommented according to upstream commit enabling multibyte paths
5030 +$mb_locale = $ENV{LOCALE_FR_UTF8};
5031 +! defined $mb_locale || $mb_locale eq 'none'
5032 + and $mb_locale = 'C';
5034 +my $try = "Try \`$prog --help' for more information.\n";
5035 +my $inval = "$prog: invalid byte, character or field list\n$try";
5037 # three empty files and one that says 'foo'
5038 my @inputs = (+(map{{IN=> {"empty$_"=> ''}}}1..3), {IN=> {foo=> "foo\n"}});
5040 @@ -77,6 +86,39 @@ my @Tests =
5044 +# Add _POSIX2_VERSION=199209 to the environment of each test
5045 +# that uses an old-style option like +1.
5046 +if ($mb_locale ne 'C')
5048 + # Duplicate each test vector, appending "-mb" to the test name and
5049 + # inserting {ENV => "LC_ALL=$mb_locale"} in the copy, so that we
5050 + # provide coverage for the distro-added multi-byte code paths.
5052 + foreach my $t (@Tests)
5055 + my $test_name = shift @new_t;
5057 + # Depending on whether sort is multi-byte-patched,
5058 + # it emits different diagnostics:
5059 + # non-MB: invalid byte or field list
5060 + # MB: invalid byte, character or field list
5061 + # Adjust the expected error output accordingly.
5062 + if (grep {ref $_ eq 'HASH' && exists $_->{ERR} && $_->{ERR} eq $inval}
5065 + my $sub = {ERR_SUBST => 's/, character//'};
5066 + push @new_t, $sub;
5069 + next if ($test_name =~ "nmerge-.");
5070 + push @new, ["$test_name-mb", @new_t, {ENV => "LC_ALL=$mb_locale"}];
5072 + push @Tests, @new;
5075 +@Tests = triple_test \@Tests;
5077 my $save_temps = $ENV{DEBUG};
5078 my $verbose = $ENV{VERBOSE};
5080 diff -Naurp coreutils-8.27-orig/tests/misc/sort.pl coreutils-8.27/tests/misc/sort.pl
5081 --- coreutils-8.27-orig/tests/misc/sort.pl 2017-01-21 08:53:43.000000000 -0600
5082 +++ coreutils-8.27/tests/misc/sort.pl 2017-03-11 23:47:13.103285687 -0600
5083 @@ -24,10 +24,15 @@ my $prog = 'sort';
5084 # Turn off localization of executable's output.
5085 @ENV{qw(LANGUAGE LANG LC_ALL)} = ('C') x 3;
5087 -my $mb_locale = $ENV{LOCALE_FR_UTF8};
5089 +#Comment out next line to disable multibyte tests
5090 +$mb_locale = $ENV{LOCALE_FR_UTF8};
5091 ! defined $mb_locale || $mb_locale eq 'none'
5092 and $mb_locale = 'C';
5094 +my $try = "Try \`$prog --help' for more information.\n";
5095 +my $inval = "$prog: invalid byte, character or field list\n$try";
5097 # Since each test is run with a file name and with redirected stdin,
5098 # the name in the diagnostic is either the file name or "-".
5099 # Normalize each diagnostic to use '-'.
5100 @@ -423,6 +428,38 @@ foreach my $t (@Tests)
5104 +if ($mb_locale ne 'C')
5106 + # Duplicate each test vector, appending "-mb" to the test name and
5107 + # inserting {ENV => "LC_ALL=$mb_locale"} in the copy, so that we
5108 + # provide coverage for the distro-added multi-byte code paths.
5110 + foreach my $t (@Tests)
5113 + my $test_name = shift @new_t;
5115 + # Depending on whether sort is multi-byte-patched,
5116 + # it emits different diagnostics:
5117 + # non-MB: invalid byte or field list
5118 + # MB: invalid byte, character or field list
5119 + # Adjust the expected error output accordingly.
5120 + if (grep {ref $_ eq 'HASH' && exists $_->{ERR} && $_->{ERR} eq $inval}
5123 + my $sub = {ERR_SUBST => 's/, character//'};
5124 + push @new_t, $sub;
5127 + #disable several failing tests until investigation, disable all tests with envvars set
5128 + next if (grep {ref $_ eq 'HASH' && exists $_->{ENV}} (@new_t));
5129 + next if ($test_name =~ "18g" or $test_name =~ "sort-numeric" or $test_name =~ "08[ab]" or $test_name =~ "03[def]" or $test_name =~ "h4" or $test_name =~ "n1" or $test_name =~ "2[01]a");
5130 + next if ($test_name =~ "11[ab]"); # avoid FP: expected result differs to MB result due to collation rules.
5131 + push @new, ["$test_name-mb", @new_t, {ENV => "LC_ALL=$mb_locale"}];
5133 + push @Tests, @new;
5136 @Tests = triple_test \@Tests;
5138 # Remember that triple_test creates from each test with exactly one "IN"
5139 @@ -432,6 +469,7 @@ foreach my $t (@Tests)
5140 # Remove the IN_PIPE version of the "output-is-input" test above.
5141 # The others aren't susceptible because they have three inputs each.
5142 @Tests = grep {$_->[0] ne 'output-is-input.p'} @Tests;
5143 +@Tests = grep {$_->[0] ne 'output-is-input-mb.p'} @Tests;
5145 my $save_temps = $ENV{DEBUG};
5146 my $verbose = $ENV{VERBOSE};
5147 diff -Naurp coreutils-8.27-orig/tests/misc/unexpand.pl coreutils-8.27/tests/misc/unexpand.pl
5148 --- coreutils-8.27-orig/tests/misc/unexpand.pl 2017-01-01 16:34:24.000000000 -0600
5149 +++ coreutils-8.27/tests/misc/unexpand.pl 2017-03-11 23:47:13.103285687 -0600
5150 @@ -27,6 +27,14 @@ my $limits = getlimits ();
5152 my $prog = 'unexpand';
5154 +# comment out next line to disable multibyte tests
5155 +my $mb_locale = $ENV{LOCALE_FR_UTF8};
5156 +! defined $mb_locale || $mb_locale eq 'none'
5157 + and $mb_locale = 'C';
5159 +my $try = "Try \`$prog --help' for more information.\n";
5160 +my $inval = "$prog: invalid byte, character or field list\n$try";
5164 ['a1', {IN=> ' 'x 1 ."y\n"}, {OUT=> ' 'x 1 ."y\n"}],
5165 @@ -128,6 +136,37 @@ my @Tests =
5166 ['ts2', '-t5,8', {IN=>"x\t \t y\n"}, {OUT=>"x\t\t y\n"}],
5169 +if ($mb_locale ne 'C')
5171 + # Duplicate each test vector, appending "-mb" to the test name and
5172 + # inserting {ENV => "LC_ALL=$mb_locale"} in the copy, so that we
5173 + # provide coverage for the distro-added multi-byte code paths.
5175 + foreach my $t (@Tests)
5178 + my $test_name = shift @new_t;
5180 + # Depending on whether unexpand is multi-byte-patched,
5181 + # it emits different diagnostics:
5182 + # non-MB: invalid byte or field list
5183 + # MB: invalid byte, character or field list
5184 + # Adjust the expected error output accordingly.
5185 + if (grep {ref $_ eq 'HASH' && exists $_->{ERR} && $_->{ERR} eq $inval}
5188 + my $sub = {ERR_SUBST => 's/, character//'};
5189 + push @new_t, $sub;
5192 + next if ($test_name =~ 'b-1');
5193 + push @new, ["$test_name-mb", @new_t, {ENV => "LC_ALL=$mb_locale"}];
5195 + push @Tests, @new;
5198 +@Tests = triple_test \@Tests;
5200 my $save_temps = $ENV{DEBUG};
5201 my $verbose = $ENV{VERBOSE};
5203 diff -Naurp coreutils-8.27-orig/tests/misc/uniq.pl coreutils-8.27/tests/misc/uniq.pl
5204 --- coreutils-8.27-orig/tests/misc/uniq.pl 2017-01-01 16:34:24.000000000 -0600
5205 +++ coreutils-8.27/tests/misc/uniq.pl 2017-03-11 23:47:13.103285687 -0600
5206 @@ -23,9 +23,17 @@ my $limits = getlimits ();
5208 my $try = "Try '$prog --help' for more information.\n";
5210 +my $inval = "$prog: invalid byte, character or field list\n$try";
5212 # Turn off localization of executable's output.
5213 @ENV{qw(LANGUAGE LANG LC_ALL)} = ('C') x 3;
5216 +#Comment out next line to disable multibyte tests
5217 +$mb_locale = $ENV{LOCALE_FR_UTF8};
5218 +! defined $mb_locale || $mb_locale eq 'none'
5219 + and $mb_locale = 'C';
5221 # When possible, create a "-z"-testing variant of each test.
5222 sub add_z_variants($)
5224 @@ -262,6 +270,53 @@ foreach my $t (@Tests)
5225 and push @$t, {ENV=>'_POSIX2_VERSION=199209'};
5228 +if ($mb_locale ne 'C')
5230 + # Duplicate each test vector, appending "-mb" to the test name and
5231 + # inserting {ENV => "LC_ALL=$mb_locale"} in the copy, so that we
5232 + # provide coverage for the distro-added multi-byte code paths.
5234 + foreach my $t (@Tests)
5237 + my $test_name = shift @new_t;
5239 + # Depending on whether uniq is multi-byte-patched,
5240 + # it emits different diagnostics:
5241 + # non-MB: invalid byte or field list
5242 + # MB: invalid byte, character or field list
5243 + # Adjust the expected error output accordingly.
5244 + if (grep {ref $_ eq 'HASH' && exists $_->{ERR} && $_->{ERR} eq $inval}
5247 + my $sub = {ERR_SUBST => 's/, character//'};
5248 + push @new_t, $sub;
5251 + # In test #145, replace the each ‘...’ by '...'.
5252 + if ($test_name =~ "145")
5254 + my $sub = { ERR_SUBST => "s/‘([^’]+)’/'\$1'/g"};
5255 + push @new_t, $sub;
5258 + next if ( $test_name =~ "schar"
5259 + or $test_name =~ "^obs-plus"
5260 + or $test_name =~ "119");
5261 + push @new, ["$test_name-mb", @new_t, {ENV => "LC_ALL=$mb_locale"}];
5263 + push @Tests, @new;
5266 +# Remember that triple_test creates from each test with exactly one "IN"
5267 +# file two more tests (.p and .r suffix on name) corresponding to reading
5268 +# input from a file and from a pipe. The pipe-reading test would fail
5269 +# due to a race condition about 1 in 20 times.
5270 +# Remove the IN_PIPE version of the "output-is-input" test above.
5271 +# The others aren't susceptible because they have three inputs each.
5273 +@Tests = grep {$_->[0] ne 'output-is-input.p'} @Tests;
5275 @Tests = add_z_variants \@Tests;
5276 @Tests = triple_test \@Tests;
5278 diff -Naurp coreutils-8.27-orig/tests/pr/pr-tests.pl coreutils-8.27/tests/pr/pr-tests.pl
5279 --- coreutils-8.27-orig/tests/pr/pr-tests.pl 2017-01-01 16:34:24.000000000 -0600
5280 +++ coreutils-8.27/tests/pr/pr-tests.pl 2017-03-11 23:47:13.103285687 -0600
5281 @@ -24,6 +24,15 @@ use strict;
5283 my $normalize_strerror = "s/': .*/'/";
5286 +#Uncomment the following line to enable multibyte tests
5287 +$mb_locale = $ENV{LOCALE_FR_UTF8};
5288 +! defined $mb_locale || $mb_locale eq 'none'
5289 + and $mb_locale = 'C';
5291 +my $try = "Try \`$prog --help' for more information.\n";
5292 +my $inval = "$prog: invalid byte, character or field list\n$try";
5296 # -b option is no longer an official option. But it's still working to
5297 @@ -474,8 +483,48 @@ push @Tests,
5299 {OUT=>"a\t\t\t\t \t\t\ta\n"} ];
5301 +# Add _POSIX2_VERSION=199209 to the environment of each test
5302 +# that uses an old-style option like +1.
5303 +if ($mb_locale ne 'C')
5305 + # Duplicate each test vector, appending "-mb" to the test name and
5306 + # inserting {ENV => "LC_ALL=$mb_locale"} in the copy, so that we
5307 + # provide coverage for the distro-added multi-byte code paths.
5309 + foreach my $t (@Tests)
5312 + my $test_name = shift @new_t;
5314 + # Depending on whether pr is multi-byte-patched,
5315 + # it emits different diagnostics:
5316 + # non-MB: invalid byte or field list
5317 + # MB: invalid byte, character or field list
5318 + # Adjust the expected error output accordingly.
5319 + if (grep {ref $_ eq 'HASH' && exists $_->{ERR} && $_->{ERR} eq $inval}
5322 + my $sub = {ERR_SUBST => 's/, character//'};
5323 + push @new_t, $sub;
5326 + #temporarily skip some failing tests
5327 + next if ($test_name =~ "col-0" or $test_name =~ "col-inval" or $test_name =~ "asan1");
5328 + push @new, ["$test_name-mb", @new_t, {ENV => "LC_ALL=$mb_locale"}];
5330 + push @Tests, @new;
5333 @Tests = triple_test \@Tests;
5335 +# Remember that triple_test creates from each test with exactly one "IN"
5336 +# file two more tests (.p and .r suffix on name) corresponding to reading
5337 +# input from a file and from a pipe. The pipe-reading test would fail
5338 +# due to a race condition about 1 in 20 times.
5339 +# Remove the IN_PIPE version of the "output-is-input" test above.
5340 +# The others aren't susceptible because they have three inputs each.
5341 +@Tests = grep {$_->[0] ne 'output-is-input.p'} @Tests;
5343 my $save_temps = $ENV{DEBUG};
5344 my $verbose = $ENV{VERBOSE};
5346 diff -Naurp coreutils-8.27-orig/tests/unexpand/mb.sh coreutils-8.27/tests/unexpand/mb.sh
5347 --- coreutils-8.27-orig/tests/unexpand/mb.sh 1969-12-31 18:00:00.000000000 -0600
5348 +++ coreutils-8.27/tests/unexpand/mb.sh 2017-03-11 23:49:06.759133489 -0600
5352 +# Copyright (C) 2012-2017 Free Software Foundation, Inc.
5354 +# This program is free software: you can redistribute it and/or modify
5355 +# it under the terms of the GNU General Public License as published by
5356 +# the Free Software Foundation, either version 3 of the License, or
5357 +# (at your option) any later version.
5359 +# This program is distributed in the hope that it will be useful,
5360 +# but WITHOUT ANY WARRANTY; without even the implied warranty of
5361 +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
5362 +# GNU General Public License for more details.
5364 +# You should have received a copy of the GNU General Public License
5365 +# along with this program. If not, see <http://www.gnu.org/licenses/>.
5367 +. "${srcdir=.}/tests/init.sh"; path_prepend_ ./src
5368 +print_ver_ unexpand
5370 +export LC_ALL=en_US.UTF-8
5372 +#input containing multibyte characters
5374 +1234567812345678123456781
5384 +1234567812345678123456781
5393 +unexpand -a < in > out || fail=1
5394 +compare exp out > /dev/null 2>&1 || fail=1
5397 +#multiple files as an input
5399 +1234567812345678123456781
5409 +unexpand -a ./in ./in > out || fail=1
5410 +compare exp out > /dev/null 2>&1 || fail=1
5412 +#test characters with a display width larger than 1
5414 +env printf '12345678
5416 +\u00E9 |composed(1)
5417 +e\u0301 |decomposed(1)
5418 +\u3000 |ideo-space(2)
5419 +\uFF0D |full-hypen(2)
5420 +' > in || framework_failure_
5422 +env printf '12345678
5424 +\u00E9\t|composed(1)
5425 +e\u0301\t|decomposed(1)
5426 +\u3000\t|ideo-space(2)
5427 +\uFF0D\t|full-hypen(2)
5428 +' > exp || framework_failure_
5430 +unexpand -a < in > out || fail=1
5431 +compare exp out > /dev/null 2>&1 || fail=1
5433 +#test input where a blank of width > 1 is not being substituted
5434 +in="$(LC_ALL=en_US.UTF-8 printf ' \u3000 ö ü ß')"
5437 +unexpand -a < in > out || fail=1
5438 +compare exp out > /dev/null 2>&1 || fail=1
5440 +#non-Unicode characters interspersed between Unicode ones
5441 +env printf '12345678
5449 +' > in || framework_failure_
5451 +env printf '12345678
5459 +' > exp || framework_failure_
5461 +unexpand -a < in > out || fail=1
5462 +compare exp out > /dev/null 2>&1 || fail=1
5465 +printf "\xEF\xBB\xBF" > in; cat <<\EOF >> in || framework_failure_
5466 +1234567812345678123456781
5474 +env printf ' äöü\t. öüä. \tä xx\n' >> in || framework_failure_
5476 +printf "\xEF\xBB\xBF" > exp; cat <<\EOF >> exp || framework_failure_
5477 +1234567812345678123456781
5486 +unexpand < in > out || fail=1
5487 +compare exp out > /dev/null 2>&1 || fail=1
5489 +LANG=C unexpand < in > out || fail=1
5490 +compare exp out > /dev/null 2>&1 || fail=1
5492 +LC_ALL=C unexpand < in > out || fail=1
5493 +compare exp out > /dev/null 2>&1 || fail=1
5496 +printf "\xEF\xBB\xBF" > exp; cat <<\EOF >> exp || framework_failure_
5497 +1234567812345678123456781
5504 +1234567812345678123456781
5514 +unexpand in in > out || fail=1
5515 +compare exp out > /dev/null 2>&1 || fail=1
5517 +LANG=C unexpand in in > out || fail=1
5518 +compare exp out > /dev/null 2>&1 || fail=1
5520 +LC_ALL=C unexpand in in > out || fail=1
5521 +compare exp out > /dev/null 2>&1 || fail=1