]> git.ipfire.org Git - thirdparty/glibc.git/commitdiff
Have iconv accept redundant escape sequences in IBM900, IBM903, IBM905,
authorMartin Sebor <msebor@redhat.com>
Fri, 15 Jan 2016 18:25:13 +0000 (11:25 -0700)
committerMartin Sebor <msebor@redhat.com>
Fri, 15 Jan 2016 18:25:13 +0000 (11:25 -0700)
IBM907, and IBM909.

Patch for bug #17197 changes the encoder to avoid generating redundant
shift sequences.  However, those sequences may already be present in
data encododed by prior versions of the encoder.  This change modifies
the decoder to also avoid rejecting redundant shift sequences.

        [BZ #19432]
        * iconvdata/Makefile: Add bug-iconv11.
        * iconvdata/bug-iconv11.c: New test.
        * iconvdata/ibm930.c: Do not reject redundant shift sequences.
        * iconvdata/ibm933.c: Same.
        * iconvdata/ibm935.c: Same.
        * iconvdata/ibm937.c: Same.
        * iconvdata/ibm939.c: Same.

ChangeLog
iconvdata/Makefile
iconvdata/bug-iconv11.c [new file with mode: 0644]
iconvdata/ibm930.c
iconvdata/ibm933.c
iconvdata/ibm935.c
iconvdata/ibm937.c
iconvdata/ibm939.c

index a9d575f2cbca529be5548bd99f995c9d595153ab..7f2a19d50ab764cf9182932538103b8f438ea796 100644 (file)
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,14 @@
+2016-01-15  Martin Sebor  <msebor@redhat.com>
+
+       [BZ #19432]
+       * iconvdata/Makefile: Add bug-iconv11.
+       * iconvdata/bug-iconv11.c: New test.
+       * iconvdata/ibm930.c: Do not reject redundant shift sequences.
+       * iconvdata/ibm933.c: Same.
+       * iconvdata/ibm935.c: Same.
+       * iconvdata/ibm937.c: Same.
+       * iconvdata/ibm939.c: Same.
+
 2016-01-15  Martin Sebor  <msebor@redhat.com>
 
        [BZ #19443]
index ae5dde60b433c5081666d36bbf0432900e59d72f..357530b55876396cb6676d5f5e89bdd813a9bc5d 100644 (file)
@@ -68,7 +68,7 @@ modules.so := $(addsuffix .so, $(modules))
 ifeq (yes,$(build-shared))
 tests = bug-iconv1 bug-iconv2 tst-loading tst-e2big tst-iconv4 bug-iconv4 \
        tst-iconv6 bug-iconv5 bug-iconv6 tst-iconv7 bug-iconv8 bug-iconv9 \
-       bug-iconv10
+       bug-iconv10 bug-iconv11
 ifeq ($(have-thread-library),yes)
 tests += bug-iconv3
 endif
diff --git a/iconvdata/bug-iconv11.c b/iconvdata/bug-iconv11.c
new file mode 100644 (file)
index 0000000..6cdc07d
--- /dev/null
@@ -0,0 +1,114 @@
+/* bug 19432: iconv rejects redundant escape sequences in IBM903,
+              IBM905, IBM907, and IBM909
+
+   Copyright (C) 2016 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, see
+   <http://www.gnu.org/licenses/>.  */
+
+#include <iconv.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <errno.h>
+
+// The longest test input sequence.
+#define MAXINBYTES    8
+#define MAXOUTBYTES   (MAXINBYTES * MB_LEN_MAX)
+
+/* Verify that a conversion of the INPUT sequence consisting of
+   INBYTESLEFT bytes in the encoding specified by the codeset
+   named by FROM_SET is successful.
+   Return 0 on success, non-zero on iconv() failure.  */
+
+static int
+test_ibm93x (const char *from_set, const char *input, size_t inbytesleft)
+{
+  const char to_set[] = "UTF-8";
+  iconv_t cd = iconv_open (to_set, from_set);
+  if (cd == (iconv_t) -1)
+    {
+      printf ("iconv_open(\"%s\", \"%s\"): %s\n",
+              from_set, to_set, strerror (errno));
+      return 1;
+    }
+
+  char output [MAXOUTBYTES];
+  size_t outbytesleft = sizeof output;
+
+  char *inbuf = (char*)input;
+  char *outbuf = output;
+
+  printf ("iconv(cd, %p, %zu, %p, %zu)\n",
+          inbuf, inbytesleft, outbuf, outbytesleft);
+
+  errno = 0;
+  size_t ret = iconv (cd, &inbuf, &inbytesleft, &outbuf, &outbytesleft);
+  printf ("  ==> %td: %s\n"
+          "  inbuf%+td, inbytesleft=%zu, outbuf%+td, outbytesleft=%zu\n",
+          ret, strerror (errno),
+          inbuf - input, inbytesleft, outbuf - output, outbytesleft);
+
+  // Return 0 on success, non-zero on iconv() failure.
+  return ret == (size_t)-1 || errno;
+}
+
+static int
+do_test (void)
+{
+  // State-dependent encodings to exercise.
+  static const char* const to_code[] = {
+    "IBM930", "IBM933", "IBM935", "IBM937", "IBM939"
+  };
+
+  static const size_t ncodesets = sizeof to_code / sizeof *to_code;
+
+  static const struct {
+    char txt[MAXINBYTES];
+    size_t len;
+  } input[] = {
+#define DATA(s) { s, sizeof s - 1 }
+    /* <SI>: denotes the shift-in 1-byte escape sequence, changing
+             the encoder from a sigle-byte encoding to multibyte
+       <SO>: denotes the shift-out 1-byte escape sequence, switching
+             the encoder from a multibyte to a single-byte state  */
+
+    DATA ("\x0e"),               // <SI> (not redundant)
+    DATA ("\x0f"),               // <S0> (redundant with initial state)
+    DATA ("\x0e\x0e"),           // <SI><SI>
+    DATA ("\x0e\x0f\x0f"),       // <SI><SO><SO>
+    DATA ("\x0f\x0f"),           // <SO><SO>
+    DATA ("\x0f\x0e\x0e"),       // <SO><SI><SI>
+    DATA ("\x0e\x0f\xc7\x0f"),   // <SI><SO><G><SO>
+    DATA ("\xc7\x0f")            // <G><SO> (redundant with initial state)
+  };
+
+  static const size_t ninputs = sizeof input / sizeof *input;
+
+  int ret = 0;
+
+  size_t i, j;
+
+  /* Iterate over the IBM93x codesets above and exercise each with
+     the input sequences above.  */
+  for (i = 0; i != ncodesets; ++i)
+    for (j = 0; j != ninputs; ++j)
+      ret += test_ibm93x (to_code [i], input [i].txt, input [i].len);
+
+  return ret;
+}
+
+#define TEST_FUNCTION do_test ()
+#include "../test-skeleton.c"
index 1f60962d86f08014e90ca3f1e9734a703e346714..9d35734b3ae6b92c0a80cd862306ca3be4ce6c7c 100644 (file)
@@ -110,24 +110,14 @@ enum
                                                                              \
     if (__builtin_expect (ch, 0) == SO)                                              \
       {                                                                              \
-       /* Shift OUT, change to DBCS converter.  */                           \
-       if (curcs == db)                                                      \
-         {                                                                   \
-           result = __GCONV_ILLEGAL_INPUT;                                   \
-           break;                                                            \
-         }                                                                   \
+       /* Shift OUT, change to DBCS converter (redundant escape okay).  */   \
        curcs = db;                                                           \
        ++inptr;                                                              \
        continue;                                                             \
       }                                                                              \
     else if (__builtin_expect (ch, 0) == SI)                                 \
       {                                                                              \
-       /* Shift IN, change to SBCS converter */                              \
-       if (curcs == sb)                                                      \
-         {                                                                   \
-           result = __GCONV_ILLEGAL_INPUT;                                   \
-           break;                                                            \
-         }                                                                   \
+       /* Shift IN, change to SBCS converter (redundant escape okay).  */    \
        curcs = sb;                                                           \
        ++inptr;                                                              \
        continue;                                                             \
index c984cd1733245388a1e3428bf8efb15fd92bf6f4..669e3570665889e4c052e882e83b0652bf46f431 100644 (file)
@@ -108,24 +108,14 @@ enum
                                                                              \
     if (__builtin_expect (ch, 0) == SO)                                              \
       {                                                                              \
-       /* Shift OUT, change to DBCS converter.  */                           \
-       if (curcs == db)                                                      \
-         {                                                                   \
-           result = __GCONV_ILLEGAL_INPUT;                                   \
-           break;                                                            \
-         }                                                                   \
+       /* Shift OUT, change to DBCS converter (redundant escape okay).  */   \
        curcs = db;                                                           \
        ++inptr;                                                              \
        continue;                                                             \
       }                                                                              \
     else if (__builtin_expect (ch, 0) == SI)                                 \
       {                                                                              \
-       /* Shift IN, change to SBCS converter.  */                            \
-       if (curcs == sb)                                                      \
-         {                                                                   \
-           result = __GCONV_ILLEGAL_INPUT;                                   \
-           break;                                                            \
-         }                                                                   \
+       /* Shift IN, change to SBCS converter (redundant escape okay).  */    \
        curcs = sb;                                                           \
        ++inptr;                                                              \
        continue;                                                             \
index 07f4bf255899e88a0abee1d012b160db5dab590c..2d3065e54cdeb2050bbaf0c2f421d511c7866ff8 100644 (file)
@@ -109,24 +109,14 @@ enum
                                                                              \
     if (__builtin_expect(ch, 0) == SO)                                       \
       {                                                                              \
-       /* Shift OUT, change to DBCS converter.  */                           \
-       if (curcs == db)                                                      \
-         {                                                                   \
-           result = __GCONV_ILLEGAL_INPUT;                                   \
-           break;                                                            \
-         }                                                                   \
+       /* Shift OUT, change to DBCS converter (redundant escape okay).  */   \
        curcs = db;                                                           \
        ++inptr;                                                              \
        continue;                                                             \
       }                                                                              \
     else if (__builtin_expect (ch, 0) == SI)                                 \
       {                                                                              \
-       /* Shift IN, change to SBCS converter.  */                            \
-       if (curcs == sb)                                                      \
-         {                                                                   \
-           result = __GCONV_ILLEGAL_INPUT;                                   \
-           break;                                                            \
-         }                                                                   \
+       /* Shift IN, change to SBCS converter (redundant escape okay).  */    \
        curcs = sb;                                                           \
        ++inptr;                                                              \
        continue;                                                             \
index 88344a445b158f3c3ceff8e8e36c8e51706a51f6..3ed647936672039a5ebc749632481628031b9f1a 100644 (file)
@@ -109,24 +109,14 @@ enum
                                                                              \
     if (__builtin_expect (ch, 0) == SO)                                              \
       {                                                                              \
-       /* Shift OUT, change to DBCS converter.  */                           \
-       if (curcs == db)                                                      \
-         {                                                                   \
-           result = __GCONV_ILLEGAL_INPUT;                                   \
-           break;                                                            \
-         }                                                                   \
+       /* Shift OUT, change to DBCS converter (redundant escape okay).  */   \
        curcs = db;                                                           \
        ++inptr;                                                              \
        continue;                                                             \
       }                                                                              \
     else if (__builtin_expect (ch, 0) == SI)                                 \
       {                                                                              \
-       /* Shift IN, change to SBCS converter.  */                            \
-       if (curcs == sb)                                                      \
-         {                                                                   \
-           result = __GCONV_ILLEGAL_INPUT;                                   \
-           break;                                                            \
-         }                                                                   \
+       /* Shift IN, change to SBCS converter (redundant escape okay).  */    \
        curcs = sb;                                                           \
        ++inptr;                                                              \
        continue;                                                             \
index dbdda628a3d5437a2a102fd83cc6af66c2e3acdc..c0a75f7797bb081928e35167cbac1a8c0230ad58 100644 (file)
@@ -109,24 +109,14 @@ enum
                                                                              \
     if (__builtin_expect (ch, 0) == SO)                                              \
       {                                                                              \
-       /* Shift OUT, change to DBCS converter.  */                           \
-       if (curcs == db)                                                      \
-         {                                                                   \
-           result = __GCONV_ILLEGAL_INPUT;                                   \
-           break;                                                            \
-         }                                                                   \
+       /* Shift OUT, change to DBCS converter (redundant escape okay).  */   \
        curcs = db;                                                           \
        ++inptr;                                                              \
        continue;                                                             \
       }                                                                              \
     else if (__builtin_expect (ch, 0) == SI)                                 \
       {                                                                              \
-       /* Shift IN, change to SBCS converter.  */                            \
-       if (curcs == sb)                                                      \
-         {                                                                   \
-           result = __GCONV_ILLEGAL_INPUT;                                   \
-           break;                                                            \
-         }                                                                   \
+       /* Shift IN, change to SBCS converter (redundant escape okay).  */    \
        curcs = sb;                                                           \
        ++inptr;                                                              \
        continue;                                                             \