iconv/iconv_charmap.c

   1 /* Convert using charmaps and possibly iconv().
   2    Copyright (C) 2001-2021 Free Software Foundation, Inc.
   3    This file is part of the GNU C Library.
   4    Contributed by Ulrich Drepper <drepper@redhat.com>, 2001.
   5
   6    This program is free software; you can redistribute it and/or modify
   7    it under the terms of the GNU General Public License as published
   8    by the Free Software Foundation; version 2 of the License, or
   9    (at your option) any later version.
  10
  11    This program is distributed in the hope that it will be useful,
  12    but WITHOUT ANY WARRANTY; without even the implied warranty of
  13    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14    GNU General Public License for more details.
  15
  16    You should have received a copy of the GNU General Public License
  17    along with this program; if not, see <https://www.gnu.org/licenses/>.  */
  18
  19 #include <assert.h>
  20 #include <errno.h>
  21 #include <error.h>
  22 #include <fcntl.h>
  23 #include <iconv.h>
  24 #include <libintl.h>
  25 #include <stdio.h>
  26 #include <stdlib.h>
  27 #include <unistd.h>
  28 #include <stdint.h>
  29 #include <sys/mman.h>
  30 #include <sys/stat.h>
  31
  32 #include "iconv_prog.h"
  33
  34
  35 /* Prototypes for a few program-wide used functions.  */
  36 #include <programs/xmalloc.h>
  37
  38
  39 struct convtable
  40 {
  41   int term[256 / 8];
  42   union
  43   {
  44     struct convtable *sub;
  45     struct charseq *out;
  46   } val[256];
  47 };
  48
  49
  50 static inline struct convtable *
  51 allocate_table (void)
  52 {
  53   return (struct convtable *) xcalloc (1, sizeof (struct convtable));
  54 }
  55
  56 static inline void
  57 free_table (struct convtable *tbl)
  58 {
  59   free (tbl);
  60 }
  61
  62
  63 static inline int
  64 is_term (struct convtable *tbl, unsigned int idx)
  65 {
  66   return tbl->term[idx / 8] & (1 << (idx % 8));
  67 }
  68
  69
  70 static inline void
  71 clear_term (struct convtable *tbl, unsigned int idx)
  72 {
  73   tbl->term[idx / 8] &= ~(1 << (idx % 8));
  74 }
  75
  76
  77 static inline void
  78 set_term (struct convtable *tbl, unsigned int idx)
  79 {
  80   tbl->term[idx / 8] |= 1 << (idx % 8);
  81 }
  82
  83
  84 /* Generate the conversion table.  */
  85 static struct convtable *use_from_charmap (struct charmap_t *from_charmap,
  86                                            const char *to_code);
  87 static struct convtable *use_to_charmap (const char *from_code,
  88                                          struct charmap_t *to_charmap);
  89 static struct convtable *use_both_charmaps (struct charmap_t *from_charmap,
  90                                             struct charmap_t *to_charmap);
  91
  92 /* Prototypes for the functions doing the actual work.  */
  93 static int process_block (struct convtable *tbl, char *addr, size_t len,
  94                           FILE *output);
  95 static int process_fd (struct convtable *tbl, int fd, FILE *output);
  96 static int process_file (struct convtable *tbl, FILE *input, FILE *output);
  97
  98
  99 int
 100 charmap_conversion (const char *from_code, struct charmap_t *from_charmap,
 101                     const char *to_code, struct charmap_t *to_charmap,
 102                     int argc, int remaining, char *argv[],
 103                     const char *output_file)
 104 {
 105   struct convtable *cvtbl;
 106   int status = EXIT_SUCCESS;
 107
 108   /* We have three different cases to handle:
 109
 110      - both, from_charmap and to_charmap, are available.  This means we
 111        can assume that the symbolic names match and use them to create
 112        the mapping.
 113
 114      - only from_charmap is available.  In this case we can only hope that
 115        the symbolic names used are of the <Uxxxx> form in which case we
 116        can use a UCS4->"to_code" iconv() conversion for the second step.
 117
 118      - only to_charmap is available.  This is similar, only that we would
 119        use iconv() for the "to_code"->UCS4 conversion.
 120
 121        We first create a table which maps input bytes into output bytes.
 122        Once this is done we can handle all three of the cases above
 123        equally.  */
 124   if (from_charmap != NULL)
 125     {
 126       if (to_charmap == NULL)
 127         cvtbl = use_from_charmap (from_charmap, to_code);
 128       else
 129         cvtbl = use_both_charmaps (from_charmap, to_charmap);
 130     }
 131   else
 132     {
 133       assert (to_charmap != NULL);
 134       cvtbl = use_to_charmap (from_code, to_charmap);
 135     }
 136
 137   /* If we couldn't generate a table stop now.  */
 138   if (cvtbl == NULL)
 139     return EXIT_FAILURE;
 140
 141   /* Determine output file.  */
 142   FILE *output;
 143   if (output_file != NULL && strcmp (output_file, "-") != 0)
 144     {
 145       output = fopen (output_file, "w");
 146       if (output == NULL)
 147         error (EXIT_FAILURE, errno, _("cannot open output file"));
 148     }
 149   else
 150     output = stdout;
 151
 152   /* We can now start the conversion.  */
 153   if (remaining == argc)
 154     {
 155       if (process_file (cvtbl, stdin, output) != 0)
 156         status = EXIT_FAILURE;
 157     }
 158   else
 159     do
 160       {
 161         int fd;
 162
 163         if (verbose)
 164           printf ("%s:\n", argv[remaining]);
 165         if (strcmp (argv[remaining], "-") == 0)
 166           fd = 0;
 167         else
 168           {
 169             fd = open (argv[remaining], O_RDONLY);
 170
 171             if (fd == -1)
 172               {
 173                 error (0, errno, _("cannot open input file `%s'"),
 174                        argv[remaining]);
 175                 status = EXIT_FAILURE;
 176                 continue;
 177               }
 178           }
 179
 180 #ifdef _POSIX_MAPPED_FILES
 181         struct stat64 st;
 182         char *addr;
 183         /* We have possibilities for reading the input file.  First try
 184            to mmap() it since this will provide the fastest solution.  */
 185         if (fstat64 (fd, &st) == 0
 186             && ((addr = mmap (NULL, st.st_size, PROT_READ, MAP_PRIVATE,
 187                               fd, 0)) != MAP_FAILED))
 188           {
 189             /* Yes, we can use mmap().  The descriptor is not needed
 190                anymore.  */
 191             if (close (fd) != 0)
 192               error (EXIT_FAILURE, errno,
 193                      _("error while closing input `%s'"), argv[remaining]);
 194
 195             if (process_block (cvtbl, addr, st.st_size, output) < 0)
 196               {
 197                 /* Something went wrong.  */
 198                 status = EXIT_FAILURE;
 199
 200                 /* We don't need the input data anymore.  */
 201                 munmap ((void *) addr, st.st_size);
 202
 203                 /* We cannot go on with producing output since it might
 204                    lead to problem because the last output might leave
 205                    the output stream in an undefined state.  */
 206                 break;
 207               }
 208
 209             /* We don't need the input data anymore.  */
 210             munmap ((void *) addr, st.st_size);
 211           }
 212         else
 213 #endif  /* _POSIX_MAPPED_FILES */
 214           {
 215             /* Read the file in pieces.  */
 216             if (process_fd (cvtbl, fd, output) != 0)
 217               {
 218                 /* Something went wrong.  */
 219                 status = EXIT_FAILURE;
 220
 221                 /* We don't need the input file anymore.  */
 222                 close (fd);
 223
 224                 /* We cannot go on with producing output since it might
 225                    lead to problem because the last output might leave
 226                    the output stream in an undefined state.  */
 227                 break;
 228               }
 229
 230             /* Now close the file.  */
 231             close (fd);
 232           }
 233       }
 234     while (++remaining < argc);
 235
 236   /* All done.  */
 237   free_table (cvtbl);
 238   return status;
 239 }
 240
 241
 242 /* Add the IN->OUT mapping to TBL.  OUT is potentially stored in the table.
 243    IN is used only here, so it need not be kept live afterwards.  */
 244 static void
 245 add_bytes (struct convtable *tbl, const struct charseq *in, struct charseq *out)
 246 {
 247   int n = 0;
 248   unsigned int byte;
 249
 250   assert (in->nbytes > 0);
 251
 252   byte = ((unsigned char *) in->bytes)[n];
 253   while (n + 1 < in->nbytes)
 254     {
 255       if (is_term (tbl, byte) || tbl->val[byte].sub == NULL)
 256         {
 257           /* Note that we simply ignore a definition for a byte sequence
 258              which is also the prefix for a longer one.  */
 259           clear_term (tbl, byte);
 260           tbl->val[byte].sub =
 261             (struct convtable *) xcalloc (1, sizeof (struct convtable));
 262         }
 263
 264       tbl = tbl->val[byte].sub;
 265
 266       byte = ((unsigned char *) in->bytes)[++n];
 267     }
 268
 269   /* Only add the new sequence if there is none yet and the byte sequence
 270      is not part of an even longer one.  */
 271   if (! is_term (tbl, byte) && tbl->val[byte].sub == NULL)
 272     {
 273       set_term (tbl, byte);
 274       tbl->val[byte].out = out;
 275     }
 276 }
 277
 278 /* Try to convert SEQ from WCHAR_T format using CD.
 279    Returns a malloc'd struct or NULL.  */
 280 static struct charseq *
 281 convert_charseq (iconv_t cd, const struct charseq *seq)
 282 {
 283   struct charseq *result = NULL;
 284
 285   if (seq->ucs4 != UNINITIALIZED_CHAR_VALUE)
 286     {
 287       /* There is a chance.  Try the iconv module.  */
 288       wchar_t inbuf[1] = { seq->ucs4 };
 289       unsigned char outbuf[64];
 290       char *inptr = (char *) inbuf;
 291       size_t inlen = sizeof (inbuf);
 292       char *outptr = (char *) outbuf;
 293       size_t outlen = sizeof (outbuf);
 294
 295       (void) iconv (cd, &inptr, &inlen, &outptr, &outlen);
 296
 297       if (outptr != (char *) outbuf)
 298         {
 299           /* We got some output.  Good, use it.  */
 300           outlen = sizeof (outbuf) - outlen;
 301           assert ((char *) outbuf + outlen == outptr);
 302
 303           result = xmalloc (sizeof (struct charseq) + outlen);
 304           result->name = seq->name;
 305           result->ucs4 = seq->ucs4;
 306           result->nbytes = outlen;
 307           memcpy (result->bytes, outbuf, outlen);
 308         }
 309
 310       /* Clear any possible state left behind.  */
 311       (void) iconv (cd, NULL, NULL, NULL, NULL);
 312     }
 313
 314   return result;
 315 }
 316
 317
 318 static struct convtable *
 319 use_from_charmap (struct charmap_t *from_charmap, const char *to_code)
 320 {
 321   /* We iterate over all entries in the from_charmap and for those which
 322      have a known UCS4 representation we use an iconv() call to determine
 323      the mapping to the to_code charset.  */
 324   struct convtable *rettbl;
 325   iconv_t cd;
 326   void *ptr = NULL;
 327   const void *key;
 328   size_t keylen;
 329   void *data;
 330
 331   cd = iconv_open (to_code, "WCHAR_T");
 332   if (cd == (iconv_t) -1)
 333     /* We cannot do anything.  */
 334     return NULL;
 335
 336   rettbl = allocate_table ();
 337
 338   while (iterate_table (&from_charmap->char_table, &ptr, &key, &keylen, &data)
 339          >= 0)
 340     {
 341       struct charseq *in = data;
 342       struct charseq *newp = convert_charseq (cd, in);
 343       if (newp != NULL)
 344         add_bytes (rettbl, in, newp);
 345     }
 346
 347   iconv_close (cd);
 348
 349   return rettbl;
 350 }
 351
 352
 353 static struct convtable *
 354 use_to_charmap (const char *from_code, struct charmap_t *to_charmap)
 355 {
 356   /* We iterate over all entries in the to_charmap and for those which
 357      have a known UCS4 representation we use an iconv() call to determine
 358      the mapping to the from_code charset.  */
 359   struct convtable *rettbl;
 360   iconv_t cd;
 361   void *ptr = NULL;
 362   const void *key;
 363   size_t keylen;
 364   void *data;
 365
 366   /* Note that the conversion we use here is the reverse direction.  Without
 367      exhaustive search we cannot figure out which input yields the UCS4
 368      character we are looking for.  Therefore we determine it the other
 369      way round.  */
 370   cd = iconv_open (from_code, "WCHAR_T");
 371   if (cd == (iconv_t) -1)
 372     /* We cannot do anything.  */
 373     return NULL;
 374
 375   rettbl = allocate_table ();
 376
 377   while (iterate_table (&to_charmap->char_table, &ptr, &key, &keylen, &data)
 378          >= 0)
 379     {
 380       struct charseq *out = data;
 381       struct charseq *newp = convert_charseq (cd, out);
 382       if (newp != NULL)
 383         {
 384           add_bytes (rettbl, newp, out);
 385           free (newp);
 386         }
 387     }
 388
 389   iconv_close (cd);
 390
 391   return rettbl;
 392 }
 393
 394
 395 static struct convtable *
 396 use_both_charmaps (struct charmap_t *from_charmap,
 397                    struct charmap_t *to_charmap)
 398 {
 399   /* In this case we iterate over all the entries in the from_charmap,
 400      determine the internal name, and find an appropriate entry in the
 401      to_charmap (if it exists).  */
 402   struct convtable *rettbl = allocate_table ();
 403   void *ptr = NULL;
 404   const void *key;
 405   size_t keylen;
 406   void *data;
 407
 408   while (iterate_table (&from_charmap->char_table, &ptr, &key, &keylen, &data)
 409          >= 0)
 410     {
 411       struct charseq *in = (struct charseq *) data;
 412       struct charseq *out = charmap_find_value (to_charmap, key, keylen);
 413
 414       if (out != NULL)
 415         add_bytes (rettbl, in, out);
 416     }
 417
 418   return rettbl;
 419 }
 420
 421
 422 static int
 423 process_block (struct convtable *tbl, char *addr, size_t len, FILE *output)
 424 {
 425   size_t n = 0;
 426
 427   while (n < len)
 428     {
 429       struct convtable *cur = tbl;
 430       unsigned char *curp = (unsigned char *) addr;
 431       unsigned int byte = *curp;
 432       int cnt;
 433       struct charseq *out;
 434
 435       while (! is_term (cur, byte))
 436         if (cur->val[byte].sub == NULL)
 437           {
 438             /* This is an invalid sequence.  Skip the first byte if we are
 439                ignoring errors.  Otherwise punt.  */
 440             if (! omit_invalid)
 441               {
 442                 error (0, 0, _("illegal input sequence at position %Zd"), n);
 443                 return -1;
 444               }
 445
 446             n -= curp - (unsigned char *) addr;
 447
 448             byte = *(curp = (unsigned char *) ++addr);
 449             if (++n >= len)
 450               /* All converted.  */
 451               return 0;
 452
 453             cur = tbl;
 454           }
 455         else
 456           {
 457             cur = cur->val[byte].sub;
 458
 459             if (++n >= len)
 460               {
 461                 error (0, 0, _("\
 462 incomplete character or shift sequence at end of buffer"));
 463                 return -1;
 464               }
 465
 466             byte = *++curp;
 467           }
 468
 469       /* We found a final byte.  Write the output bytes.  */
 470       out = cur->val[byte].out;
 471       for (cnt = 0; cnt < out->nbytes; ++cnt)
 472         fputc_unlocked (out->bytes[cnt], output);
 473
 474       addr = (char *) curp + 1;
 475       ++n;
 476     }
 477
 478   return 0;
 479 }
 480
 481
 482 static int
 483 process_fd (struct convtable *tbl, int fd, FILE *output)
 484 {
 485   /* We have a problem with reading from a descriptor since we must not
 486      provide the iconv() function an incomplete character or shift
 487      sequence at the end of the buffer.  Since we have to deal with
 488      arbitrary encodings we must read the whole text in a buffer and
 489      process it in one step.  */
 490   static char *inbuf = NULL;
 491   static size_t maxlen = 0;
 492   char *inptr = inbuf;
 493   size_t actlen = 0;
 494
 495   while (actlen < maxlen)
 496     {
 497       ssize_t n = read (fd, inptr, maxlen - actlen);
 498
 499       if (n == 0)
 500         /* No more text to read.  */
 501         break;
 502
 503       if (n == -1)
 504         {
 505           /* Error while reading.  */
 506           error (0, errno, _("error while reading the input"));
 507           return -1;
 508         }
 509
 510       inptr += n;
 511       actlen += n;
 512     }
 513
 514   if (actlen == maxlen)
 515     while (1)
 516       {
 517         ssize_t n;
 518         char *new_inbuf;
 519
 520         /* Increase the buffer.  */
 521         new_inbuf = (char *) realloc (inbuf, maxlen + 32768);
 522         if (new_inbuf == NULL)
 523           {
 524             error (0, errno, _("unable to allocate buffer for input"));
 525             return -1;
 526           }
 527         inbuf = new_inbuf;
 528         maxlen += 32768;
 529         inptr = inbuf + actlen;
 530
 531         do
 532           {
 533             n = read (fd, inptr, maxlen - actlen);
 534
 535             if (n == 0)
 536               /* No more text to read.  */
 537               break;
 538
 539             if (n == -1)
 540               {
 541                 /* Error while reading.  */
 542                 error (0, errno, _("error while reading the input"));
 543                 return -1;
 544               }
 545
 546             inptr += n;
 547             actlen += n;
 548           }
 549         while (actlen < maxlen);
 550
 551         if (n == 0)
 552           /* Break again so we leave both loops.  */
 553           break;
 554       }
 555
 556   /* Now we have all the input in the buffer.  Process it in one run.  */
 557   return process_block (tbl, inbuf, actlen, output);
 558 }
 559
 560
 561 static int
 562 process_file (struct convtable *tbl, FILE *input, FILE *output)
 563 {
 564   /* This should be safe since we use this function only for `stdin' and
 565      we haven't read anything so far.  */
 566   return process_fd (tbl, fileno (input), output);
 567 }