localedata/unicode-gen/gen_translit_fraction.py

   1 #!/usr/bin/python3
   2 # -*- coding: utf-8 -*-
   3 #
   4 # Generate a translit_fraction file from a UnicodeData file.
   5 # Copyright (C) 2015-2019 Free Software Foundation, Inc.
   6 # This file is part of the GNU C Library.
   7 #
   8 # The GNU C Library is free software; you can redistribute it and/or
   9 # modify it under the terms of the GNU Lesser General Public
  10 # License as published by the Free Software Foundation; either
  11 # version 2.1 of the License, or (at your option) any later version.
  12 #
  13 # The GNU C Library is distributed in the hope that it will be useful,
  14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  16 # Lesser General Public License for more details.
  17 #
  18 # You should have received a copy of the GNU Lesser General Public
  19 # License along with the GNU C Library; if not, see
  20 # <http://www.gnu.org/licenses/>.
  21
  22 '''
  23 Generate a translit_fraction file from UnicodeData.txt
  24
  25 To see how this script is used, call it with the “-h” option:
  26
  27     $ ./gen_translit_fraction -h
  28     … prints usage message …
  29 '''
  30
  31 import argparse
  32 import time
  33 import unicode_utils
  34
  35 def read_input_file(filename):
  36     '''Reads the original glibc translit_fraction file to get the
  37     original head and tail.
  38
  39     We want to replace only the part of the file between
  40     “translit_start” and “translit_end”
  41     '''
  42     head = tail = ''
  43     with open(filename, mode='r') as translit_file:
  44         for line in translit_file:
  45             head = head + line
  46             if line.startswith('translit_start'):
  47                 break
  48         for line in translit_file:
  49             if line.startswith('translit_end'):
  50                 tail = line
  51                 break
  52         for line in translit_file:
  53             tail = tail + line
  54     return (head, tail)
  55
  56 def output_head(translit_file, unicode_version, head=''):
  57     '''Write the header of the output file, i.e. the part of the file
  58     before the “translit_start” line.
  59     '''
  60     if ARGS.input_file and head:
  61         translit_file.write(head)
  62     else:
  63         translit_file.write('escape_char /\n')
  64         translit_file.write('comment_char %\n')
  65         translit_file.write(unicode_utils.COMMENT_HEADER)
  66         translit_file.write('\n')
  67         translit_file.write('% Transliterations of fractions.\n')
  68         translit_file.write('% Generated automatically from UnicodeData.txt '
  69                             + 'by gen_translit_fraction.py '
  70                             + 'on {:s} '.format(time.strftime('%Y-%m-%d'))
  71                             + 'for Unicode {:s}.\n'.format(unicode_version))
  72         translit_file.write('% The replacements have been surrounded ')
  73         translit_file.write('with spaces, because fractions are\n')
  74         translit_file.write('% often preceded by a decimal number and ')
  75         translit_file.write('followed by a unit or a math symbol.\n')
  76         translit_file.write('\n')
  77         translit_file.write('LC_CTYPE\n')
  78         translit_file.write('\n')
  79         translit_file.write('translit_start\n')
  80
  81 def output_tail(translit_file, tail=''):
  82     '''Write the tail of the output file'''
  83     if ARGS.input_file and tail:
  84         translit_file.write(tail)
  85     else:
  86         translit_file.write('translit_end\n')
  87         translit_file.write('\n')
  88         translit_file.write('END LC_CTYPE\n')
  89
  90 def special_decompose(code_point_list):
  91     '''
  92     Decompositions which are not in UnicodeData.txt at all but which
  93     were used in the original translit_fraction file in glibc and
  94     which seem to make sense.  I want to keep the update of
  95     translit_fraction close to the spirit of the original file,
  96     therefore I added this special decomposition rules here.
  97     '''
  98     special_decompose_dict = {
  99         (0x2044,): [0x002F], # ⁄ → /
 100     }
 101     if tuple(code_point_list) in special_decompose_dict:
 102         return special_decompose_dict[tuple(code_point_list)]
 103     else:
 104         return code_point_list
 105
 106 def output_transliteration(translit_file):
 107     '''Write the new transliteration to the output file'''
 108     translit_file.write('\n')
 109     for code_point in sorted(unicode_utils.UNICODE_ATTRIBUTES):
 110         name = unicode_utils.UNICODE_ATTRIBUTES[code_point]['name']
 111         decomposition = unicode_utils.UNICODE_ATTRIBUTES[
 112             code_point]['decomposition']
 113         if decomposition.startswith('<fraction>'):
 114             decomposition = decomposition[11:]
 115             decomposed_code_points = [[int(x, 16)
 116                                        for x in decomposition.split(' ')]]
 117             if decomposed_code_points[0]:
 118                 decomposed_code_points[0] = [0x0020] \
 119                                             + decomposed_code_points[0] \
 120                                             + [0x0020]
 121                 while True:
 122                     special_decomposed_code_points = special_decompose(
 123                         decomposed_code_points[-1])
 124                     if (special_decomposed_code_points
 125                             != decomposed_code_points[-1]):
 126                         decomposed_code_points.append(
 127                             special_decomposed_code_points)
 128                         continue
 129                     special_decomposed_code_points = []
 130                     for decomposed_code_point in decomposed_code_points[-1]:
 131                         special_decomposed_code_points += special_decompose(
 132                             [decomposed_code_point])
 133                     if (special_decomposed_code_points
 134                             == decomposed_code_points[-1]):
 135                         break
 136                     decomposed_code_points.append(
 137                         special_decomposed_code_points)
 138                 translit_file.write('% {:s}\n'.format(name))
 139                 translit_file.write('{:s} '.format(
 140                     unicode_utils.ucs_symbol(code_point)))
 141                 for index in range(0, len(decomposed_code_points)):
 142                     if index > 0:
 143                         translit_file.write(';')
 144                     if len(decomposed_code_points[index]) > 1:
 145                         translit_file.write('"')
 146                     for decomposed_code_point in decomposed_code_points[index]:
 147                         translit_file.write('{:s}'.format(
 148                             unicode_utils.ucs_symbol(decomposed_code_point)))
 149                     if len(decomposed_code_points[index]) > 1:
 150                         translit_file.write('"')
 151                 translit_file.write('\n')
 152     translit_file.write('\n')
 153
 154 if __name__ == "__main__":
 155     PARSER = argparse.ArgumentParser(
 156         description='''
 157         Generate a translit_cjk_compat file from UnicodeData.txt.
 158         ''')
 159     PARSER.add_argument(
 160         '-u', '--unicode_data_file',
 161         nargs='?',
 162         type=str,
 163         default='UnicodeData.txt',
 164         help=('The UnicodeData.txt file to read, '
 165               + 'default: %(default)s'))
 166     PARSER.add_argument(
 167         '-i', '--input_file',
 168         nargs='?',
 169         type=str,
 170         help=''' The original glibc/localedata/locales/translit_fraction
 171         file.''')
 172     PARSER.add_argument(
 173         '-o', '--output_file',
 174         nargs='?',
 175         type=str,
 176         default='translit_fraction.new',
 177         help='''The new translit_fraction file, default: %(default)s.  If the
 178         original glibc/localedata/locales/translit_fraction file has
 179         been given as an option, the header up to the
 180         “translit_start” line and the tail from the “translit_end”
 181         line to the end of the file will be copied unchanged into the
 182         output file.  ''')
 183     PARSER.add_argument(
 184         '--unicode_version',
 185         nargs='?',
 186         required=True,
 187         type=str,
 188         help='The Unicode version of the input files used.')
 189     ARGS = PARSER.parse_args()
 190
 191     unicode_utils.fill_attributes(ARGS.unicode_data_file)
 192     HEAD = TAIL = ''
 193     if ARGS.input_file:
 194         (HEAD, TAIL) = read_input_file(ARGS.input_file)
 195     with open(ARGS.output_file, mode='w') as TRANSLIT_FILE:
 196         output_head(TRANSLIT_FILE, ARGS.unicode_version, head=HEAD)
 197         output_transliteration(TRANSLIT_FILE)
 198         output_tail(TRANSLIT_FILE, tail=TAIL)