From 96dde0b8bdf6e4930f22556412d1680721929cd1 Mon Sep 17 00:00:00 2001 From: Daiki Ueno Date: Wed, 28 Jan 2015 16:50:20 +0900 Subject: [PATCH] msgfilter: Add --newline option The filter program was supposed to handle translation without a newline character at the end of line. This was causing portability problems with standard text processing programs on some platforms (BSD sed, for instance) and not friendly towards POSIX, where a "text file" is required to have an ending newline. The new --newline option controls the behavior. If it is given, both filter input and output are assumed to end with a newline character. * gettext-tools/src/msgfilter.c (newline): New variable. (long_options): Add --newline option. (main): Handle --newline option. (usage): Document --newline option. (process_string_with_newline): New function which wraps process_string. (process_message): Use process_string_with_newline instead of process_string if --newline is specified. * gettext-tools/tests/msgfilter-8: New file. * gettext-tools/tests/Makefile.am (TESTS): Add new test. * gettext-tools/doc/msgfilter.texi: Document --newline option. --- gettext-tools/doc/ChangeLog | 4 + gettext-tools/doc/msgfilter.texi | 23 +++-- gettext-tools/src/ChangeLog | 20 ++++ gettext-tools/src/msgfilter.c | 49 +++++++++- gettext-tools/tests/ChangeLog | 5 + gettext-tools/tests/Makefile.am | 2 +- gettext-tools/tests/msgfilter-8 | 152 +++++++++++++++++++++++++++++++ 7 files changed, 246 insertions(+), 9 deletions(-) create mode 100755 gettext-tools/tests/msgfilter-8 diff --git a/gettext-tools/doc/ChangeLog b/gettext-tools/doc/ChangeLog index 109505bfc..5f618a0a8 100644 --- a/gettext-tools/doc/ChangeLog +++ b/gettext-tools/doc/ChangeLog @@ -1,3 +1,7 @@ +2015-01-28 Daiki Ueno + + * msgfilter.texi: Document --newline option. + 2015-01-05 Daiki Ueno * gettext.texi (Vala): New section. diff --git a/gettext-tools/doc/msgfilter.texi b/gettext-tools/doc/msgfilter.texi index 1fae2514b..758f7a463 100644 --- a/gettext-tools/doc/msgfilter.texi +++ b/gettext-tools/doc/msgfilter.texi @@ -73,6 +73,14 @@ input and writes a modified translation to standard output. A frequently used filter is @samp{sed}. A few particular built-in filters are also recognized. +@table @samp +@itemx --newline +@opindex --newline@r{, @code{msgfilter} option} +Add newline at the end of each input line and also strip the ending +newline from the output line. + +@end table + @cindex @code{msgfilter} filter and catalog encoding Note: If the filter is not a built-in filter, you have to care about encodings: It is your responsibility to ensure that the @var{filter} can cope @@ -86,13 +94,14 @@ you can first convert the translation catalog to UTF-8 using the locale, by using the @code{LC_ALL} environment variable. @cindex portability problems with @code{sed} -Note: Most translations in a translation catalog don't end with a newline -character. For this reason, it is important that the @var{filter} -recognizes its last input line even if it ends without a newline, and that -it doesn't add an undesired trailing newline at the end. The @samp{sed} -program on some platforms is known to ignore the last line of input if it -is not terminated with a newline. You can use GNU @code{sed} instead; it -does not have this limitation. +Note: Most translations in a translation catalog don't end with a +newline character. For this reason, unless the @code{--newline} +option is used, it is important that the @var{filter} recognizes its +last input line even if it ends without a newline, and that it doesn't +add an undesired trailing newline at the end. The @samp{sed} program on +some platforms is known to ignore the last line of input if it is not +terminated with a newline. You can use GNU @code{sed} instead; it does +not have this limitation. @subsection Useful @var{filter-option}s when the @var{filter} is @samp{sed} diff --git a/gettext-tools/src/ChangeLog b/gettext-tools/src/ChangeLog index af08eda8b..7266c641b 100644 --- a/gettext-tools/src/ChangeLog +++ b/gettext-tools/src/ChangeLog @@ -1,3 +1,23 @@ +2015-01-28 Daiki Ueno + + msgfilter: Add --newline option + The filter program was supposed to handle translation without a + newline character at the end of line. This was causing + portability problems with standard text processing programs on + some platforms (BSD sed, for instance) and not friendly towards + POSIX, where a "text file" is required to have an ending newline. + The new --newline option controls the behavior. If it is given, + both filter input and output are assumed to end with a newline + character. + * msgfilter.c (newline): New variable. + (long_options): Add --newline option. + (main): Handle --newline option. + (usage): Document --newline option. + (process_string_with_newline): New function which wraps + process_string. + (process_message): Use process_string_with_newline instead of + process_string if --newline is specified. + 2015-01-24 Daiki Ueno xgettext, msgmerge: Avoid undefined non-null argument behavior diff --git a/gettext-tools/src/msgfilter.c b/gettext-tools/src/msgfilter.c index f9cb1cc28..470f82f4a 100644 --- a/gettext-tools/src/msgfilter.c +++ b/gettext-tools/src/msgfilter.c @@ -81,6 +81,8 @@ static const char *sub_path; static const char **sub_argv; static int sub_argc; +static bool newline; + /* Filter function. */ static void (*filter) (const char *str, size_t len, char **resultp, size_t *lengthp); @@ -96,6 +98,7 @@ static const struct option long_options[] = { "indent", no_argument, NULL, CHAR_MAX + 1 }, { "input", required_argument, NULL, 'i' }, { "keep-header", no_argument, &keep_header, 1 }, + { "newline", no_argument, NULL, CHAR_MAX + 9 }, { "no-escape", no_argument, NULL, CHAR_MAX + 2 }, { "no-location", no_argument, NULL, CHAR_MAX + 8 }, { "no-wrap", no_argument, NULL, CHAR_MAX + 3 }, @@ -269,6 +272,10 @@ main (int argc, char **argv) message_print_style_filepos (filepos_comment_none); break; + case CHAR_MAX + 9: /* --newline */ + newline = true; + break; + default: usage (EXIT_FAILURE); break; @@ -438,6 +445,12 @@ and writes a modified translation to standard output.\n\ ")); printf ("\n"); printf (_("\ +Filter input and output:\n")); + printf (_("\ + --newline add a newline at the end of input and\n\ + remove a newline from the end of output")); + printf ("\n"); + printf (_("\ Useful FILTER-OPTIONs when the FILTER is 'sed':\n")); printf (_("\ -e, --expression=SCRIPT add SCRIPT to the commands to be executed\n")); @@ -628,6 +641,35 @@ process_string (const char *str, size_t len, char **resultp, size_t *lengthp) } +/* Do the same thing as process_string but append a newline to STR + before processing, and remove a newline from the result. + */ +static void +process_string_with_newline (const char *str, size_t len, char **resultp, + size_t *lengthp) +{ + char *newstr; + char *result; + size_t length; + + newstr = XNMALLOC (len + 1, char); + memcpy (newstr, str, len); + newstr[len] = '\n'; + + process_string (newstr, len + 1, &result, &length); + + free (newstr); + + if (length > 0 && result[length - 1] == '\n') + result[--length] = '\0'; + else + error (0, 0, _("filter output is not terminated with a newline")); + + *resultp = result; + *lengthp = length; +} + + static void process_message (message_ty *mp) { @@ -705,7 +747,12 @@ process_message (message_ty *mp) } else unsetenv ("MSGFILTER_PLURAL_FORM"); - process_string (p, strlen (p), &result, &length); + + if (newline) + process_string_with_newline (p, strlen (p), &result, &length); + else + process_string (p, strlen (p), &result, &length); + result = (char *) xrealloc (result, length + 1); result[length] = '\0'; substrings[k] = result; diff --git a/gettext-tools/tests/ChangeLog b/gettext-tools/tests/ChangeLog index a08f91d78..c60f35a37 100644 --- a/gettext-tools/tests/ChangeLog +++ b/gettext-tools/tests/ChangeLog @@ -1,3 +1,8 @@ +2015-01-28 Daiki Ueno + + * msgfilter-8: New file. + * Makefile.am (TESTS): Add new test. + 2015-01-16 Daiki Ueno * xgettext-desktop-1: Check "invalid non-blank character" warning. diff --git a/gettext-tools/tests/Makefile.am b/gettext-tools/tests/Makefile.am index 5a0d3c0e4..c098a6f48 100644 --- a/gettext-tools/tests/Makefile.am +++ b/gettext-tools/tests/Makefile.am @@ -42,7 +42,7 @@ TESTS = gettext-1 gettext-2 gettext-3 gettext-4 gettext-5 gettext-6 gettext-7 \ msgen-1 msgen-2 msgen-3 msgen-4 \ msgexec-1 msgexec-2 msgexec-3 msgexec-4 msgexec-5 \ msgfilter-1 msgfilter-2 msgfilter-3 msgfilter-4 msgfilter-5 \ - msgfilter-6 msgfilter-7 \ + msgfilter-6 msgfilter-7 msgfilter-8 \ msgfilter-sr-latin-1 msgfilter-quote-1 \ msgfmt-1 msgfmt-2 msgfmt-3 msgfmt-4 msgfmt-5 msgfmt-6 msgfmt-7 \ msgfmt-8 msgfmt-9 msgfmt-10 msgfmt-11 msgfmt-12 msgfmt-13 msgfmt-14 \ diff --git a/gettext-tools/tests/msgfilter-8 b/gettext-tools/tests/msgfilter-8 new file mode 100755 index 000000000..49cf2c0fd --- /dev/null +++ b/gettext-tools/tests/msgfilter-8 @@ -0,0 +1,152 @@ +#! /bin/sh +. "${srcdir=.}/init.sh"; path_prepend_ . ../src + +# Test --newline option. + +cat <<\EOF > mfi-test8.po +# HEADER. +# +msgid "" +msgstr "" +"Project-Id-Version: Bonnie Tyler\n" +"Content-Type: text/plain; charset=ISO-8859-1\n" +"Content-Transfer-Encoding: 8bit\n" + +#: married-men:4 +#, fuzzy +msgid "The world is full of married men" +msgstr "So viele verheiratete Männer" + +#: married-men:5 +msgid "with wives who never understand" +msgstr "und ihre Frauen verstehen sie nicht" + +#: married-men:6 +msgid "They're looking for someone to share" +msgstr "" + +# schwer zu übersetzen... +#: married-men:7 +msgid "the excitement of a love affair" +msgstr "" + +#: married-men:8 +msgid "Just as soon as they find you" +msgstr "" + +#: married-men:9 +msgid "They warn you and darn you" +msgstr "" + +#~ msgid "You fly on the wings of romance" +#~ msgstr "Die Flügel der frischen Liebe heben dich zum Himmel" + +#, fuzzy +#~ msgid "In the eyes of the world" +#~ msgstr "Für die anderen" + +# Etwas freie Übersetzung. +#~ msgid "You're just another crazy girl" +#~ msgstr "bist du bloß ein verrücktes dummes Ding" + +#~ msgid "Who loves a married man" +#~ msgstr "das einen verheirateten Mann liebt" +EOF + +cat <<\EOF > filter.sh +#!/bin/sh +cat +if test "$MSGFILTER_MSGID" = "the excitement of a love affair"; then + echo -n non-terminated line +else + echo terminated line +fi +EOF + +: ${MSGFILTER=msgfilter} +: ${CONFIG_SHELL=${SHELL-/bin/sh}} +LC_ALL=C ${MSGFILTER} --newline -i mfi-test8.po -o mfi-test8.out \ + ${CONFIG_SHELL} filter.sh >mfi-test8.err 2>&1 +result=$? +cat mfi-test8.err | grep -v 'warning: Locale charset' | grep -v '^ ' +test $result = 0 || { exit 1; } +cat mfi-test8.err | grep 'msgfilter: filter output is not terminated with a newline' >/dev/null +test $result = 0 || { exit 1; } + +cat <<\EOF > mfi-test8.ok +# HEADER. +# +msgid "" +msgstr "" +"Project-Id-Version: Bonnie Tyler\n" +"Content-Type: text/plain; charset=ISO-8859-1\n" +"Content-Transfer-Encoding: 8bit\n" +"\n" +"terminated line" + +#: married-men:4 +#, fuzzy +msgid "The world is full of married men" +msgstr "" +"So viele verheiratete Männer\n" +"terminated line" + +#: married-men:5 +msgid "with wives who never understand" +msgstr "" +"und ihre Frauen verstehen sie nicht\n" +"terminated line" + +#: married-men:6 +msgid "They're looking for someone to share" +msgstr "" +"\n" +"terminated line" + +# schwer zu übersetzen... +#: married-men:7 +msgid "the excitement of a love affair" +msgstr "" +"\n" +"non-terminated line" + +#: married-men:8 +msgid "Just as soon as they find you" +msgstr "" +"\n" +"terminated line" + +#: married-men:9 +msgid "They warn you and darn you" +msgstr "" +"\n" +"terminated line" + +#~ msgid "You fly on the wings of romance" +#~ msgstr "" +#~ "Die Flügel der frischen Liebe heben dich zum Himmel\n" +#~ "terminated line" + +#, fuzzy +#~ msgid "In the eyes of the world" +#~ msgstr "" +#~ "Für die anderen\n" +#~ "terminated line" + +# Etwas freie Übersetzung. +#~ msgid "You're just another crazy girl" +#~ msgstr "" +#~ "bist du bloß ein verrücktes dummes Ding\n" +#~ "terminated line" + +#~ msgid "Who loves a married man" +#~ msgstr "" +#~ "das einen verheirateten Mann liebt\n" +#~ "terminated line" +EOF + +: ${DIFF=diff} +${DIFF} mfi-test8.ok mfi-test8.out +result=$? + +exit $result -- 2.47.2