From: Bruno Haible Date: Sat, 20 Oct 2007 21:32:04 +0000 (+0000) Subject: Make msgmerge's introduction of fuzzy markers more consistent with msgfmt's X-Git-Tag: v0.17~98 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=8916b54d3ddc3f28b56fa11d351ee143c63efb1b;p=thirdparty%2Fgettext.git Make msgmerge's introduction of fuzzy markers more consistent with msgfmt's --check-format. --- diff --git a/gettext-tools/src/ChangeLog b/gettext-tools/src/ChangeLog index c6fba9cc9..3a17efb79 100644 --- a/gettext-tools/src/ChangeLog +++ b/gettext-tools/src/ChangeLog @@ -1,3 +1,27 @@ +2007-10-20 Bruno Haible + + Make msgmerge's introduction of fuzzy markers more consistent with + msgfmt's --check-format. + * format.h (check_msgid_msgstr_format_i): New declaration. + * format.c (check_msgid_msgstr_format_i): New function, extracted from + check_msgid_msgstr_format. + (check_msgid_msgstr_format): Use it. + * msgl-check.h: Include plural-eval.h. + (check_plural_eval): New declaration. + * msgl-check.c (check_plural_eval): Add const to first parameter. Make + non-static. + (check_plural): Update. + * msgmerge.c: Include plural-exp.h, msgl-check.h, po-xerror.h. + (msgfmt_check_pair_fails): Remove function. + (silent_error_logger, silent_xerror): New functions. + (message_merge): Add plural_distribution, plural_distribution_length + arguments. Call check_msgid_msgstr_format_i instead of + msgfmt_check_pair_fails. + (match_domain): Extract not only the plural count, but also the + plural expression from the header entry. Determine the plural + distribution from it. Pass it to message_merge. + Reported by Chusslove Illich (Часлав Илић) . + 2007-10-20 Bruno Haible * msgl-check.c (check_plural): If there is no header entry, or if the diff --git a/gettext-tools/src/format.c b/gettext-tools/src/format.c index f85fa146d..994f7e9fd 100644 --- a/gettext-tools/src/format.c +++ b/gettext-tools/src/format.c @@ -58,6 +58,105 @@ struct formatstring_parser *formatstring_parsers[NFORMATS] = /* format_boost */ &formatstring_boost }; +/* Check whether both formats strings contain compatible format + specifications for format type i (0 <= i < NFORMATS). + PLURAL_DISTRIBUTION is either NULL or an array of nplurals elements, + PLURAL_DISTRIBUTION[j] being true if the value j appears to be assumed + infinitely often by the plural formula. + PLURAL_DISTRIBUTION_LENGTH is the length of the PLURAL_DISTRIBUTION array. + Return the number of errors that were seen. */ +int +check_msgid_msgstr_format_i (const char *msgid, const char *msgid_plural, + const char *msgstr, size_t msgstr_len, + size_t i, + const unsigned char *plural_distribution, + unsigned long plural_distribution_length, + formatstring_error_logger_t error_logger) +{ + int seen_errors = 0; + + /* At runtime, we can assume the program passes arguments that fit well for + msgid. We must signal an error if msgstr wants more arguments that msgid + accepts. + If msgstr wants fewer arguments than msgid, it wouldn't lead to a crash + at runtime, but we nevertheless give an error because + 1) this situation occurs typically after the programmer has added some + arguments to msgid, so we must make the translator specially aware + of it (more than just "fuzzy"), + 2) it is generally wrong if a translation wants to ignore arguments that + are used by other translations. */ + + struct formatstring_parser *parser = formatstring_parsers[i]; + char *invalid_reason = NULL; + void *msgid_descr = + parser->parse (msgid_plural != NULL ? msgid_plural : msgid, false, NULL, + &invalid_reason); + + if (msgid_descr != NULL) + { + char buf[18+1]; + const char *pretty_msgstr = "msgstr"; + bool has_plural_translations = (strlen (msgstr) + 1 < msgstr_len); + const char *p_end = msgstr + msgstr_len; + const char *p; + unsigned int j; + + for (p = msgstr, j = 0; p < p_end; p += strlen (p) + 1, j++) + { + void *msgstr_descr; + + if (msgid_plural != NULL) + { + sprintf (buf, "msgstr[%u]", j); + pretty_msgstr = buf; + } + + msgstr_descr = parser->parse (p, true, NULL, &invalid_reason); + + if (msgstr_descr != NULL) + { + /* Use strict checking (require same number of format + directives on both sides) if the message has no plurals, + or if msgid_plural exists but on the msgstr[] side + there is only msgstr[0], or if plural_distribution[j] + indicates that the variant applies to infinitely many + values of N. + Use relaxed checking when there are at least two + msgstr[] forms and the plural_distribution array does + not give more precise information. */ + bool strict_checking = + (msgid_plural == NULL + || !has_plural_translations + || (plural_distribution != NULL + && j < plural_distribution_length + && plural_distribution[j])); + + if (parser->check (msgid_descr, msgstr_descr, + strict_checking, + error_logger, pretty_msgstr)) + seen_errors++; + + parser->free (msgstr_descr); + } + else + { + error_logger (_("\ +'%s' is not a valid %s format string, unlike 'msgid'. Reason: %s"), + pretty_msgstr, format_language_pretty[i], + invalid_reason); + seen_errors++; + free (invalid_reason); + } + } + + parser->free (msgid_descr); + } + else + free (invalid_reason); + + return seen_errors; +} + /* Check whether both formats strings contain compatible format specifications. PLURAL_DISTRIBUTION is either NULL or an array of nplurals elements, @@ -75,7 +174,6 @@ check_msgid_msgstr_format (const char *msgid, const char *msgid_plural, { int seen_errors = 0; size_t i; - unsigned int j; /* We check only those messages for which the msgid's is_format flag is one of 'yes' or 'possible'. We don't check msgids with is_format @@ -85,85 +183,11 @@ check_msgid_msgstr_format (const char *msgid, const char *msgid_plural, anywhere where a translator wishes to use a percent sign. */ for (i = 0; i < NFORMATS; i++) if (possible_format_p (is_format[i])) - { - /* At runtime, we can assume the program passes arguments that - fit well for msgid. We must signal an error if msgstr wants - more arguments that msgid accepts. - If msgstr wants fewer arguments than msgid, it wouldn't lead - to a crash at runtime, but we nevertheless give an error because - 1) this situation occurs typically after the programmer has - added some arguments to msgid, so we must make the translator - specially aware of it (more than just "fuzzy"), - 2) it is generally wrong if a translation wants to ignore - arguments that are used by other translations. */ - - struct formatstring_parser *parser = formatstring_parsers[i]; - char *invalid_reason = NULL; - void *msgid_descr = - parser->parse (msgid_plural != NULL ? msgid_plural : msgid, - false, NULL, &invalid_reason); - - if (msgid_descr != NULL) - { - char buf[18+1]; - const char *pretty_msgstr = "msgstr"; - bool has_plural_translations = (strlen (msgstr) + 1 < msgstr_len); - const char *p_end = msgstr + msgstr_len; - const char *p; - - for (p = msgstr, j = 0; p < p_end; p += strlen (p) + 1, j++) - { - void *msgstr_descr; - - if (msgid_plural != NULL) - { - sprintf (buf, "msgstr[%u]", j); - pretty_msgstr = buf; - } - - msgstr_descr = parser->parse (p, true, NULL, &invalid_reason); - - if (msgstr_descr != NULL) - { - /* Use strict checking (require same number of format - directives on both sides) if the message has no plurals, - or if msgid_plural exists but on the msgstr[] side - there is only msgstr[0], or if plural_distribution[j] - indicates that the variant applies to infinitely many - values of N. - Use relaxed checking when there are at least two - msgstr[] forms and the plural_distribution array does - not give more precise information. */ - bool strict_checking = - (msgid_plural == NULL - || !has_plural_translations - || (plural_distribution != NULL - && j < plural_distribution_length - && plural_distribution[j])); - - if (parser->check (msgid_descr, msgstr_descr, - strict_checking, - error_logger, pretty_msgstr)) - seen_errors++; - - parser->free (msgstr_descr); - } - else - { - error_logger (_("\ -'%s' is not a valid %s format string, unlike 'msgid'. Reason: %s"), - pretty_msgstr, format_language_pretty[i], - invalid_reason); - seen_errors++; - free (invalid_reason); - } - } - - parser->free (msgid_descr); - } - else - free (invalid_reason); - } + seen_errors += check_msgid_msgstr_format_i (msgid, msgid_plural, + msgstr, msgstr_len, i, + plural_distribution, + plural_distribution_length, + error_logger); return seen_errors; } diff --git a/gettext-tools/src/format.h b/gettext-tools/src/format.h index df5d37e35..3bb96646b 100644 --- a/gettext-tools/src/format.h +++ b/gettext-tools/src/format.h @@ -138,6 +138,20 @@ extern void string. */ extern unsigned int get_python_format_unnamed_arg_count (const char *string); +/* Check whether both formats strings contain compatible format + specifications for format type i (0 <= i < NFORMATS). + PLURAL_DISTRIBUTION is either NULL or an array of nplurals elements, + PLURAL_DISTRIBUTION[j] being true if the value j appears to be assumed + infinitely often by the plural formula. + Return the number of errors that were seen. */ +extern int + check_msgid_msgstr_format_i (const char *msgid, const char *msgid_plural, + const char *msgstr, size_t msgstr_len, + size_t i, + const unsigned char *plural_distribution, + unsigned long plural_distribution_length, + formatstring_error_logger_t error_logger); + /* Check whether both formats strings contain compatible format specifications. PLURAL_DISTRIBUTION is either NULL or an array of nplurals elements, diff --git a/gettext-tools/src/msgl-check.c b/gettext-tools/src/msgl-check.c index 3da11ff13..570e1979f 100644 --- a/gettext-tools/src/msgl-check.c +++ b/gettext-tools/src/msgl-check.c @@ -48,13 +48,14 @@ /* Check the values returned by plural_eval. + Signals the errors through po_xerror. Return the number of errors that were seen. If no errors, returns in *PLURAL_DISTRIBUTION either NULL or an array of length NPLURALS_VALUE describing which plural formula values appear infinitely often and in *PLURAL_DISTRIBUTION_LENGTH the length of this array. */ -static int -check_plural_eval (struct expression *plural_expr, +int +check_plural_eval (const struct expression *plural_expr, unsigned long nplurals_value, const message_ty *header, unsigned char **plural_distribution, @@ -322,7 +323,7 @@ check_plural (message_list_ty *mlp, const char *endp; unsigned long int nplurals_value; struct parse_args args; - struct expression *plural_expr; + const struct expression *plural_expr; /* First check the number. */ nplurals += 9; diff --git a/gettext-tools/src/msgl-check.h b/gettext-tools/src/msgl-check.h index 2c5e4ab93..72123011f 100644 --- a/gettext-tools/src/msgl-check.h +++ b/gettext-tools/src/msgl-check.h @@ -20,6 +20,7 @@ #include "message.h" #include "pos.h" +#include "plural-eval.h" #ifdef __cplusplus @@ -27,6 +28,19 @@ extern "C" { #endif +/* Check the values returned by plural_eval. + Signals the errors through po_xerror. + Return the number of errors that were seen. + If no errors, returns in *PLURAL_DISTRIBUTION either NULL or an array + of length NPLURALS_VALUE describing which plural formula values appear + infinitely often and in *PLURAL_DISTRIBUTION_LENGTH the length of this + array. */ +extern int check_plural_eval (const struct expression *plural_expr, + unsigned long nplurals_value, + const message_ty *header, + unsigned char **plural_distribution, + unsigned long *plural_distribution_length); + /* Perform all checks on a non-obsolete message. PLURAL_DISTRIBUTION is either NULL or an array of nplurals elements, PLURAL_DISTRIBUTION[j] being true if the value j appears to be assumed diff --git a/gettext-tools/src/msgmerge.c b/gettext-tools/src/msgmerge.c index 4124a0799..ce73f17f1 100644 --- a/gettext-tools/src/msgmerge.c +++ b/gettext-tools/src/msgmerge.c @@ -55,7 +55,10 @@ #include "msgl-equal.h" #include "msgl-fsearch.h" #include "lock.h" +#include "plural-exp.h" #include "plural-count.h" +#include "msgl-check.h" +#include "po-xerror.h" #include "backupfile.h" #include "copy-file.h" #include "propername.h" @@ -729,57 +732,31 @@ definitions_destroy (definitions_ty *definitions) } -static bool -msgfmt_check_pair_fails (const lex_pos_ty *pos, - const char *msgid, const char *msgid_plural, - const char *msgstr, size_t msgstr_len, - size_t fmt) +/* A silent error logger. We are only interested in knowing whether errors + occurred at all. */ +static void +silent_error_logger (const char *format, ...) + __attribute__ ((__format__ (__printf__, 1, 2))); +static void +silent_error_logger (const char *format, ...) { - bool failure; - struct formatstring_parser *parser = formatstring_parsers[fmt]; - char *invalid_reason = NULL; - void *msgid_descr = - parser->parse (msgid_plural != NULL ? msgid_plural : msgid, false, NULL, - &invalid_reason); - - failure = false; - if (msgid_descr != NULL) - { - const char *p_end = msgstr + msgstr_len; - const char *p; - - for (p = msgstr; p < p_end; p += strlen (p) + 1) - { - void *msgstr_descr = - parser->parse (msgstr, true, NULL, &invalid_reason); - - if (msgstr_descr != NULL) - { - failure = parser->check (msgid_descr, msgstr_descr, - msgid_plural == NULL, NULL, NULL); - parser->free (msgstr_descr); - } - else - { - failure = true; - free (invalid_reason); - } - - if (failure) - break; - } +} - parser->free (msgid_descr); - } - else - free (invalid_reason); - return failure; +/* Another silent error logger. */ +static void +silent_xerror (int severity, + const struct message_ty *message, + const char *filename, size_t lineno, size_t column, + int multiline_p, const char *message_text) +{ } static message_ty * -message_merge (message_ty *def, message_ty *ref, bool force_fuzzy) +message_merge (message_ty *def, message_ty *ref, bool force_fuzzy, + const unsigned char *plural_distribution, + unsigned long plural_distribution_length) { const char *msgstr; size_t msgstr_len; @@ -1052,8 +1029,11 @@ message_merge (message_ty *def, message_ty *ref, bool force_fuzzy) if (!result->is_fuzzy && possible_format_p (ref->is_format[i]) && !possible_format_p (def->is_format[i]) - && msgfmt_check_pair_fails (&def->pos, ref->msgid, ref->msgid_plural, - msgstr, msgstr_len, i)) + && check_msgid_msgstr_format_i (ref->msgid, ref->msgid_plural, + msgstr, msgstr_len, i, + plural_distribution, + plural_distribution_length, + silent_error_logger) > 0) result->is_fuzzy = true; } @@ -1117,16 +1097,39 @@ match_domain (const char *fn1, const char *fn2, { message_ty *header_entry; unsigned long int nplurals; + const struct expression *plural_expr; char *untranslated_plural_msgstr; + unsigned char *plural_distribution; + unsigned long plural_distribution_length; struct search_result { message_ty *found; bool fuzzy; } *search_results; size_t j; header_entry = message_list_search (definitions_current_list (definitions), NULL, ""); - nplurals = get_plural_count (header_entry ? header_entry->msgstr : NULL); + extract_plural_expression (header_entry ? header_entry->msgstr : NULL, + &plural_expr, &nplurals); untranslated_plural_msgstr = XNMALLOC (nplurals, char); memset (untranslated_plural_msgstr, '\0', nplurals); + /* Determine the plural distribution of the plural_expr formula. */ + { + /* Disable error output temporarily. */ + void (*old_po_xerror) (int, const struct message_ty *, const char *, size_t, + size_t, int, const char *) + = po_xerror; + po_xerror = silent_xerror; + + if (check_plural_eval (plural_expr, nplurals, header_entry, + &plural_distribution, + &plural_distribution_length) > 0) + { + plural_distribution = NULL; + plural_distribution_length = 0; + } + + po_xerror = old_po_xerror; + } + /* Most of the time is spent in definitions_search_fuzzy. Perform it in a separate loop that can be parallelized by an OpenMP capable compiler. */ @@ -1195,7 +1198,9 @@ match_domain (const char *fn1, const char *fn2, #: comments from the reference, take the # comments from the definition, take the msgstr from the definition. Add this merged entry to the output message list. */ - message_ty *mp = message_merge (defmsg, refmsg, false); + message_ty *mp = + message_merge (defmsg, refmsg, false, + plural_distribution, plural_distribution_length); message_list_append (resultmlp, mp); @@ -1228,7 +1233,9 @@ this message is used but not defined...")); #: comments from the reference, take the # comments from the definition, take the msgstr from the definition. Add this merged entry to the output message list. */ - mp = message_merge (defmsg, refmsg, true); + mp = message_merge (defmsg, refmsg, true, + plural_distribution, + plural_distribution_length); message_list_append (resultmlp, mp); diff --git a/gettext-tools/tests/ChangeLog b/gettext-tools/tests/ChangeLog index be1d5b8d2..0f6310ad3 100644 --- a/gettext-tools/tests/ChangeLog +++ b/gettext-tools/tests/ChangeLog @@ -1,3 +1,9 @@ +2007-10-20 Bruno Haible + + * msgmerge-21: New file. + * Makefile.am (TESTS): Add it. + Reported by Chusslove Illich (Часлав Илић) . + 2007-10-19 Bruno Haible Avoid test suite failures on Cygwin-hosted mingw. diff --git a/gettext-tools/tests/Makefile.am b/gettext-tools/tests/Makefile.am index 405709389..f4ea05e33 100644 --- a/gettext-tools/tests/Makefile.am +++ b/gettext-tools/tests/Makefile.am @@ -53,7 +53,7 @@ TESTS = gettext-1 gettext-2 gettext-3 gettext-4 gettext-5 gettext-6 gettext-7 \ msgmerge-1 msgmerge-2 msgmerge-3 msgmerge-4 msgmerge-5 msgmerge-6 \ msgmerge-7 msgmerge-8 msgmerge-9 msgmerge-10 msgmerge-11 msgmerge-12 \ msgmerge-13 msgmerge-14 msgmerge-15 msgmerge-16 msgmerge-17 \ - msgmerge-18 msgmerge-19 msgmerge-20 \ + msgmerge-18 msgmerge-19 msgmerge-20 msgmerge-21 \ msgmerge-compendium-1 msgmerge-compendium-2 msgmerge-compendium-3 \ msgmerge-compendium-4 msgmerge-compendium-5 msgmerge-compendium-6 \ msgmerge-properties-1 msgmerge-properties-2 \ diff --git a/gettext-tools/tests/msgmerge-21 b/gettext-tools/tests/msgmerge-21 new file mode 100755 index 000000000..92f572357 --- /dev/null +++ b/gettext-tools/tests/msgmerge-21 @@ -0,0 +1,92 @@ +#! /bin/sh + +# Test msgmerge when a message's flags have been changed from c-format to +# kde-format. Reported by Chusslove Illich (Часлав Илић). + +tmpfiles="" +trap 'rm -fr $tmpfiles' 1 2 3 15 + +tmpfiles="$tmpfiles mm-test21.po" +cat <<\EOF > mm-test21.po +msgid "" +msgstr "" +"Project-Id-Version: GNU gettext-tools 0.16\n" +"Report-Msgid-Bugs-To: bug-gnu-gettext@gnu.org\n" +"POT-Creation-Date: 2007-10-18 02:57+0200\n" +"PO-Revision-Date: 2007-06-28 16:37+0200\n" +"Last-Translator: Karl Eichwalder \n" +"Language-Team: German \n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Plural-Forms: nplurals=2; plural=(n != 1);\n" + +#, c-format +msgid "Add resource from addressbook" +msgid_plural "Add %n resources from addressbook" +msgstr[0] "Engadir un recurso dende o libro de enderezos" +msgstr[1] "Engadir %n recursos dende o libro de enderezos" +EOF + +tmpfiles="$tmpfiles mm-test21.pot" +cat <<\EOF > mm-test21.pot +msgid "" +msgstr "" +"Project-Id-Version: GNU gettext-tools 0.16\n" +"Report-Msgid-Bugs-To: bug-gnu-gettext@gnu.org\n" +"POT-Creation-Date: 2007-10-19 02:57+0200\n" +"PO-Revision-Date: 2007-06-28 16:37+0200\n" +"Last-Translator: \n" +"Language-Team: \n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: 8bit\n" + +#, kde-format +msgid "Add resource from addressbook" +msgid_plural "Add %1 resources from addressbook" +msgstr[0] "" +msgstr[1] "" +EOF + +tmpfiles="$tmpfiles mm-test21.tmp.po mm-test21.new.po" +: ${MSGMERGE=msgmerge} +${MSGMERGE} -q -o mm-test21.tmp.po mm-test21.po mm-test21.pot +test $? = 0 || { rm -fr $tmpfiles; exit 1; } +tr -d '\r' < mm-test21.tmp.po > mm-test21.new.po +test $? = 0 || { rm -fr $tmpfiles; exit 1; } + +tmpfiles="$tmpfiles mm-test21.ok" +cat <<\EOF > mm-test21.ok +msgid "" +msgstr "" +"Project-Id-Version: GNU gettext-tools 0.16\n" +"Report-Msgid-Bugs-To: bug-gnu-gettext@gnu.org\n" +"POT-Creation-Date: 2007-10-19 02:57+0200\n" +"PO-Revision-Date: 2007-06-28 16:37+0200\n" +"Last-Translator: Karl Eichwalder \n" +"Language-Team: German \n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Plural-Forms: nplurals=2; plural=(n != 1);\n" + +#, fuzzy, kde-format +msgid "Add resource from addressbook" +msgid_plural "Add %1 resources from addressbook" +msgstr[0] "Engadir un recurso dende o libro de enderezos" +msgstr[1] "Engadir %n recursos dende o libro de enderezos" +EOF + +: ${DIFF=diff} +${DIFF} mm-test21.ok mm-test21.new.po +test $? = 0 || { rm -fr $tmpfiles; exit 1; } + +tmpfiles="$tmpfiles mm-test21.mo" +: ${MSGFMT=msgfmt} +${MSGFMT} --check -o mm-test21.mo mm-test21.new.po +test $? = 0 || { rm -fr $tmpfiles; exit 1; } + +rm -fr $tmpfiles + +exit 0