From: Bruno Haible Date: Thu, 27 Jul 2006 12:20:10 +0000 (+0000) Subject: Use OpenMP to speed up msgmerge on multiprocessor systems. X-Git-Tag: 0.16.x-branchpoint~311 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=ee7f1e5a036a83482c9cd056dbfb10251f666554;p=thirdparty%2Fgettext.git Use OpenMP to speed up msgmerge on multiprocessor systems. --- diff --git a/NEWS b/NEWS index 86cb0fbea..7ab8639ff 100644 --- a/NEWS +++ b/NEWS @@ -1,3 +1,6 @@ +* msgmerge is faster now on CPUs with multiple execution units, if compiled + with GCC 4.2 or newer. + Version 0.15 - July 2006 * GUI program support: diff --git a/gettext-tools/ChangeLog b/gettext-tools/ChangeLog index 0360515c0..54cede050 100644 --- a/gettext-tools/ChangeLog +++ b/gettext-tools/ChangeLog @@ -1,3 +1,7 @@ +2006-07-23 Bruno Haible + + * configure.ac: Invoke gt_OPENMP. + 2006-07-23 Bruno Haible * configure.ac: Invoke gl_LOCK and gl_TLS. diff --git a/gettext-tools/configure.ac b/gettext-tools/configure.ac index 76553bba8..b5d7c4a1e 100644 --- a/gettext-tools/configure.ac +++ b/gettext-tools/configure.ac @@ -127,6 +127,9 @@ if test "$MSGMERGE_LIBM" = "?"; then fi AC_SUBST([MSGMERGE_LIBM]) +dnl Test for compiler flags needed to support OpenMP. +gt_OPENMP + dnl Checks for header files. AC_CHECK_HEADERS(limits.h malloc.h pwd.h string.h unistd.h utime.h values.h) gl_STDARG_H diff --git a/gettext-tools/m4/ChangeLog b/gettext-tools/m4/ChangeLog index 0ccf2871e..82a874cf8 100644 --- a/gettext-tools/m4/ChangeLog +++ b/gettext-tools/m4/ChangeLog @@ -1,3 +1,8 @@ +2006-07-23 Bruno Haible + + * openmp.m4: New file. + * Makefile.am (EXTRA_DIST): Add it. + 2006-07-23 Bruno Haible * tls.m4: New file, from gnulib. diff --git a/gettext-tools/m4/Makefile.am b/gettext-tools/m4/Makefile.am index 03f38194f..a3cdf48bf 100644 --- a/gettext-tools/m4/Makefile.am +++ b/gettext-tools/m4/Makefile.am @@ -78,6 +78,7 @@ memchr.m4 \ minmax.m4 \ mkdtemp.m4 \ onceonly.m4 \ +openmp.m4 \ pathmax.m4 \ quote.m4 \ quotearg.m4 \ diff --git a/gettext-tools/src/ChangeLog b/gettext-tools/src/ChangeLog index 554747626..5f1d3606c 100644 --- a/gettext-tools/src/ChangeLog +++ b/gettext-tools/src/ChangeLog @@ -1,3 +1,16 @@ +2006-07-23 Bruno Haible + + Exploit CPUs with multiple execution units. + * msgmerge.c: Include lock.h. + (struct definitions_ty): Add 'findex_init_lock' field. + (definitions_init): Initialize it. + (definitions_init_findex): Ensure findex is initialized by the + first thread who attempts so. + (match_domain): Split the main loop into two. Parallelize the first + loop using OpenMP pragmas. + * Makefile.am (msgmerge_CFLAGS): New variable. + (msgmerge_LDADD): Add OPENMP_CFLAGS. + 2006-07-25 Bruno Haible * Makefile.msvc: Remove file. diff --git a/gettext-tools/src/Makefile.am b/gettext-tools/src/Makefile.am index c70bcf2cc..ea767673a 100644 --- a/gettext-tools/src/Makefile.am +++ b/gettext-tools/src/Makefile.am @@ -253,13 +253,16 @@ libgettextpo_la_SOURCES += ../woe32dll/gettextpo-exports.c libgettextpo_la_LDFLAGS += -Wl,--export-all-symbols endif +# Compile-time flags for particular source files. +msgmerge_CFLAGS = $(AM_CFLAGS) $(OPENMP_CFLAGS) + # Link dependencies. # INTL_MACOSX_LIBS is needed because the programs depend on libintl.la # but libtool doesn't put -Wl,-framework options into .la files. # For msginit, it is also needed because of localename.c. msgcmp_LDADD = libgettextsrc.la @INTL_MACOSX_LIBS@ msgfmt_LDADD = libgettextsrc.la @INTL_MACOSX_LIBS@ -msgmerge_LDADD = libgettextsrc.la @INTL_MACOSX_LIBS@ @MSGMERGE_LIBM@ +msgmerge_LDADD = libgettextsrc.la @INTL_MACOSX_LIBS@ @MSGMERGE_LIBM@ $(OPENMP_CFLAGS) msgunfmt_LDADD = libgettextsrc.la @INTL_MACOSX_LIBS@ xgettext_LDADD = $(LIBUNINAME) libgettextsrc.la @INTL_MACOSX_LIBS@ @LTLIBEXPAT@ msgattrib_LDADD = libgettextsrc.la @INTL_MACOSX_LIBS@ diff --git a/gettext-tools/src/msgmerge.c b/gettext-tools/src/msgmerge.c index 42391304b..4fb86a6d3 100644 --- a/gettext-tools/src/msgmerge.c +++ b/gettext-tools/src/msgmerge.c @@ -51,6 +51,7 @@ #include "msgl-iconv.h" #include "msgl-equal.h" #include "msgl-fsearch.h" +#include "lock.h" #include "plural-count.h" #include "backupfile.h" #include "copy-file.h" @@ -594,6 +595,9 @@ struct definitions_ty /* A fuzzy index of the compendiums, for speed when doing fuzzy searches. Used only if use_fuzzy_matching is true and compendiums != NULL. */ message_fuzzy_index_ty *findex; + /* A once-only execution guard for the initialization of the fuzzy index. + Needed for OpenMP. */ + gl_lock_t findex_init_lock; /* The canonical encoding of the compendiums. */ const char *canon_charset; }; @@ -601,11 +605,14 @@ struct definitions_ty static inline void definitions_init (definitions_ty *definitions, const char *canon_charset) { + gl_lock_define (static, fresh_lock) + definitions->lists = message_list_list_alloc (); message_list_list_append (definitions->lists, NULL); if (compendiums != NULL) message_list_list_append_list (definitions->lists, compendiums); definitions->findex = NULL; + memcpy (&definitions->findex_init_lock, &fresh_lock, sizeof (gl_lock_t)); definitions->canon_charset = canon_charset; } @@ -614,24 +621,30 @@ definitions_init (definitions_ty *definitions, const char *canon_charset) static inline void definitions_init_findex (definitions_ty *definitions) { - /* Combine all the compendium message lists into a single one. Don't - bother checking for duplicates. */ - message_list_ty *all_compendium; - size_t i; - - all_compendium = message_list_alloc (false); - for (i = 0; i < compendiums->nitems; i++) + /* Protect against concurrent execution. */ + gl_lock_lock (definitions->findex_init_lock); + if (definitions->findex == NULL) { - message_list_ty *mlp = compendiums->item[i]; - size_t j; + /* Combine all the compendium message lists into a single one. Don't + bother checking for duplicates. */ + message_list_ty *all_compendium; + size_t i; - for (j = 0; j < mlp->nitems; j++) - message_list_append (all_compendium, mlp->item[j]); - } + all_compendium = message_list_alloc (false); + for (i = 0; i < compendiums->nitems; i++) + { + message_list_ty *mlp = compendiums->item[i]; + size_t j; + + for (j = 0; j < mlp->nitems; j++) + message_list_append (all_compendium, mlp->item[j]); + } - /* Create the fuzzy index from it. */ - definitions->findex = - message_fuzzy_index_alloc (all_compendium, definitions->canon_charset); + /* Create the fuzzy index from it. */ + definitions->findex = + message_fuzzy_index_alloc (all_compendium, definitions->canon_charset); + } + gl_lock_unlock (definitions->findex_init_lock); } /* Return the current list of non-compendium messages. */ @@ -1042,6 +1055,7 @@ match_domain (const char *fn1, const char *fn2, message_ty *header_entry; unsigned long int nplurals; char *untranslated_plural_msgstr; + struct search_result { message_ty *found; bool fuzzy; } *search_results; size_t j; header_entry = @@ -1050,22 +1064,72 @@ match_domain (const char *fn1, const char *fn2, untranslated_plural_msgstr = (char *) xmalloc (nplurals); memset (untranslated_plural_msgstr, '\0', nplurals); - for (j = 0; j < refmlp->nitems; j++, (*processed)++) - { - message_ty *refmsg; - message_ty *defmsg; - - /* Because merging can take a while we print something to signal - we are not dead. */ - if (!quiet && verbosity_level <= 1 && *processed % DOT_FREQUENCY == 0) - fputc ('.', stderr); + /* Most of the time is spent in definitions_search_fuzzy. + Perform it in a separate loop that can be parallelized by an OpenMP + capable compiler. */ + search_results = + (struct search_result *) + xmalloc (refmlp->nitems * sizeof (struct search_result)); + { + long int nn = refmlp->nitems; + long int jj; + + /* Tell the OpenMP capable compiler to distribute this loop across + several threads. The schedule is dynamic, because for some messages + the loop body can be executed very quickly, whereas for others it takes + a long time. */ + #ifdef _OPENMP + # pragma omp parallel for schedule(dynamic) + #endif + for (jj = 0; jj < nn; jj++) + { + message_ty *refmsg = refmlp->item[jj]; + message_ty *defmsg; + + /* Because merging can take a while we print something to signal + we are not dead. */ + if (!quiet && verbosity_level <= 1 && *processed % DOT_FREQUENCY == 0) + fputc ('.', stderr); + #ifdef _OPENMP + # pragma omp atomic + #endif + (*processed)++; + + /* See if it is in the other file. */ + defmsg = + definitions_search (definitions, refmsg->msgctxt, refmsg->msgid); + if (defmsg != NULL) + { + search_results[jj].found = defmsg; + search_results[jj].fuzzy = false; + } + else if (!is_header (refmsg) + /* If the message was not defined at all, try to find a very + similar message, it could be a typo, or the suggestion may + help. */ + && use_fuzzy_matching + && ((defmsg = + definitions_search_fuzzy (definitions, + refmsg->msgctxt, + refmsg->msgid)) != NULL)) + { + search_results[jj].found = defmsg; + search_results[jj].fuzzy = true; + } + else + search_results[jj].found = NULL; + } + } - refmsg = refmlp->item[j]; + for (j = 0; j < refmlp->nitems; j++) + { + message_ty *refmsg = refmlp->item[j]; - /* See if it is in the other file. */ - defmsg = definitions_search (definitions, refmsg->msgctxt, refmsg->msgid); - if (defmsg) + /* See if it is in the other file. + This used definitions_search. */ + if (search_results[j].found != NULL && !search_results[j].fuzzy) { + message_ty *defmsg = search_results[j].found; /* Merge the reference with the definition: take the #. and #: comments from the reference, take the # comments from the definition, take the msgstr from the definition. Add @@ -1083,13 +1147,11 @@ match_domain (const char *fn1, const char *fn2, { /* If the message was not defined at all, try to find a very similar message, it could be a typo, or the suggestion may - help. */ - if (use_fuzzy_matching - && ((defmsg = - definitions_search_fuzzy (definitions, - refmsg->msgctxt, - refmsg->msgid)) != NULL)) + help. This search assumed use_fuzzy_matching and used + definitions_search_fuzzy. */ + if (search_results[j].found != NULL && search_results[j].fuzzy) { + message_ty *defmsg = search_results[j].found; message_ty *mp; if (verbosity_level > 1) @@ -1158,6 +1220,8 @@ this message is used but not defined in %s"), fn1); } } + free (search_results); + /* Now postprocess the problematic merges. This is needed because we want the result to pass the "msgfmt -c -v" check. */ {