Add --no-hard-links option to localedef (bug 23923)

Downstream distributions need consistent sets of hardlinks in
order for rpm to operate effectively. This means that even if
locales are built with a high level of parallelism that the
resulting files need to have consistent hardlink counts. The only
way to achieve this is with a post-install hardlink pass using a
program like 'hardlink' (shipped in Fedora).

If the downstream distro wants to post-process the hardlinks then
the time spent in localedef looking up sibling directories and
processing hardlinks is wasted effort.

To optimize the build and install pass we add a --no-hard-links
option to localedef to avoid doing the hardlink optimziation for
size.

Tested on x86_64 with 'make localedata/install-locale-files'
before and after. Without the patch we have files with 100+
hardlink counts. After the patch and running with --no-hard-links
all link counts are 1. This patch also alters the convenience
target 'make localedata/install-locale-files' to use the new
option.

Signed-off-by: Carlos O'Donell <carlos@redhat.com>

diff --git a/ChangeLog b/ChangeLog
index a11a9c696164e8338e4a6e29c5649a9eccd6e783..d5e7e3da4f41717325cfa879b7a4f1478ef53c56 100644 (file)
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,13 @@
+2018-12-03  Carlos O'Donell  <carlos@redhat.com>
+
+       [BZ #23923]
+       * locale/programs/localedef.c: Declare boolean hard_links default true.
+       (options): Add --no-hard-links option.
+       (parse_opt): Add OPT_NO_HARD_LINKS case and set hard_links to false.
+       * locale/programs/localedef.h: Declare prototype for hard_links.
+       * locale/programs/locfile.c (write_locale_data): Don't use hard
+       links if hard_links is false.
+
 2018-12-03  H.J. Lu  <hongjiu.lu@intel.com>
 
        * sysdeps/x86/cacheinfo.c (intel_check_word): Updated for

diff --git a/locale/programs/localedef.c b/locale/programs/localedef.c
index d718d2e9f47382bc0743da62bd5b0134042ac7ef..6c4936be6b9089abaf5fbc1de962a2f613a7d94a 100644 (file)
--- a/locale/programs/localedef.c
+++ b/locale/programs/localedef.c
@@ -85,6 +85,9 @@ static bool replace_archive;
 /* If true list archive content.  */
 static bool list_archive;
 
+/* If true create hard links to other locales (default).  */
+bool hard_links = true;
+
 /* Maximum number of retries when opening the locale archive.  */
 int max_locarchive_open_retry = 10;
 
@@ -105,6 +108,7 @@ void (*argp_program_version_hook) (FILE *, struct argp_state *) = print_version;
 #define OPT_BIG_ENDIAN 401
 #define OPT_NO_WARN 402
 #define OPT_WARN 403
+#define OPT_NO_HARD_LINKS 404
 
 /* Definitions of arguments for argp functions.  */
 static const struct argp_option options[] =
@@ -120,6 +124,8 @@ static const struct argp_option options[] =
   { NULL, 0, NULL, 0, N_("Output control:") },
   { "force", 'c', NULL, 0,
     N_("Create output even if warning messages were issued") },
+  { "no-hard-links", OPT_NO_HARD_LINKS, NULL, 0,
+    N_("Do not create hard links between installed locales") },
   { "prefix", OPT_PREFIX, N_("PATH"), 0, N_("Optional output file prefix") },
   { "posix", OPT_POSIX, NULL, 0, N_("Strictly conform to POSIX") },
   { "quiet", OPT_QUIET, NULL, 0,
@@ -389,6 +395,10 @@ parse_opt (int key, char *arg, struct argp_state *state)
       /* Enable the warnings.  */
       set_warnings (arg, true);
       break;
+    case OPT_NO_HARD_LINKS:
+      /* Do not hard link to other locales.  */
+      hard_links = false;
+      break;
     case 'c':
       force_output = 1;
       break;

diff --git a/locale/programs/localedef.h b/locale/programs/localedef.h
index 0083faceabbf3dd92819b5a268d085e6c1a54813..e2b39e78f320b2dc6cab8cb7ed48cceaade4bdba 100644 (file)
--- a/locale/programs/localedef.h
+++ b/locale/programs/localedef.h
@@ -118,6 +118,7 @@ extern const char *repertoire_global;
 extern int max_locarchive_open_retry;
 extern bool no_archive;
 extern const char *alias_file;
+extern bool hard_links;
 
 
 /* Prototypes for a few program-wide used functions.  */

diff --git a/locale/programs/locfile.c b/locale/programs/locfile.c
index 32e5f761f2dac3d91c97282341a56f27c9735d58..1555231deb9f936efc545df2ffefec4223040f9d 100644 (file)
--- a/locale/programs/locfile.c
+++ b/locale/programs/locfile.c
@@ -702,7 +702,7 @@ write_locale_data (const char *output_path, int catidx, const char *category,
   size_t cnt, step, maxiov;
   int fd;
   char *fname;
-  const char **other_paths;
+  const char **other_paths = NULL;
   uint32_t header[2];
   size_t n_elem;
   struct iovec vec[3];
@@ -827,9 +827,22 @@ failure while writing data for category `%s'"), category);
 
   close (fd);
 
-  /* Compare the file with the locale data files for the same category in
-     other locales, and see if we can reuse it, to save disk space.  */
-  other_paths = siblings (output_path);
+  /* Compare the file with the locale data files for the same category
+     in other locales, and see if we can reuse it, to save disk space.
+     If the user specified --no-hard-links to localedef then hard_links
+     is false, other_paths remains NULL and we skip the optimization
+     below.  The use of --no-hard-links is distribution specific since
+     some distros have post-processing hard-link steps and so doing this
+     here is a waste of time.  Worse than a waste of time in rpm-based
+     distributions it can result in build determinism issues from
+     build-to-build since some files may get a hard link in one pass but
+     not in another (if the files happened to be created in parallel).  */
+  if (hard_links)
+    other_paths = siblings (output_path);
+
+  /* If there are other paths, then walk the sibling paths looking for
+     files with the same content so we can hard link and reduce disk
+     space usage.  */
   if (other_paths != NULL)
     {
       struct stat64 fname_stat;

diff --git a/localedata/Makefile b/localedata/Makefile
index 0eea396ad86da956ec79da248f6bc6f5a7bd8093..b245f0ff54e55f82449c6cdbd61e7969f6d5c967 100644 (file)
--- a/localedata/Makefile
+++ b/localedata/Makefile
@@ -423,7 +423,7 @@ $(INSTALL-SUPPORTED-LOCALE-ARCHIVE): install-locales-dir
        $(build-one-locale)
 
 $(INSTALL-SUPPORTED-LOCALE-FILES): install-locales-dir
-       @flags="-c --no-archive"; \
+       @flags="-c --no-archive --no-hard-links"; \
        $(build-one-locale)
 
 tst-setlocale-ENV = LC_ALL=ja_JP.EUC-JP