ld: Cache and reuse the IR archive file descriptor

Linker plugin_object_p opens the IR archive for each IR archive member.
For GCC plugin, plugin_object_p closes the archive file descriptor.  But
for LLVM plugin, the archive file descriptor remains open.  If there are
3000 IR archive members, there are 3000 file descriptors for them.  We
can run out of file descriptors petty easily.

1. Add archive_plugin_fd and archive_plugin_fd_open_count to bfd so that
we can cache and reuse the IR archive file descriptor for all IR archive
members in the archive.
2. Add bfd_plugin_close_file_descriptor to properly close the IR archive
file descriptor.

bfd/

	PR ld/28040
	* archive.c (_bfd_archive_close_and_cleanup): Close the archive
	plugin file descriptor if needed.
	* bfd.c (bfd): Add archive_plugin_fd and
	archive_plugin_fd_open_count.
	* opncls.c (_bfd_new_bfd): Initialize to -1.
	* plugin.c (bfd_plugin_open_input): Cache and reuse the archive
	plugin file descriptor.
	(bfd_plugin_close_file_descriptor): New function.
	(try_claim): Call bfd_plugin_close_file_descriptor.
	* plugin.h (bfd_plugin_close_file_descriptor): New.
	* bfd-in2.h: Regenerated.

ld/

	PR ld/28040
	* plugin.c (plugin_input_file): Add ibfd.
	(release_plugin_file_descriptor): New function.
	(release_input_file): Call release_plugin_file_descriptor to
	close input->fd.
	(plugin_object_p): Call release_plugin_file_descriptor to close
	input->fd.  Also call release_plugin_file_descriptor if not
	claimed.
	* testsuite/config/default.exp (RANLIB): New.
	* testsuite/ld-plugin/lto.exp: Run ranlib test.

(cherry picked from commit 918172470430ea6fa082c941e6789add88331197)

diff --git a/bfd/ChangeLog b/bfd/ChangeLog
index 15b00bb05f187d3cdf7610d92e98cd688ad827bc..78f84b149e03dce09e7a8411012f730c2c164dc5 100644 (file)
--- a/bfd/ChangeLog
+++ b/bfd/ChangeLog
@@ -1,3 +1,18 @@
+2021-07-05  H.J. Lu  <hongjiu.lu@intel.com>
+
+       PR ld/28040
+       * archive.c (_bfd_archive_close_and_cleanup): Close the archive
+       plugin file descriptor if needed.
+       * bfd.c (bfd): Add archive_plugin_fd and
+       archive_plugin_fd_open_count.
+       * opncls.c (_bfd_new_bfd): Initialize to -1.
+       * plugin.c (bfd_plugin_open_input): Cache and reuse the archive
+       plugin file descriptor.
+       (bfd_plugin_close_file_descriptor): New function.
+       (try_claim): Call bfd_plugin_close_file_descriptor.
+       * plugin.h (bfd_plugin_close_file_descriptor): New.
+       * bfd-in2.h: Regenerated.
+
 2021-07-05  Nick Clifton  <nickc@redhat.com>
 
        * po/fr.po: Updated French translation.

diff --git a/bfd/archive.c b/bfd/archive.c
index 2f3a1d4194c21fc0b0b6a8813bc00e145b3e6bcf..5f350b8e5ce3991ae1df6d81382afed2db7a2dba 100644 (file)
--- a/bfd/archive.c
+++ b/bfd/archive.c
@@ -2845,6 +2845,10 @@ _bfd_archive_close_and_cleanup (bfd *abfd)
          htab_delete (htab);
          bfd_ardata (abfd)->cache = NULL;
        }
+
+      /* Close the archive plugin file descriptor if needed.  */
+      if (abfd->archive_plugin_fd > 0)
+       close (abfd->archive_plugin_fd);
     }
 
   _bfd_unlink_from_archive_parent (abfd);

diff --git a/bfd/bfd-in2.h b/bfd/bfd-in2.h
index 57b3c45364930e6167dd7433c576586cdd476f70..9a698316980528610641b494a2ce642524c0ee3d 100644 (file)
--- a/bfd/bfd-in2.h
+++ b/bfd/bfd-in2.h
@@ -6707,6 +6707,12 @@ struct bfd
   /* The number of sections.  */
   unsigned int section_count;
 
+  /* The archive plugin file descriptor.  */
+  int archive_plugin_fd;
+
+  /* The number of opens on the archive plugin file descriptor.  */
+  unsigned int archive_plugin_fd_open_count;
+
   /* A field used by _bfd_generic_link_add_archive_symbols.  This will
      be used only for archive elements.  */
   int archive_pass;

diff --git a/bfd/bfd.c b/bfd/bfd.c
index 6e5d3397d151c738c95c07e8ade6f5b6c0075d9b..0952aaea19c64bfea1427cbaabee231b4a4ca8f6 100644 (file)
--- a/bfd/bfd.c
+++ b/bfd/bfd.c
@@ -276,6 +276,12 @@ CODE_FRAGMENT
 .  {* The number of sections.  *}
 .  unsigned int section_count;
 .
+.  {* The archive plugin file descriptor.  *}
+.  int archive_plugin_fd;
+.
+.  {* The number of opens on the archive plugin file descriptor.  *}
+.  unsigned int archive_plugin_fd_open_count;
+.
 .  {* A field used by _bfd_generic_link_add_archive_symbols.  This will
 .     be used only for archive elements.  *}
 .  int archive_pass;

diff --git a/bfd/opncls.c b/bfd/opncls.c
index 74df4c2f5183cc82570165b2bdc731decd8b1b33..4fb79324c0ad39a6606790f459b369157e00090a 100644 (file)
--- a/bfd/opncls.c
+++ b/bfd/opncls.c
@@ -89,6 +89,8 @@ _bfd_new_bfd (void)
       return NULL;
     }
 
+  nbfd->archive_plugin_fd = -1;
+
   return nbfd;
 }
 

diff --git a/bfd/plugin.c b/bfd/plugin.c
index 1fee4d0c87003012dac38d627fe52123d0838d9d..b3d6739dabbf172b439454358bfa2af62ef6ff18 100644 (file)
--- a/bfd/plugin.c
+++ b/bfd/plugin.c
@@ -192,6 +192,7 @@ int
 bfd_plugin_open_input (bfd *ibfd, struct ld_plugin_input_file *file)
 {
   bfd *iobfd;
+  int fd;
 
   iobfd = ibfd;
   while (iobfd->my_archive
@@ -202,50 +203,60 @@ bfd_plugin_open_input (bfd *ibfd, struct ld_plugin_input_file *file)
   if (!iobfd->iostream && !bfd_open_file (iobfd))
     return 0;
 
-  /* The plugin API expects that the file descriptor won't be closed
-     and reused as done by the bfd file cache.  So open it again.
-     dup isn't good enough.  plugin IO uses lseek/read while BFD uses
-     fseek/fread.  It isn't wise to mix the unistd and stdio calls on
-     the same underlying file descriptor.  */
-  file->fd = open (file->name, O_RDONLY | O_BINARY);
-  if (file->fd < 0)
+  /* Reuse the archive plugin file descriptor.  */
+  if (iobfd != ibfd)
+    fd = iobfd->archive_plugin_fd;
+  else
+    fd = -1;
+
+  if (fd < 0)
     {
+      /* The plugin API expects that the file descriptor won't be closed
+        and reused as done by the bfd file cache.  So open it again.
+        dup isn't good enough.  plugin IO uses lseek/read while BFD uses
+        fseek/fread.  It isn't wise to mix the unistd and stdio calls on
+        the same underlying file descriptor.  */
+      fd = open (file->name, O_RDONLY | O_BINARY);
+      if (fd < 0)
+       {
 #ifndef EMFILE
-      return 0;
+         return 0;
 #else
-      if (errno != EMFILE)
-       return 0;
+         if (errno != EMFILE)
+           return 0;
 
 #ifdef HAVE_GETRLIMIT
-      struct rlimit lim;
-
-      /* Complicated links involving lots of files and/or large archives
-        can exhaust the number of file descriptors available to us.
-        If possible, try to allocate more descriptors.  */
-      if (getrlimit (RLIMIT_NOFILE, & lim) == 0
-         && lim.rlim_cur < lim.rlim_max)
-       {
-         lim.rlim_cur = lim.rlim_max;
-         if (setrlimit (RLIMIT_NOFILE, &lim) == 0)
-           file->fd = open (file->name, O_RDONLY | O_BINARY);
-       }
+         struct rlimit lim;
+
+         /* Complicated links involving lots of files and/or large
+            archives can exhaust the number of file descriptors
+            available to us.  If possible, try to allocate more
+            descriptors.  */
+         if (getrlimit (RLIMIT_NOFILE, & lim) == 0
+             && lim.rlim_cur < lim.rlim_max)
+           {
+             lim.rlim_cur = lim.rlim_max;
+             if (setrlimit (RLIMIT_NOFILE, &lim) == 0)
+               fd = open (file->name, O_RDONLY | O_BINARY);
+           }
 
-      if (file->fd < 0)
+         if (fd < 0)
 #endif
-       {
-         _bfd_error_handler (_("plugin framework: out of file descriptors. Try using fewer objects/archives\n"));
-         return 0;
-       } 
+           {
+             _bfd_error_handler (_("plugin framework: out of file descriptors. Try using fewer objects/archives\n"));
+             return 0;
+           }
 #endif
-   }
+       }
+    }
 
   if (iobfd == ibfd)
     {
       struct stat stat_buf;
 
-      if (fstat (file->fd, &stat_buf))
+      if (fstat (fd, &stat_buf))
        {
-         close(file->fd);
+         close (fd);
          return 0;
        }
 
@@ -254,12 +265,44 @@ bfd_plugin_open_input (bfd *ibfd, struct ld_plugin_input_file *file)
     }
   else
     {
+      /* Cache the archive plugin file descriptor.  */
+      iobfd->archive_plugin_fd = fd;
+      iobfd->archive_plugin_fd_open_count++;
+
       file->offset = ibfd->origin;
       file->filesize = arelt_size (ibfd);
     }
+
+  file->fd = fd;
   return 1;
 }
 
+/* Close the plugin file descriptor.  */
+
+void
+bfd_plugin_close_file_descriptor (bfd *abfd, int fd)
+{
+  bfd *iobfd;
+
+  iobfd = abfd;
+  while (iobfd->my_archive
+        && !bfd_is_thin_archive (iobfd->my_archive))
+    iobfd = iobfd->my_archive;
+  if (iobfd == abfd)
+    close (fd);
+  else
+    {
+      iobfd->archive_plugin_fd_open_count--;
+      /* Dup the archive plugin file descriptor for later use, which
+        will be closed by _bfd_archive_close_and_cleanup.  */
+      if (iobfd->archive_plugin_fd_open_count == 0)
+       {
+         iobfd->archive_plugin_fd = dup (fd);
+         close (fd);
+       }
+    }
+}
+
 static int
 try_claim (bfd *abfd)
 {
@@ -271,7 +314,7 @@ try_claim (bfd *abfd)
       && current_plugin->claim_file)
     {
       current_plugin->claim_file (&file, &claimed);
-      close (file.fd);
+      bfd_plugin_close_file_descriptor (abfd, file.fd);
     }
 
   return claimed;

diff --git a/bfd/plugin.h b/bfd/plugin.h
index ce3fe5818eaf503fef95980756927045b135f8b3..709478e0065301a9e9791a5be68f222122b43147 100644 (file)
--- a/bfd/plugin.h
+++ b/bfd/plugin.h
@@ -28,6 +28,7 @@ bool bfd_plugin_target_p (const bfd_target *);
 bool bfd_plugin_specified_p (void);
 bool bfd_link_plugin_object_p (bfd *);
 void register_ld_plugin_object_p (bfd_cleanup (*object_p) (bfd *));
+void bfd_plugin_close_file_descriptor (bfd *, int);
 
 typedef struct plugin_data_struct
 {

diff --git a/ld/ChangeLog b/ld/ChangeLog
index 49296e02e0d1cf18c96f0a370c870a87d5040b5d..537b5df6311b41390cfc1f47ced3baa145711115 100644 (file)
--- a/ld/ChangeLog
+++ b/ld/ChangeLog
@@ -1,3 +1,16 @@
+2021-07-05  H.J. Lu  <hongjiu.lu@intel.com>
+
+       PR ld/28040
+       * plugin.c (plugin_input_file): Add ibfd.
+       (release_plugin_file_descriptor): New function.
+       (release_input_file): Call release_plugin_file_descriptor to
+       close input->fd.
+       (plugin_object_p): Call release_plugin_file_descriptor to close
+       input->fd.  Also call release_plugin_file_descriptor if not
+       claimed.
+       * testsuite/config/default.exp (RANLIB): New.
+       * testsuite/ld-plugin/lto.exp: Run ranlib test.
+
 2021-07-05  Nick Clifton  <nickc@redhat.com>
 
        * po/fr.po: Updated French translation.

diff --git a/ld/plugin.c b/ld/plugin.c
index 98a83bc9593e111d0502d339ffa60fdd55c32489..bb369f46955eb641ff9bfcc19b57e83f9e216e5e 100644 (file)
--- a/ld/plugin.c
+++ b/ld/plugin.c
@@ -103,6 +103,7 @@ typedef struct view_buffer
 typedef struct plugin_input_file
 {
   bfd *abfd;
+  bfd *ibfd;
   view_buffer_t view_buffer;
   char *name;
   int fd;
@@ -605,17 +606,25 @@ get_view (const void *handle, const void **viewp)
   return LDPS_OK;
 }
 
+/* Release plugin file descriptor.  */
+
+static void
+release_plugin_file_descriptor (plugin_input_file_t *input)
+{
+  if (input->fd != -1)
+    {
+      bfd_plugin_close_file_descriptor (input->ibfd, input->fd);
+      input->fd = -1;
+    }
+}
+
 /* Release the input file.  */
 static enum ld_plugin_status
 release_input_file (const void *handle)
 {
   plugin_input_file_t *input = (plugin_input_file_t *) handle;
   ASSERT (called_plugin);
-  if (input->fd != -1)
-    {
-      close (input->fd);
-      input->fd = -1;
-    }
+  release_plugin_file_descriptor (input);
   return LDPS_OK;
 }
 
@@ -1211,6 +1220,7 @@ plugin_object_p (bfd *ibfd)
 
   file.handle = input;
   input->abfd = abfd;
+  input->ibfd = ibfd;
   input->view_buffer.addr = NULL;
   input->view_buffer.filesize = 0;
   input->view_buffer.offset = 0;
@@ -1226,7 +1236,8 @@ plugin_object_p (bfd *ibfd)
     einfo (_("%F%P: %s: plugin reported error claiming file\n"),
           plugin_error_plugin ());
 
-  if (input->fd != -1 && !bfd_plugin_target_p (ibfd->xvec))
+  if (input->fd != -1
+      && (!claimed || !bfd_plugin_target_p (ibfd->xvec)))
     {
       /* FIXME: fd belongs to us, not the plugin.  GCC plugin, which
         doesn't need fd after plugin_call_claim_file, doesn't use
@@ -1236,8 +1247,7 @@ plugin_object_p (bfd *ibfd)
         release_input_file after it is done, uses BFD plugin target
         vector.  This scheme doesn't work when a plugin needs fd and
         doesn't use BFD plugin target vector neither.  */
-      close (input->fd);
-      input->fd = -1;
+      release_plugin_file_descriptor (input);
     }
 
   if (claimed)

diff --git a/ld/testsuite/config/default.exp b/ld/testsuite/config/default.exp
index 4090f1287c468b2f105e5cd374c4cc5f8c5b0d3b..0795ea81bfb6e52031f0540d5bdb9713cd8f1459 100644 (file)
--- a/ld/testsuite/config/default.exp
+++ b/ld/testsuite/config/default.exp
@@ -286,6 +286,10 @@ if ![info exists OBJCOPYFLAGS] then {
     set OBJCOPYFLAGS {}
 }
 
+if ![info exists RANLIB] then {
+    set RANLIB [findfile $base_dir/../binutils/ranlib]
+}
+
 if ![info exists READELF] then {
     set READELF [findfile $base_dir/../binutils/readelf]
 }

diff --git a/ld/testsuite/ld-plugin/lto.exp b/ld/testsuite/ld-plugin/lto.exp
index 2271211367b919b6e15f1cfab6096fd96a52d25b..def69e43ab3060907a02fbc0df7b0f929daa0e8d 100644 (file)
--- a/ld/testsuite/ld-plugin/lto.exp
+++ b/ld/testsuite/ld-plugin/lto.exp
@@ -786,6 +786,13 @@ if { [at_least_gcc_version 4 7] } {
     } else {
        fail "PR binutils/23460"
     }
+    set exec_output [run_host_cmd "$RANLIB" "$plug_opt tmpdir/libpr23460.a"]
+    set exec_output [prune_warnings $exec_output]
+    if [string match "" $exec_output] then {
+       pass "PR binutils/23460"
+    } else {
+       fail "PR binutils/23460"
+    }
 }
 
 # Run "ld -r" to generate inputs for complex LTO tests.