From fc8b3540d23168b5a35988b35336ba70c4174091 Mon Sep 17 00:00:00 2001 From: Ilya Verbin Date: Thu, 13 Nov 2014 13:47:46 +0000 Subject: [PATCH] [PATCH 4/7] OpenMP 4.0 offloading infrastructure: lto-wrapper. gcc/ * gcc.c (spec_host_machine, accel_dir_suffix): New variables. (process_command): Tweak path construction for the possibility of being configured as an offload compiler. (driver::maybe_putenv_OFFLOAD_TARGETS): New function. (driver::main): Call maybe_putenv_OFFLOAD_TARGETS. (driver::set_up_specs): Tweak path construction for the possibility of being configured as an offload compiler. * lto-wrapper.c (OFFLOAD_TARGET_NAMES_ENV): Define. (offload_names, offloadbegin, offloadend): New static variables. (free_array_of_ptrs, parse_env_var, access_check, compile_offload_image) (compile_images_for_offload_targets, copy_file, find_offloadbeginend): New static functions. (run_gcc): Determine whether offload sections are present. If so, run compile_images_for_offload_targets and return the names of new generated objects to linker. If there are offload sections, but no LTO sections, then return the copies of input objects without link-time recompilation. lto-plugin/ * lto-plugin.c (OFFLOAD_SECTION, OFFLOAD_SECTION_LEN): Define. (struct plugin_objfile): Add new field "offload". (process_offload_section): New static function. (claim_file_handler): Claim file if it contains offload sections. Co-Authored-By: Andrey Turetskiy Co-Authored-By: Bernd Schmidt Co-Authored-By: Michael Zolotukhin From-SVN: r217491 --- gcc/ChangeLog | 22 ++++ gcc/gcc.c | 50 ++++++-- gcc/lto-wrapper.c | 270 ++++++++++++++++++++++++++++++++++++++++ lto-plugin/ChangeLog | 10 ++ lto-plugin/lto-plugin.c | 25 +++- 5 files changed, 365 insertions(+), 12 deletions(-) diff --git a/gcc/ChangeLog b/gcc/ChangeLog index b70b758c1aa2..c40487d2d877 100644 --- a/gcc/ChangeLog +++ b/gcc/ChangeLog @@ -1,3 +1,25 @@ +2014-11-13 Ilya Verbin + Bernd Schmidt + Andrey Turetskiy + Michael Zolotukhin + + * gcc.c (spec_host_machine, accel_dir_suffix): New variables. + (process_command): Tweak path construction for the possibility + of being configured as an offload compiler. + (driver::maybe_putenv_OFFLOAD_TARGETS): New function. + (driver::main): Call maybe_putenv_OFFLOAD_TARGETS. + (driver::set_up_specs): Tweak path construction for the possibility of + being configured as an offload compiler. + * lto-wrapper.c (OFFLOAD_TARGET_NAMES_ENV): Define. + (offload_names, offloadbegin, offloadend): New static variables. + (free_array_of_ptrs, parse_env_var, access_check, compile_offload_image) + (compile_images_for_offload_targets, copy_file, find_offloadbeginend): + New static functions. + (run_gcc): Determine whether offload sections are present. If so, run + compile_images_for_offload_targets and return the names of new generated + objects to linker. If there are offload sections, but no LTO sections, + then return the copies of input objects without link-time recompilation. + 2014-11-13 Richard Biener * genmatch.c (dt_node::gen_kids): Fix placement of break statement. diff --git a/gcc/gcc.c b/gcc/gcc.c index 7e6af22a7c39..abadfbf2e930 100644 --- a/gcc/gcc.c +++ b/gcc/gcc.c @@ -157,6 +157,7 @@ static const char *const spec_version = DEFAULT_TARGET_VERSION; /* The target machine. */ static const char *spec_machine = DEFAULT_TARGET_MACHINE; +static const char *spec_host_machine = DEFAULT_REAL_TARGET_MACHINE; /* Nonzero if cross-compiling. When -b is used, the value comes from the `specs' file. */ @@ -1296,6 +1297,9 @@ static const char *const standard_startfile_prefix_2 relative to the driver. */ static const char *const tooldir_base_prefix = TOOLDIR_BASE_PREFIX; +/* A prefix to be used when this is an accelerator compiler. */ +static const char *const accel_dir_suffix = ACCEL_DIR_SUFFIX; + /* Subdirectory to use for locating libraries. Set by set_multilib_dir based on the compilation options. */ @@ -4129,15 +4133,15 @@ process_command (unsigned int decoded_options_count, } gcc_assert (!IS_ABSOLUTE_PATH (tooldir_base_prefix)); - tooldir_prefix2 = concat (tooldir_base_prefix, spec_machine, + tooldir_prefix2 = concat (tooldir_base_prefix, spec_host_machine, dir_separator_str, NULL); /* Look for tools relative to the location from which the driver is running, or, if that is not available, the configured prefix. */ tooldir_prefix = concat (gcc_exec_prefix ? gcc_exec_prefix : standard_exec_prefix, - spec_machine, dir_separator_str, - spec_version, dir_separator_str, tooldir_prefix2, NULL); + spec_host_machine, dir_separator_str, spec_version, + accel_dir_suffix, dir_separator_str, tooldir_prefix2, NULL); free (tooldir_prefix2); add_prefix (&exec_prefixes, @@ -6749,6 +6753,7 @@ class driver void set_up_specs () const; void putenv_COLLECT_GCC (const char *argv0) const; void maybe_putenv_COLLECT_LTO_WRAPPER () const; + void maybe_putenv_OFFLOAD_TARGETS () const; void handle_unrecognized_options () const; int maybe_print_and_exit () const; bool prepare_infiles (); @@ -6791,6 +6796,7 @@ driver::main (int argc, char **argv) set_up_specs (); putenv_COLLECT_GCC (argv[0]); maybe_putenv_COLLECT_LTO_WRAPPER (); + maybe_putenv_OFFLOAD_TARGETS (); handle_unrecognized_options (); if (!maybe_print_and_exit ()) @@ -6960,6 +6966,7 @@ driver::build_multilib_strings () const void driver::set_up_specs () const { + const char *spec_machine_suffix; char *specs_file; size_t i; @@ -6983,8 +6990,8 @@ driver::set_up_specs () const /* Read specs from a file if there is one. */ - machine_suffix = concat (spec_machine, dir_separator_str, - spec_version, dir_separator_str, NULL); + machine_suffix = concat (spec_host_machine, dir_separator_str, spec_version, + accel_dir_suffix, dir_separator_str, NULL); just_machine_suffix = concat (spec_machine, dir_separator_str, NULL); specs_file = find_a_file (&startfile_prefixes, "specs", R_OK, true); @@ -6994,13 +7001,18 @@ driver::set_up_specs () const else init_spec (); - /* We need to check standard_exec_prefix/just_machine_suffix/specs +#ifdef ACCEL_COMPILER + spec_machine_suffix = machine_suffix; +#else + spec_machine_suffix = just_machine_suffix; +#endif + + /* We need to check standard_exec_prefix/spec_machine_suffix/specs for any override of as, ld and libraries. */ specs_file = (char *) alloca (strlen (standard_exec_prefix) - + strlen (just_machine_suffix) + sizeof ("specs")); - + + strlen (spec_machine_suffix) + sizeof ("specs")); strcpy (specs_file, standard_exec_prefix); - strcat (specs_file, just_machine_suffix); + strcat (specs_file, spec_machine_suffix); strcat (specs_file, "specs"); if (access (specs_file, R_OK) == 0) read_specs (specs_file, true, false); @@ -7182,8 +7194,9 @@ driver::set_up_specs () const /* If we have a GCC_EXEC_PREFIX envvar, modify it for cpp's sake. */ if (gcc_exec_prefix) - gcc_exec_prefix = concat (gcc_exec_prefix, spec_machine, dir_separator_str, - spec_version, dir_separator_str, NULL); + gcc_exec_prefix = concat (gcc_exec_prefix, spec_host_machine, + dir_separator_str, spec_version, + accel_dir_suffix, dir_separator_str, NULL); /* Now we have the specs. Set the `valid' bits for switches that match anything in any spec. */ @@ -7234,6 +7247,21 @@ driver::maybe_putenv_COLLECT_LTO_WRAPPER () const } +/* Set up to remember the names of offload targets. */ + +void +driver::maybe_putenv_OFFLOAD_TARGETS () const +{ + if (strlen (OFFLOAD_TARGETS) > 0) + { + obstack_grow (&collect_obstack, "OFFLOAD_TARGET_NAMES=", + sizeof ("OFFLOAD_TARGET_NAMES=") - 1); + obstack_grow (&collect_obstack, OFFLOAD_TARGETS, + strlen (OFFLOAD_TARGETS) + 1); + xputenv (XOBFINISH (&collect_obstack, char *)); + } +} + /* Reject switches that no pass was interested in. */ void diff --git a/gcc/lto-wrapper.c b/gcc/lto-wrapper.c index 8033b155baa6..cbda36bface3 100644 --- a/gcc/lto-wrapper.c +++ b/gcc/lto-wrapper.c @@ -49,6 +49,10 @@ along with GCC; see the file COPYING3. If not see #include "lto-section-names.h" #include "collect-utils.h" +/* Environment variable, used for passing the names of offload targets from GCC + driver to lto-wrapper. */ +#define OFFLOAD_TARGET_NAMES_ENV "OFFLOAD_TARGET_NAMES" + enum lto_mode_d { LTO_MODE_NONE, /* Not doing LTO. */ LTO_MODE_LTO, /* Normal LTO. */ @@ -63,6 +67,8 @@ static char *flto_out; static unsigned int nr; static char **input_names; static char **output_names; +static char **offload_names; +static const char *offloadbegin, *offloadend; static char *makefile; const char tool_name[] = "lto-wrapper"; @@ -364,6 +370,223 @@ merge_and_complain (struct cl_decoded_option **decoded_options, } } +/* Auxiliary function that frees elements of PTR and PTR itself. + N is number of elements to be freed. If PTR is NULL, nothing is freed. + If an element is NULL, subsequent elements are not freed. */ + +static void ** +free_array_of_ptrs (void **ptr, unsigned n) +{ + if (!ptr) + return NULL; + for (unsigned i = 0; i < n; i++) + { + if (!ptr[i]) + break; + free (ptr[i]); + } + free (ptr); + return NULL; +} + +/* Parse STR, saving found tokens into PVALUES and return their number. + Tokens are assumed to be delimited by ':'. If APPEND is non-null, + append it to every token we find. */ + +static unsigned +parse_env_var (const char *str, char ***pvalues, const char *append) +{ + const char *curval, *nextval; + char **values; + unsigned num = 1, i; + + curval = strchr (str, ':'); + while (curval) + { + num++; + curval = strchr (curval + 1, ':'); + } + + values = (char**) xmalloc (num * sizeof (char*)); + curval = str; + nextval = strchrnul (curval, ':'); + + int append_len = append ? strlen (append) : 0; + for (i = 0; i < num; i++) + { + int l = nextval - curval; + values[i] = (char*) xmalloc (l + 1 + append_len); + memcpy (values[i], curval, l); + values[i][l] = 0; + if (append) + strcat (values[i], append); + curval = nextval + 1; + nextval = strchrnul (curval, ':'); + } + *pvalues = values; + return num; +} + +/* Check whether NAME can be accessed in MODE. This is like access, + except that it never considers directories to be executable. */ + +static int +access_check (const char *name, int mode) +{ + if (mode == X_OK) + { + struct stat st; + + if (stat (name, &st) < 0 + || S_ISDIR (st.st_mode)) + return -1; + } + + return access (name, mode); +} + +/* Prepare a target image for offload TARGET, using mkoffload tool from + COMPILER_PATH. Return the name of the resultant object file. */ + +static char * +compile_offload_image (const char *target, const char *compiler_path, + unsigned in_argc, char *in_argv[]) +{ + char *filename = NULL; + char **argv; + char *suffix + = XALLOCAVEC (char, sizeof ("/accel//mkoffload") + strlen (target)); + strcpy (suffix, "/accel/"); + strcat (suffix, target); + strcat (suffix, "/mkoffload"); + + char **paths = NULL; + unsigned n_paths = parse_env_var (compiler_path, &paths, suffix); + + const char *compiler = NULL; + for (unsigned i = 0; i < n_paths; i++) + if (access_check (paths[i], X_OK) == 0) + { + compiler = paths[i]; + break; + } + + if (compiler) + { + /* Generate temporary output file name. */ + filename = make_temp_file (".target.o"); + + struct obstack argv_obstack; + obstack_init (&argv_obstack); + obstack_ptr_grow (&argv_obstack, compiler); + obstack_ptr_grow (&argv_obstack, "-o"); + obstack_ptr_grow (&argv_obstack, filename); + + for (unsigned i = 1; i < in_argc; i++) + obstack_ptr_grow (&argv_obstack, in_argv[i]); + obstack_ptr_grow (&argv_obstack, NULL); + + argv = XOBFINISH (&argv_obstack, char **); + fork_execute (argv[0], argv, true); + obstack_free (&argv_obstack, NULL); + } + + free_array_of_ptrs ((void **) paths, n_paths); + return filename; +} + + +/* The main routine dealing with offloading. + The routine builds a target image for each offload target. IN_ARGC and + IN_ARGV specify options and input object files. As all of them could contain + target sections, we pass them all to target compilers. */ + +static void +compile_images_for_offload_targets (unsigned in_argc, char *in_argv[]) +{ + char **names = NULL; + const char *target_names = getenv (OFFLOAD_TARGET_NAMES_ENV); + if (!target_names) + return; + unsigned num_targets = parse_env_var (target_names, &names, NULL); + + const char *compiler_path = getenv ("COMPILER_PATH"); + if (!compiler_path) + goto out; + + /* Prepare an image for each target and save the name of the resultant object + file to the OFFLOAD_NAMES array. It is terminated by a NULL entry. */ + offload_names = XCNEWVEC (char *, num_targets + 1); + for (unsigned i = 0; i < num_targets; i++) + { + offload_names[i] = compile_offload_image (names[i], compiler_path, + in_argc, in_argv); + if (!offload_names[i]) + fatal_error ("problem with building target image for %s\n", names[i]); + } + + out: + free_array_of_ptrs ((void **) names, num_targets); +} + +/* Copy a file from SRC to DEST. */ + +static void +copy_file (const char *dest, const char *src) +{ + FILE *d = fopen (dest, "wb"); + FILE *s = fopen (src, "rb"); + char buffer[512]; + while (!feof (s)) + { + size_t len = fread (buffer, 1, 512, s); + if (ferror (s) != 0) + fatal_error ("reading input file"); + if (len > 0) + { + fwrite (buffer, 1, len, d); + if (ferror (d) != 0) + fatal_error ("writing output file"); + } + } +} + +/* Find the crtoffloadbegin.o and crtoffloadend.o files in LIBRARY_PATH, make + copies and store the names of the copies in offloadbegin and offloadend. */ + +static void +find_offloadbeginend (void) +{ + char **paths = NULL; + const char *library_path = getenv ("LIBRARY_PATH"); + if (!library_path) + return; + unsigned n_paths = parse_env_var (library_path, &paths, "/crtoffloadbegin.o"); + + unsigned i; + for (i = 0; i < n_paths; i++) + if (access_check (paths[i], R_OK) == 0) + { + size_t len = strlen (paths[i]); + char *tmp = xstrdup (paths[i]); + strcpy (paths[i] + len - strlen ("begin.o"), "end.o"); + if (access_check (paths[i], R_OK) != 0) + fatal_error ("installation error, can't find crtoffloadend.o"); + /* The linker will delete the filenames we give it, so make + copies. */ + offloadbegin = make_temp_file (".o"); + offloadend = make_temp_file (".o"); + copy_file (offloadbegin, tmp); + copy_file (offloadend, paths[i]); + free (tmp); + break; + } + if (i == n_paths) + fatal_error ("installation error, can't find crtoffloadbegin.o"); + + free_array_of_ptrs ((void **) paths, n_paths); +} + /* Execute gcc. ARGC is the number of arguments. ARGV contains the arguments. */ static void @@ -384,6 +607,8 @@ run_gcc (unsigned argc, char *argv[]) unsigned int decoded_options_count; struct obstack argv_obstack; int new_head_argc; + bool have_lto = false; + bool have_offload = false; /* Get the driver and options. */ collect_gcc = getenv ("COLLECT_GCC"); @@ -432,6 +657,9 @@ run_gcc (unsigned argc, char *argv[]) close (fd); continue; } + if (simple_object_find_section (sobj, OFFLOAD_SECTION_NAME_PREFIX ".opts", + &offset, &length, &errmsg, &err)) + have_offload = true; if (!simple_object_find_section (sobj, LTO_SECTION_NAME_PREFIX "." "opts", &offset, &length, &errmsg, &err)) { @@ -439,6 +667,7 @@ run_gcc (unsigned argc, char *argv[]) close (fd); continue; } + have_lto = true; lseek (fd, file_offset + offset, SEEK_SET); data = (char *)xmalloc (length); read (fd, data, length); @@ -633,6 +862,43 @@ run_gcc (unsigned argc, char *argv[]) /* Remember at which point we can scrub args to re-use the commons. */ new_head_argc = obstack_object_size (&argv_obstack) / sizeof (void *); + if (have_offload) + { + compile_images_for_offload_targets (argc, argv); + if (offload_names) + { + find_offloadbeginend (); + for (i = 0; offload_names[i]; i++) + printf ("%s\n", offload_names[i]); + free_array_of_ptrs ((void **) offload_names, i); + } + } + + if (offloadbegin) + printf ("%s\n", offloadbegin); + + /* If object files contain offload sections, but do not contain LTO sections, + then there is no need to perform a link-time recompilation, i.e. + lto-wrapper is used only for a compilation of offload images. */ + if (have_offload && !have_lto) + { + for (i = 1; i < argc; ++i) + if (strncmp (argv[i], "-fresolution=", sizeof ("-fresolution=") - 1)) + { + char *out_file; + /* Can be ".o" or ".so". */ + char *ext = strrchr (argv[i], '.'); + if (ext == NULL) + out_file = make_temp_file (""); + else + out_file = make_temp_file (ext); + /* The linker will delete the files we give it, so make copies. */ + copy_file (out_file, argv[i]); + printf ("%s\n", out_file); + } + goto finish; + } + if (lto_mode == LTO_MODE_LTO) { flto_out = make_temp_file (".lto.o"); @@ -859,6 +1125,10 @@ cont: obstack_free (&env_obstack, NULL); } + finish: + if (offloadend) + printf ("%s\n", offloadend); + obstack_free (&argv_obstack, NULL); } diff --git a/lto-plugin/ChangeLog b/lto-plugin/ChangeLog index 195ab453a4a2..69af7814e6f1 100644 --- a/lto-plugin/ChangeLog +++ b/lto-plugin/ChangeLog @@ -1,3 +1,13 @@ +2014-11-13 Ilya Verbin + Bernd Schmidt + Andrey Turetskiy + Michael Zolotukhin + + * lto-plugin.c (OFFLOAD_SECTION, OFFLOAD_SECTION_LEN): Define. + (struct plugin_objfile): Add new field "offload". + (process_offload_section): New static function. + (claim_file_handler): Claim file if it contains offload sections. + 2014-11-13 Bernd Schmidt Thomas Schwinge Ilya Verbin diff --git a/lto-plugin/lto-plugin.c b/lto-plugin/lto-plugin.c index 910e23cd6092..fb6555daf911 100644 --- a/lto-plugin/lto-plugin.c +++ b/lto-plugin/lto-plugin.c @@ -86,6 +86,8 @@ along with this program; see the file COPYING3. If not see #define LTO_SECTION_PREFIX ".gnu.lto_.symtab" #define LTO_SECTION_PREFIX_LEN (sizeof (LTO_SECTION_PREFIX) - 1) +#define OFFLOAD_SECTION ".gnu.offload_lto_.opts" +#define OFFLOAD_SECTION_LEN (sizeof (OFFLOAD_SECTION) - 1) /* The part of the symbol table the plugin has to keep track of. Note that we must keep SYMS until all_symbols_read is called to give the linker time to @@ -111,6 +113,7 @@ struct plugin_symtab struct plugin_objfile { int found; + int offload; simple_object_read *objfile; struct plugin_symtab *out; const struct ld_plugin_input_file *file; @@ -862,6 +865,21 @@ err: return 0; } +/* Find an offload section of an object file. */ + +static int +process_offload_section (void *data, const char *name, off_t offset, off_t len) +{ + if (!strncmp (name, OFFLOAD_SECTION, OFFLOAD_SECTION_LEN)) + { + struct plugin_objfile *obj = (struct plugin_objfile *) data; + obj->offload = 1; + return 0; + } + + return 1; +} + /* Callback used by gold to check if the plugin will claim FILE. Writes the result in CLAIMED. */ @@ -899,6 +917,7 @@ claim_file_handler (const struct ld_plugin_input_file *file, int *claimed) *claimed = 0; obj.file = file; obj.found = 0; + obj.offload = 0; obj.out = <o_file.symtab; errmsg = NULL; obj.objfile = simple_object_start_read (file->fd, file->offset, LTO_SEGMENT_NAME, @@ -920,7 +939,11 @@ claim_file_handler (const struct ld_plugin_input_file *file, int *claimed) goto err; } - if (obj.found == 0) + if (obj.objfile) + simple_object_find_sections (obj.objfile, process_offload_section, + &obj, &err); + + if (obj.found == 0 && obj.offload == 0) goto err; if (obj.found > 1) -- 2.39.2