mutex.c proc.c sem.c bar.c ptrlock.c time.c fortran.c affinity.c \
target.c splay-tree.c libgomp-plugin.c oacc-parallel.c oacc-host.c \
oacc-init.c oacc-mem.c oacc-async.c oacc-plugin.c oacc-cuda.c \
- priority_queue.c affinity-fmt.c teams.c allocator.c oacc-profiling.c \
- oacc-target.c target-indirect.c target-cxa-dso-dtor.c
+ priority_queue.c affinity-fmt.c teams.c allocator.c simple-allocator.c \
+ oacc-profiling.c oacc-target.c target-indirect.c target-cxa-dso-dtor.c
include $(top_srcdir)/plugin/Makefrag.am
affinity.lo target.lo splay-tree.lo libgomp-plugin.lo \
oacc-parallel.lo oacc-host.lo oacc-init.lo oacc-mem.lo \
oacc-async.lo oacc-plugin.lo oacc-cuda.lo priority_queue.lo \
- affinity-fmt.lo teams.lo allocator.lo oacc-profiling.lo \
- oacc-target.lo target-indirect.lo target-cxa-dso-dtor.lo \
- $(am__objects_1)
+ affinity-fmt.lo teams.lo allocator.lo simple-allocator.lo \
+ oacc-profiling.lo oacc-target.lo target-indirect.lo \
+ target-cxa-dso-dtor.lo $(am__objects_1)
libgomp_la_OBJECTS = $(am_libgomp_la_OBJECTS)
AM_V_P = $(am__v_P_@AM_V@)
am__v_P_ = $(am__v_P_@AM_DEFAULT_V@)
fortran.c affinity.c target.c splay-tree.c libgomp-plugin.c \
oacc-parallel.c oacc-host.c oacc-init.c oacc-mem.c \
oacc-async.c oacc-plugin.c oacc-cuda.c priority_queue.c \
- affinity-fmt.c teams.c allocator.c oacc-profiling.c \
- oacc-target.c target-indirect.c target-cxa-dso-dtor.c \
- $(am__append_3)
+ affinity-fmt.c teams.c allocator.c simple-allocator.c \
+ oacc-profiling.c oacc-target.c target-indirect.c \
+ target-cxa-dso-dtor.c $(am__append_3)
# Nvidia PTX OpenACC plugin.
@PLUGIN_NVPTX_TRUE@libgomp_plugin_nvptx_version_info = -version-info $(libtool_VERSION)
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/scope.Plo@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/sections.Plo@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/sem.Plo@am__quote@
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/simple-allocator.Plo@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/single.Plo@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/splay-tree.Plo@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/target-cxa-dso-dtor.Plo@am__quote@
/* This is a basic "malloc" implementation intended for use with small,
low-latency memories.
+ Compared to the "simple" allocator, this one is designed to keep the
+ metadata and heap together (no slow memory needed), and prioritize
+ space-efficiency over algorithm speed (the memory already being
+ low-latency).
+
To use this template, define BASIC_ALLOC_PREFIX, and then #include the
source file. The other configuration macros are optional.
#define _GNU_SOURCE
#include <sys/mman.h>
+#include <unistd.h>
#include <string.h>
#include <assert.h>
#include "libgomp.h"
static int using_device_for_page_locked
= /* uninitialized */ -1;
+
+static gomp_simple_alloc_ctx_p pin_ctx = NULL;
+static pthread_once_t ctxlock = PTHREAD_ONCE_INIT;
+
+static void
+linux_init_pin_ctx ()
+{
+ pin_ctx = gomp_simple_alloc_init_context ();
+}
+
static void *
linux_memspace_alloc (omp_memspace_handle_t memspace, size_t size, int pin,
bool init0)
{
- void *addr;
+ void *addr = NULL;
if (pin)
{
}
if (using_device == 0)
{
- addr = mmap (NULL, size, PROT_READ | PROT_WRITE,
- MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
- if (addr == MAP_FAILED)
- addr = NULL;
- else
- {
- /* 'mmap' zero-initializes. */
- init0 = false;
+ static int pagesize = 0;
+ static void *addrhint = NULL;
- if (mlock (addr, size))
+ if (!pagesize)
+ pagesize = sysconf(_SC_PAGE_SIZE);
+
+ while (1)
+ {
+ addr = gomp_simple_alloc (pin_ctx, size);
+ if (addr)
+ break;
+
+ /* Round up to a whole page. */
+ size_t misalignment = size % pagesize;
+ size_t mmap_size = (misalignment > 0
+ ? size + pagesize - misalignment
+ : size);
+ void *newpage = mmap (addrhint, mmap_size, PROT_READ | PROT_WRITE,
+ MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
+ if (newpage == MAP_FAILED)
+ break;
+ else
{
+ if (mlock (newpage, size))
+ {
#ifdef HAVE_INTTYPES_H
- gomp_debug (0, "libgomp: failed to pin %"PRIu64" bytes of"
- " memory (ulimit too low?)\n", (uint64_t) size);
+ gomp_debug (0, "libgomp: failed to pin %"PRIu64" bytes"
+ " of memory (ulimit too low?)\n",
+ (uint64_t) size);
#else
- gomp_debug (0, "libgomp: failed to pin %lu bytes of memory"
- " (ulimit too low?)\n", (unsigned long) size);
+ gomp_debug (0, "libgomp: failed to pin %lu bytes of"
+ " memory (ulimit too low?)\n",
+ (unsigned long) size);
#endif
- munmap (addr, size);
- addr = NULL;
+ munmap (newpage, size);
+ break;
+ }
+
+ addrhint = newpage + mmap_size;
+
+ pthread_once (&ctxlock, linux_init_pin_ctx);
+ gomp_simple_alloc_register_memory (pin_ctx, newpage,
+ mmap_size);
}
}
}
if (using_device == 1)
gomp_page_locked_host_free (addr);
else
- /* 'munlock'ing is implicit with following 'munmap'. */
- munmap (addr, size);
+ /* The "simple" allocator does not (currently) munmap locked pages
+ (meaning that the number of locked pages never decreases), but it
+ can reuse the freed memory in subsequent gomp_simple_alloc calls. */
+ gomp_simple_free (pin_ctx, addr);
}
else
free (addr);
{
if (oldpin && pin)
{
- /* We can only expect to be able to just 'mremap' if not using a device
- for page-locked memory. */
int using_device
= __atomic_load_n (&using_device_for_page_locked,
MEMMODEL_RELAXED);
- if (using_device != 0)
- goto manual_realloc;
-
- void *newaddr = mremap (addr, oldsize, size, MREMAP_MAYMOVE);
- if (newaddr == MAP_FAILED)
- return NULL;
-
- return newaddr;
+ /* The device plugin API does not support realloc,
+ but the gomp_simple_alloc allocator does. */
+ if (using_device == 0)
+ {
+ /* This can fail if there is insufficient pinned memory free. */
+ void *newaddr = gomp_simple_realloc (pin_ctx, addr, size);
+ if (newaddr)
+ return newaddr;
+ }
}
else if (oldpin || pin)
- goto manual_realloc;
+ /* Moving from pinned to unpinned memory cannot be done in-place. */
+ ;
else
return realloc (addr, size);
-manual_realloc:;
+ /* In-place reallocation failed. Fall back to copy. */
void *newaddr = linux_memspace_alloc (memspace, size, pin, false);
if (newaddr)
{
}
#endif
+/* simple-allocator.c */
+
+typedef struct gomp_simple_alloc_context *gomp_simple_alloc_ctx_p;
+
+gomp_simple_alloc_ctx_p gomp_simple_alloc_init_context ();
+void gomp_simple_alloc_register_memory (gomp_simple_alloc_ctx_p ctx,
+ char *base, size_t size);
+void *gomp_simple_alloc (gomp_simple_alloc_ctx_p ctx, size_t size);
+void gomp_simple_free (gomp_simple_alloc_ctx_p ctx, void *addr);
+void *gomp_simple_realloc (gomp_simple_alloc_ctx_p ctx, void *addr,
+ size_t newsize);
+
#endif /* LIBGOMP_H */
@itemize
@item The @code{pinned} trait is supported on Linux hosts, but is usually
subject to the OS @code{ulimit}/@code{rlimit} locked memory settings (see
- @ref{Offload-Target Specifics} for exceptions). It currently uses
- @code{mmap} and is therefore optimized for few allocations, including
- large data.
+ @ref{Offload-Target Specifics} for exceptions). The implementation
+ uses a custom allocator to try to use as few memory pages as possible.
+ At present, freed pinned memory is not returned to the OS (although it
+ may be reused by subsequent pinned allocations).
@item The default for the @code{pool_size} trait is no pool and for every
(re)allocation the associated library routine is called, which might
internally use a memory pool. Currently, the same applies when a
--- /dev/null
+/* Copyright (C) 2025 Free Software Foundation, Inc.
+
+ This file is part of the GNU Offloading and Multi Processing Library
+ (libgomp).
+
+ Libgomp is free software; you can redistribute it and/or modify it
+ under the terms of the GNU General Public License as published by
+ the Free Software Foundation; either version 3, or (at your option)
+ any later version.
+
+ Libgomp is distributed in the hope that it will be useful, but WITHOUT ANY
+ WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+ FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ more details.
+
+ Under Section 7 of GPL version 3, you are granted additional
+ permissions described in the GCC Runtime Library Exception, version
+ 3.1, as published by the Free Software Foundation.
+
+ You should have received a copy of the GNU General Public License and
+ a copy of the GCC Runtime Library Exception along with this program;
+ see the files COPYING3 and COPYING.RUNTIME respectively. If not, see
+ <http://www.gnu.org/licenses/>. */
+
+/* This is a simple "malloc" implementation intended for use with device
+ Managed Memory and Pinned Memory. It allocates memory from a pool allocated
+ and configured by the device plugin, or the OS-specific allocator
+ (for pinned).
+
+ Unlike the "basic" allocator, this implementation keeps the allocated/free
+ chain in a side-table (splay tree) to ensure that the allocation routine
+ does not have the side-effect of migrating all the managed memory pages back
+ into host memory. Keeping the meta-data elsewhere is also useful for pinned
+ memory, which is typically an extremely limited resource. */
+
+#include <string.h>
+#include "libgomp.h"
+
+/* Use a splay tree to track allocations. */
+
+typedef struct simple_alloc_splay_tree_node_s *simple_alloc_splay_tree_node;
+typedef struct simple_alloc_splay_tree_s *simple_alloc_splay_tree;
+typedef struct simple_alloc_splay_tree_key_s *simple_alloc_splay_tree_key;
+
+struct simple_alloc_splay_tree_key_s {
+ void *base;
+ size_t size;
+};
+
+static inline int
+simple_alloc_splay_compare (simple_alloc_splay_tree_key x,
+ simple_alloc_splay_tree_key y)
+{
+ return (x->base == y->base ? 0
+ : x->base > y->base ? 1
+ : -1);
+}
+
+#define splay_tree_prefix simple_alloc
+#include "splay-tree.h"
+
+/* 128-byte granularity means GPU cache-line aligned. */
+#define ALIGN(VAR) (((VAR) + 127) & ~127)
+
+/* The context data prevents the need for global state. */
+struct gomp_simple_alloc_context {
+ struct simple_alloc_splay_tree_s allocations;
+ struct simple_alloc_splay_tree_s free_space;
+ gomp_mutex_t lock;
+};
+
+gomp_simple_alloc_ctx_p
+gomp_simple_alloc_init_context ()
+{
+ return calloc (1, sizeof (struct gomp_simple_alloc_context));
+}
+
+/* Coalesce contiguous free space into one entry. This considers the entries
+ either side of the root node only, so it should be called each time a new
+ entry in inserted into the root. */
+
+static void
+simple_alloc_coalesce_free_space (gomp_simple_alloc_ctx_p ctx)
+{
+ simple_alloc_splay_tree_node prev, next, node = ctx->free_space.root;
+
+ for (prev = node->left; prev && prev->right; prev = prev->right)
+ ;
+ for (next = node->right; next && next->left; next = next->left)
+ ;
+
+ /* Coalesce adjacent free chunks. */
+ if (next
+ && node->key.base + node->key.size == next->key.base)
+ {
+ /* Free chunk follows. */
+ node->key.size += next->key.size;
+ simple_alloc_splay_tree_remove (&ctx->free_space, &next->key);
+ free (next);
+ }
+ if (prev
+ && prev->key.base + prev->key.size == node->key.base)
+ {
+ /* Free chunk precedes. */
+ prev->key.size += node->key.size;
+ simple_alloc_splay_tree_remove (&ctx->free_space, &node->key);
+ free (node);
+ }
+}
+
+/* Add a new memory region into the free chain. This is how our heap is
+ initialized and extended (using memory acquired by an external caller). If
+ the new region is contiguous with an existing region then any free space
+ will be coalesced. */
+
+void
+gomp_simple_alloc_register_memory (gomp_simple_alloc_ctx_p ctx, char *base,
+ size_t size)
+{
+ if (base == NULL || ctx == NULL)
+ return;
+
+ gomp_mutex_lock (&ctx->lock);
+
+ simple_alloc_splay_tree_node node;
+ node = malloc (sizeof (struct simple_alloc_splay_tree_node_s));
+ node->key.base = base;
+ node->key.size = size;
+ node->left = NULL;
+ node->right = NULL;
+ simple_alloc_splay_tree_insert (&ctx->free_space, node);
+ simple_alloc_coalesce_free_space (ctx);
+
+ gomp_mutex_unlock (&ctx->lock);
+}
+
+/* This splay_tree_foreach callback selects the first free space large enough
+ to hold the allocation needed. Since the splay_tree walk may start in the
+ middle the "first" isn't necessarily the "leftmost" entry. */
+
+struct simple_alloc_callback_data {
+ size_t size;
+ simple_alloc_splay_tree_node found;
+};
+
+static int
+simple_alloc_callback (simple_alloc_splay_tree_key key, void *data)
+{
+ struct simple_alloc_callback_data *cbd
+ = (struct simple_alloc_callback_data *)data;
+
+ if (key->size >= cbd->size)
+ {
+ cbd->found = (simple_alloc_splay_tree_node)key;
+ return 1;
+ }
+
+ return 0;
+}
+
+/* Simple "malloc". Selects and moves and address range from ctx->free_space to
+ ctx->allocations, while leaving any excess in ctx->free_space. */
+
+void *
+gomp_simple_alloc (gomp_simple_alloc_ctx_p ctx, size_t size)
+{
+ if (ctx == NULL)
+ return NULL;
+
+ /* Memory is allocated in N-byte granularity. */
+ size = ALIGN (size);
+
+ gomp_mutex_lock (&ctx->lock);
+
+ if (!ctx->free_space.root)
+ {
+ /* No memory registered, or no free space. */
+ gomp_mutex_unlock (&ctx->lock);
+ return NULL;
+ }
+
+ /* Find a suitable free block. */
+ struct simple_alloc_callback_data cbd = {size, NULL};
+ simple_alloc_splay_tree_foreach_lazy (&ctx->free_space,
+ simple_alloc_callback, &cbd);
+ simple_alloc_splay_tree_node freenode = cbd.found;
+
+ void *result = NULL;
+ if (freenode)
+ {
+ /* Allocation successful. */
+ result = freenode->key.base;
+ simple_alloc_splay_tree_node allocnode = malloc (sizeof (*allocnode));
+ allocnode->key.base = result;
+ allocnode->key.size = size;
+ allocnode->left = NULL;
+ allocnode->right = NULL;
+ simple_alloc_splay_tree_insert (&ctx->allocations, allocnode);
+
+ /* Update the free chain. */
+ size_t stillfree_size = freenode->key.size - size;
+ if (stillfree_size > 0)
+ {
+ freenode->key.base = freenode->key.base + size;
+ freenode->key.size = stillfree_size;
+ }
+ else
+ {
+ simple_alloc_splay_tree_remove (&ctx->free_space, &freenode->key);
+ free (freenode);
+ }
+ }
+
+ gomp_mutex_unlock (&ctx->lock);
+
+ return result;
+}
+
+/* Simple "free". Moves an address range from ctx->allocations to
+ ctx->free_space and merges that record with any contiguous free memory. */
+
+void
+gomp_simple_free (gomp_simple_alloc_ctx_p ctx, void *addr)
+{
+ if (ctx == NULL)
+ return;
+
+ gomp_mutex_lock (&ctx->lock);
+
+ /* Convert the memory map to free. */
+ struct simple_alloc_splay_tree_key_s key = {addr};
+ simple_alloc_splay_tree_key found
+ = simple_alloc_splay_tree_lookup (&ctx->allocations, &key);
+ if (!found)
+ GOMP_PLUGIN_fatal ("invalid free");
+ simple_alloc_splay_tree_remove (&ctx->allocations, &key);
+ simple_alloc_splay_tree_insert (&ctx->free_space,
+ (simple_alloc_splay_tree_node)found);
+ simple_alloc_coalesce_free_space (ctx);
+
+ gomp_mutex_unlock (&ctx->lock);
+}
+
+/* Simple "realloc". Works in-place, if possible; reallocates otherwise. */
+
+void *
+gomp_simple_realloc (gomp_simple_alloc_ctx_p ctx, void *addr, size_t newsize)
+{
+ if (ctx == NULL)
+ return NULL;
+
+ newsize = ALIGN (newsize);
+
+ gomp_mutex_lock (&ctx->lock);
+
+ /* Convert the memory map to free. */
+ struct simple_alloc_splay_tree_key_s key = {addr};
+ simple_alloc_splay_tree_key found
+ = simple_alloc_splay_tree_lookup (&ctx->allocations, &key);
+ if (!found)
+ GOMP_PLUGIN_fatal ("invalid realloc");
+
+ if (newsize == found->size)
+ ; /* Nothing to do. */
+ else if (newsize < found->size)
+ {
+ /* We're reducing the allocation size. */
+ simple_alloc_splay_tree_node newfree = malloc (sizeof (*newfree));
+ newfree->key.base = found->base + newsize;
+ newfree->key.size = found->size - newsize;
+ newfree->left = NULL;
+ newfree->right = NULL;
+ simple_alloc_splay_tree_insert (&ctx->free_space, newfree);
+ simple_alloc_coalesce_free_space (ctx);
+ }
+ else
+ {
+ /* We're extending the allocation. */
+ struct simple_alloc_splay_tree_key_s freekey = {addr + found->size};
+ simple_alloc_splay_tree_key foundfree;
+ foundfree = simple_alloc_splay_tree_lookup (&ctx->free_space, &freekey);
+ if (foundfree && foundfree->size >= newsize - found->size)
+ {
+ /* Allocation can be expanded in place. */
+ foundfree->base += found->size;
+ foundfree->size -= newsize - found->size;
+ found->size = newsize;
+
+ if (foundfree->size == 0)
+ simple_alloc_splay_tree_remove (&ctx->free_space, &freekey);
+ }
+ else
+ {
+ /* Allocation must be relocated.
+ Release the lock and use alloc/free. */
+ gomp_mutex_unlock (&ctx->lock);
+
+ void *newaddr = gomp_simple_alloc (ctx, newsize);
+ if (!newaddr)
+ return NULL;
+
+ memcpy (newaddr, addr, found->size);
+ gomp_simple_free (ctx, addr);
+ return newaddr;
+ }
+ }
+
+ gomp_mutex_unlock (&ctx->lock);
+ return addr;
+}
+
+/* Include the splay tree code inline, with the prefixes added. */
+#define splay_tree_prefix simple_alloc
+#define splay_tree_c
+#include "splay-tree.h"
--- /dev/null
+/* { dg-do run } */
+
+/* { dg-skip-if "Pinning not implemented on this host" { ! *-*-linux-gnu* } } */
+
+/* { dg-additional-options -DOFFLOAD_DEVICE_NVPTX { target offload_device_nvptx } } */
+
+/* Test that pinned memory works for small allocations. */
+
+#include <stdio.h>
+#include <stdlib.h>
+
+#ifdef __linux__
+#include <sys/types.h>
+#include <unistd.h>
+
+#include <sys/mman.h>
+#include <sys/resource.h>
+
+#define PAGE_SIZE sysconf(_SC_PAGESIZE)
+#define CHECK_SIZE(SIZE) { \
+ struct rlimit limit; \
+ if (getrlimit (RLIMIT_MEMLOCK, &limit) \
+ || limit.rlim_cur <= SIZE) \
+ fprintf (stderr, "insufficient lockable memory; please increase ulimit\n"); \
+ }
+
+int
+get_pinned_mem ()
+{
+ int pid = getpid ();
+ char buf[100];
+ sprintf (buf, "/proc/%d/status", pid);
+
+ FILE *proc = fopen (buf, "r");
+ if (!proc)
+ abort ();
+ while (fgets (buf, 100, proc))
+ {
+ int val;
+ if (sscanf (buf, "VmLck: %d", &val))
+ {
+ fclose (proc);
+ return val;
+ }
+ }
+ abort ();
+}
+#else
+#error "OS unsupported"
+#endif
+
+static void
+verify0 (char *p, size_t s)
+{
+ for (size_t i = 0; i < s; ++i)
+ if (p[i] != 0)
+ abort ();
+}
+
+#include <omp.h>
+
+int
+main ()
+{
+ /* Choose a small size where all our allocations fit on one page. */
+ const int SIZE = 10;
+#ifndef OFFLOAD_DEVICE_NVPTX
+ CHECK_SIZE (SIZE*4);
+#endif
+
+ const omp_alloctrait_t traits[] = {
+ { omp_atk_pinned, 1 }
+ };
+ omp_allocator_handle_t allocator = omp_init_allocator (omp_default_mem_space, 1, traits);
+
+ // Sanity check
+ if (get_pinned_mem () != 0)
+ abort ();
+
+ void *p = omp_alloc (SIZE, allocator);
+ if (!p)
+ abort ();
+
+ int amount = get_pinned_mem ();
+#ifdef OFFLOAD_DEVICE_NVPTX
+ /* This doesn't show up as process 'VmLck'ed memory. */
+ if (amount != 0)
+ abort ();
+#else
+ if (amount == 0)
+ abort ();
+#endif
+
+ p = omp_realloc (p, SIZE * 2, allocator, allocator);
+
+ int amount2 = get_pinned_mem ();
+#ifdef OFFLOAD_DEVICE_NVPTX
+ /* This doesn't show up as process 'VmLck'ed memory. */
+ if (amount2 != 0)
+ abort ();
+#else
+ /* A small allocation should not allocate another page. */
+ if (amount2 != amount)
+ abort ();
+#endif
+
+ p = omp_calloc (1, SIZE, allocator);
+
+#ifdef OFFLOAD_DEVICE_NVPTX
+ /* This doesn't show up as process 'VmLck'ed memory. */
+ if (get_pinned_mem () != 0)
+ abort ();
+#else
+ /* A small allocation should not allocate another page. */
+ if (get_pinned_mem () != amount2)
+ abort ();
+#endif
+
+ verify0 (p, SIZE);
+
+ return 0;
+}
projects / thirdparty / gcc.git / commitdiff
