enum {
PAI_PMU_CRYPTO, /* Index of PMU pai_crypto */
+ PAI_PMU_EXT, /* Index of PMU pai_ext */
PAI_PMU_MAX /* # of PAI PMUs */
};
+enum {
+ PAIE1_CB_SZ = 0x200, /* Size of PAIE1 control block */
+ PAIE1_CTRBLOCK_SZ = 0x400 /* Size of PAIE1 counter blocks */
+};
+
struct pai_userdata {
u16 num;
u64 value;
} __packed;
+/* Create the PAI extension 1 control block area.
+ * The PAI extension control block 1 is pointed to by lowcore
+ * address 0x1508 for each CPU. This control block is 512 bytes in size
+ * and requires a 512 byte boundary alignment.
+ */
+struct paiext_cb { /* PAI extension 1 control block */
+ u64 header; /* Not used */
+ u64 reserved1;
+ u64 acc; /* Addr to analytics counter control block */
+ u8 reserved2[488];
+} __packed;
+
struct pai_map {
unsigned long *area; /* Area for CPU to store counters */
struct pai_userdata *save; /* Page to store no-zero counters */
refcount_t refcnt; /* Reference count mapped buffers */
struct perf_event *event; /* Perf event for sampling */
struct list_head syswide_list; /* List system-wide sampling events */
+ struct paiext_cb *paiext_cb; /* PAI extension control block area */
+ bool fullpage; /* True: counter area is a full page */
};
struct pai_mapptr {
/* Free all memory allocated for event counting/sampling setup */
static void pai_free(struct pai_mapptr *mp)
{
- free_page((unsigned long)mp->mapptr->area);
+ if (mp->mapptr->fullpage)
+ free_page((unsigned long)mp->mapptr->area);
+ else
+ kfree(mp->mapptr->area);
+ kfree(mp->mapptr->paiext_cb);
kvfree(mp->mapptr->save);
kfree(mp->mapptr);
mp->mapptr = NULL;
{
int rc, idx = PAI_PMU_IDX(event);
struct pai_map *cpump = NULL;
+ bool need_paiext_cb = false;
struct pai_mapptr *mp;
mutex_lock(&pai_reserve_mutex);
* Only the first counting event has to allocate a page.
*/
mp->mapptr = cpump;
- cpump->area = (unsigned long *)get_zeroed_page(GFP_KERNEL);
+ if (idx == PAI_PMU_CRYPTO) {
+ cpump->area = (unsigned long *)get_zeroed_page(GFP_KERNEL);
+ /* free_page() can handle 0x0 address */
+ cpump->fullpage = true;
+ } else { /* PAI_PMU_EXT */
+ /*
+ * Allocate memory for counter area and counter extraction.
+ * These are
+ * - a 512 byte block and requires 512 byte boundary
+ * alignment.
+ * - a 1KB byte block and requires 1KB boundary
+ * alignment.
+ * Only the first counting event has to allocate the area.
+ *
+ * Note: This works with commit 59bb47985c1d by default.
+ * Backporting this to kernels without this commit might
+ * needs adjustment.
+ */
+ cpump->area = kzalloc(pai_pmu[idx].area_size, GFP_KERNEL);
+ cpump->paiext_cb = kzalloc(PAIE1_CB_SZ, GFP_KERNEL);
+ need_paiext_cb = true;
+ }
cpump->save = kvmalloc_array(pai_pmu[idx].num_avail + 1,
sizeof(struct pai_userdata),
GFP_KERNEL);
- if (!cpump->area || !cpump->save) {
+ if (!cpump->area || !cpump->save ||
+ (need_paiext_cb && !cpump->paiext_cb)) {
pai_free(mp);
goto undo;
}
/* PAI crypto event must be in valid range, try others if not */
if (a->config < pp->base || a->config > pp->base + pp->num_avail)
return -ENOENT;
+ if (idx == PAI_PMU_EXT && a->exclude_user)
+ return -EINVAL;
PAI_PMU_IDX(event) = idx;
return 0;
}
int idx = PAI_PMU_IDX(event);
struct pai_mapptr *mp = this_cpu_ptr(pai_root[idx].mapptr);
struct pai_map *cpump = mp->mapptr;
+ struct paiext_cb *pcb = cpump->paiext_cb;
unsigned long ccd;
if (++cpump->active_events == 1) {
- ccd = virt_to_phys(cpump->area) | PAI_CRYPTO_KERNEL_OFFSET;
- WRITE_ONCE(get_lowcore()->ccd, ccd);
- local_ctl_set_bit(0, CR0_CRYPTOGRAPHY_COUNTER_BIT);
+ if (!pcb) { /* PAI crypto */
+ ccd = virt_to_phys(cpump->area) | PAI_CRYPTO_KERNEL_OFFSET;
+ WRITE_ONCE(get_lowcore()->ccd, ccd);
+ local_ctl_set_bit(0, CR0_CRYPTOGRAPHY_COUNTER_BIT);
+ } else { /* PAI extension 1 */
+ ccd = virt_to_phys(pcb);
+ WRITE_ONCE(get_lowcore()->aicd, ccd);
+ pcb->acc = virt_to_phys(cpump->area) | 0x1;
+ /* Enable CPU instruction lookup for PAIE1 control block */
+ local_ctl_set_bit(0, CR0_PAI_EXTENSION_BIT);
+ }
}
if (flags & PERF_EF_START)
pai_pmu[idx].pmu->start(event, PERF_EF_RELOAD);
int idx = PAI_PMU_IDX(event);
struct pai_mapptr *mp = this_cpu_ptr(pai_root[idx].mapptr);
struct pai_map *cpump = mp->mapptr;
+ struct paiext_cb *pcb = cpump->paiext_cb;
pai_pmu[idx].pmu->stop(event, PERF_EF_UPDATE);
if (--cpump->active_events == 0) {
- local_ctl_clear_bit(0, CR0_CRYPTOGRAPHY_COUNTER_BIT);
- WRITE_ONCE(get_lowcore()->ccd, 0);
+ if (!pcb) { /* PAI crypto */
+ local_ctl_clear_bit(0, CR0_CRYPTOGRAPHY_COUNTER_BIT);
+ WRITE_ONCE(get_lowcore()->ccd, 0);
+ } else { /* PAI extension 1 */
+ /* Disable CPU instruction lookup for PAIE1 control block */
+ local_ctl_clear_bit(0, CR0_PAI_EXTENSION_BIT);
+ pcb->acc = 0;
+ WRITE_ONCE(get_lowcore()->aicd, 0);
+ }
}
}
pai_have_samples(PAI_PMU_CRYPTO);
}
+/* ============================= paiext ====================================*/
+
+static void paiext_event_destroy(struct perf_event *event)
+{
+ pai_event_destroy(event);
+}
+
+/* Might be called on different CPU than the one the event is intended for. */
+static int paiext_event_init(struct perf_event *event)
+{
+ int rc = pai_event_init(event, PAI_PMU_EXT);
+
+ if (!rc) {
+ event->attr.exclude_kernel = true; /* No kernel space part */
+ event->destroy = paiext_event_destroy;
+ /* Offset of NNPA in paiext_cb */
+ event->hw.config_base = offsetof(struct paiext_cb, acc);
+ }
+ return rc;
+}
+
+static u64 paiext_getall(struct perf_event *event)
+{
+ return pai_getdata(event, false);
+}
+
+static void paiext_read(struct perf_event *event)
+{
+ pai_read(event, paiext_getall);
+}
+
+static void paiext_start(struct perf_event *event, int flags)
+{
+ pai_start(event, flags, paiext_getall);
+}
+
+static int paiext_add(struct perf_event *event, int flags)
+{
+ return pai_add(event, flags);
+}
+
+static void paiext_stop(struct perf_event *event, int flags)
+{
+ pai_stop(event, flags);
+}
+
+static void paiext_del(struct perf_event *event, int flags)
+{
+ pai_del(event, flags);
+}
+
+/* Called on schedule-in and schedule-out. No access to event structure,
+ * but for sampling only event NNPA_ALL is allowed.
+ */
+static void paiext_sched_task(struct perf_event_pmu_context *pmu_ctx,
+ struct task_struct *task, bool sched_in)
+{
+ /* We started with a clean page on event installation. So read out
+ * results on schedule_out and if page was dirty, save old values.
+ */
+ if (!sched_in)
+ pai_have_samples(PAI_PMU_EXT);
+}
+
/* Attribute definitions for paicrypt interface. As with other CPU
* Measurement Facilities, there is one attribute per mapped counter.
* The number of mapped counters may vary per machine generation. Use
[172] = "PCKMO_ENCRYPT_AES_XTS_256",
};
+static struct attribute *paiext_format_attr[] = {
+ &format_attr_event.attr,
+ NULL,
+};
+
+static struct attribute_group paiext_events_group = {
+ .name = "events",
+ .attrs = NULL, /* Filled in attr_event_init() */
+};
+
+static struct attribute_group paiext_format_group = {
+ .name = "format",
+ .attrs = paiext_format_attr,
+};
+
+static const struct attribute_group *paiext_attr_groups[] = {
+ &paiext_events_group,
+ &paiext_format_group,
+ NULL,
+};
+
+/* Performance monitoring unit for mapped counters */
+static struct pmu paiext = {
+ .task_ctx_nr = perf_hw_context,
+ .event_init = paiext_event_init,
+ .add = paiext_add,
+ .del = paiext_del,
+ .start = paiext_start,
+ .stop = paiext_stop,
+ .read = paiext_read,
+ .sched_task = paiext_sched_task,
+ .attr_groups = paiext_attr_groups,
+};
+
+/* List of symbolic PAI extension 1 NNPA counter names. */
+static const char * const paiext_ctrnames[] = {
+ [0] = "NNPA_ALL",
+ [1] = "NNPA_ADD",
+ [2] = "NNPA_SUB",
+ [3] = "NNPA_MUL",
+ [4] = "NNPA_DIV",
+ [5] = "NNPA_MIN",
+ [6] = "NNPA_MAX",
+ [7] = "NNPA_LOG",
+ [8] = "NNPA_EXP",
+ [9] = "NNPA_IBM_RESERVED_9",
+ [10] = "NNPA_RELU",
+ [11] = "NNPA_TANH",
+ [12] = "NNPA_SIGMOID",
+ [13] = "NNPA_SOFTMAX",
+ [14] = "NNPA_BATCHNORM",
+ [15] = "NNPA_MAXPOOL2D",
+ [16] = "NNPA_AVGPOOL2D",
+ [17] = "NNPA_LSTMACT",
+ [18] = "NNPA_GRUACT",
+ [19] = "NNPA_CONVOLUTION",
+ [20] = "NNPA_MATMUL_OP",
+ [21] = "NNPA_MATMUL_OP_BCAST23",
+ [22] = "NNPA_SMALLBATCH",
+ [23] = "NNPA_LARGEDIM",
+ [24] = "NNPA_SMALLTENSOR",
+ [25] = "NNPA_1MFRAME",
+ [26] = "NNPA_2GFRAME",
+ [27] = "NNPA_ACCESSEXCEPT",
+ [28] = "NNPA_TRANSFORM",
+ [29] = "NNPA_GELU",
+ [30] = "NNPA_MOMENTS",
+ [31] = "NNPA_LAYERNORM",
+ [32] = "NNPA_MATMUL_OP_BCAST1",
+ [33] = "NNPA_SQRT",
+ [34] = "NNPA_INVSQRT",
+ [35] = "NNPA_NORM",
+ [36] = "NNPA_REDUCE",
+};
+
static void __init attr_event_free(struct attribute **attrs)
{
struct perf_pmu_events_attr *pa;
.exit = pai_pmu_exit,
.pmu = &paicrypt,
.event_group = &paicrypt_events_group
+ },
+ [PAI_PMU_EXT] = {
+ .pmuname = "pai_ext",
+ .facility_nr = 197,
+ .num_named = ARRAY_SIZE(paiext_ctrnames),
+ .names = paiext_ctrnames,
+ .base = PAI_NNPA_BASE,
+ .kernel_offset = 0,
+ .area_size = PAIE1_CTRBLOCK_SZ,
+ .init = pai_pmu_init,
+ .exit = pai_pmu_exit,
+ .pmu = &paiext,
+ .event_group = &paiext_events_group
}
};
continue;
}
break;
+ case PAI_PMU_EXT:
+ p->num_avail = ib.num_nnpa;
+ break;
}
p->num_avail += 1; /* Add xxx_ALL event */
if (p->init) {
+++ /dev/null
-// SPDX-License-Identifier: GPL-2.0
-/*
- * Performance event support - Processor Activity Instrumentation Extension
- * Facility
- *
- * Copyright IBM Corp. 2022
- * Author(s): Thomas Richter <tmricht@linux.ibm.com>
- */
-#define KMSG_COMPONENT "pai_ext"
-#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
-
-#include <linux/kernel.h>
-#include <linux/kernel_stat.h>
-#include <linux/percpu.h>
-#include <linux/notifier.h>
-#include <linux/init.h>
-#include <linux/io.h>
-#include <linux/perf_event.h>
-#include <asm/ctlreg.h>
-#include <asm/pai.h>
-#include <asm/debug.h>
-
-#define PAIE1_CB_SZ 0x200 /* Size of PAIE1 control block */
-#define PAIE1_CTRBLOCK_SZ 0x400 /* Size of PAIE1 counter blocks */
-
-static debug_info_t *paiext_dbg;
-static unsigned int paiext_cnt; /* Extracted with QPACI instruction */
-
-struct pai_userdata {
- u16 num;
- u64 value;
-} __packed;
-
-/* Create the PAI extension 1 control block area.
- * The PAI extension control block 1 is pointed to by lowcore
- * address 0x1508 for each CPU. This control block is 512 bytes in size
- * and requires a 512 byte boundary alignment.
- */
-struct paiext_cb { /* PAI extension 1 control block */
- u64 header; /* Not used */
- u64 reserved1;
- u64 acc; /* Addr to analytics counter control block */
- u8 reserved2[488];
-} __packed;
-
-struct paiext_map {
- unsigned long *area; /* Area for CPU to store counters */
- struct pai_userdata *save; /* Area to store non-zero counters */
- unsigned int active_events; /* # of PAI Extension users */
- refcount_t refcnt;
- struct perf_event *event; /* Perf event for sampling */
- struct paiext_cb *paiext_cb; /* PAI extension control block area */
- struct list_head syswide_list; /* List system-wide sampling events */
-};
-
-struct paiext_mapptr {
- struct paiext_map *mapptr;
-};
-
-static struct paiext_root { /* Anchor to per CPU data */
- refcount_t refcnt; /* Overall active events */
- struct paiext_mapptr __percpu *mapptr;
-} paiext_root;
-
-/* Free per CPU data when the last event is removed. */
-static void paiext_root_free(void)
-{
- if (refcount_dec_and_test(&paiext_root.refcnt)) {
- free_percpu(paiext_root.mapptr);
- paiext_root.mapptr = NULL;
- }
- debug_sprintf_event(paiext_dbg, 5, "%s root.refcount %d\n", __func__,
- refcount_read(&paiext_root.refcnt));
-}
-
-/* On initialization of first event also allocate per CPU data dynamically.
- * Start with an array of pointers, the array size is the maximum number of
- * CPUs possible, which might be larger than the number of CPUs currently
- * online.
- */
-static int paiext_root_alloc(void)
-{
- if (!refcount_inc_not_zero(&paiext_root.refcnt)) {
- /* The memory is already zeroed. */
- paiext_root.mapptr = alloc_percpu(struct paiext_mapptr);
- if (!paiext_root.mapptr) {
- /* Returning without refcnt adjustment is ok. The
- * error code is handled by paiext_alloc() which
- * decrements refcnt when an event can not be
- * created.
- */
- return -ENOMEM;
- }
- refcount_set(&paiext_root.refcnt, 1);
- }
- return 0;
-}
-
-/* Protects against concurrent increment of sampler and counter member
- * increments at the same time and prohibits concurrent execution of
- * counting and sampling events.
- * Ensures that analytics counter block is deallocated only when the
- * sampling and counting on that cpu is zero.
- * For details see paiext_alloc().
- */
-static DEFINE_MUTEX(paiext_reserve_mutex);
-
-/* Free all memory allocated for event counting/sampling setup */
-static void paiext_free(struct paiext_mapptr *mp)
-{
- kfree(mp->mapptr->area);
- kfree(mp->mapptr->paiext_cb);
- kvfree(mp->mapptr->save);
- kfree(mp->mapptr);
- mp->mapptr = NULL;
-}
-
-/* Release the PMU if event is the last perf event */
-static void paiext_event_destroy_cpu(struct perf_event *event, int cpu)
-{
- struct paiext_mapptr *mp = per_cpu_ptr(paiext_root.mapptr, cpu);
- struct paiext_map *cpump = mp->mapptr;
-
- mutex_lock(&paiext_reserve_mutex);
- if (refcount_dec_and_test(&cpump->refcnt)) /* Last reference gone */
- paiext_free(mp);
- paiext_root_free();
- mutex_unlock(&paiext_reserve_mutex);
-}
-
-static void paiext_event_destroy(struct perf_event *event)
-{
- int cpu;
-
- free_page(PAI_SAVE_AREA(event));
- if (event->cpu == -1) {
- struct cpumask *mask = PAI_CPU_MASK(event);
-
- for_each_cpu(cpu, mask)
- paiext_event_destroy_cpu(event, cpu);
- kfree(mask);
- } else {
- paiext_event_destroy_cpu(event, event->cpu);
- }
- debug_sprintf_event(paiext_dbg, 4, "%s cpu %d\n", __func__,
- event->cpu);
-}
-
-/* Used to avoid races in checking concurrent access of counting and
- * sampling for pai_extension events.
- *
- * Only one instance of event pai_ext/NNPA_ALL/ for sampling is
- * allowed and when this event is running, no counting event is allowed.
- * Several counting events are allowed in parallel, but no sampling event
- * is allowed while one (or more) counting events are running.
- *
- * This function is called in process context and it is safe to block.
- * When the event initialization functions fails, no other call back will
- * be invoked.
- *
- * Allocate the memory for the event.
- */
-static int paiext_alloc_cpu(struct perf_event *event, int cpu)
-{
- struct paiext_mapptr *mp;
- struct paiext_map *cpump;
- int rc;
-
- mutex_lock(&paiext_reserve_mutex);
- rc = paiext_root_alloc();
- if (rc)
- goto unlock;
-
- mp = per_cpu_ptr(paiext_root.mapptr, cpu);
- cpump = mp->mapptr;
- if (!cpump) { /* Paiext_map allocated? */
- rc = -ENOMEM;
- cpump = kzalloc(sizeof(*cpump), GFP_KERNEL);
- if (!cpump)
- goto undo;
-
- /* Allocate memory for counter area and counter extraction.
- * These are
- * - a 512 byte block and requires 512 byte boundary alignment.
- * - a 1KB byte block and requires 1KB boundary alignment.
- * Only the first counting event has to allocate the area.
- *
- * Note: This works with commit 59bb47985c1d by default.
- * Backporting this to kernels without this commit might
- * need adjustment.
- */
- mp->mapptr = cpump;
- cpump->area = kzalloc(PAIE1_CTRBLOCK_SZ, GFP_KERNEL);
- cpump->paiext_cb = kzalloc(PAIE1_CB_SZ, GFP_KERNEL);
- cpump->save = kvmalloc_array(paiext_cnt + 1,
- sizeof(struct pai_userdata),
- GFP_KERNEL);
- if (!cpump->save || !cpump->area || !cpump->paiext_cb) {
- paiext_free(mp);
- goto undo;
- }
- INIT_LIST_HEAD(&cpump->syswide_list);
- refcount_set(&cpump->refcnt, 1);
- rc = 0;
- } else {
- refcount_inc(&cpump->refcnt);
- }
-
-undo:
- if (rc) {
- /* Error in allocation of event, decrement anchor. Since
- * the event in not created, its destroy() function is never
- * invoked. Adjust the reference counter for the anchor.
- */
- paiext_root_free();
- }
-unlock:
- mutex_unlock(&paiext_reserve_mutex);
- /* If rc is non-zero, no increment of counter/sampler was done. */
- return rc;
-}
-
-static int paiext_alloc(struct perf_event *event)
-{
- struct cpumask *maskptr;
- int cpu, rc = -ENOMEM;
-
- maskptr = kzalloc(sizeof(*maskptr), GFP_KERNEL);
- if (!maskptr)
- goto out;
-
- for_each_online_cpu(cpu) {
- rc = paiext_alloc_cpu(event, cpu);
- if (rc) {
- for_each_cpu(cpu, maskptr)
- paiext_event_destroy_cpu(event, cpu);
- kfree(maskptr);
- goto out;
- }
- cpumask_set_cpu(cpu, maskptr);
- }
-
- /*
- * On error all cpumask are freed and all events have been destroyed.
- * Save of which CPUs data structures have been allocated for.
- * Release them in paicrypt_event_destroy call back function
- * for this event.
- */
- PAI_CPU_MASK(event) = maskptr;
- rc = 0;
-out:
- return rc;
-}
-
-/* The PAI extension 1 control block supports up to 128 entries. Return
- * the index within PAIE1_CB given the event number. Also validate event
- * number.
- */
-static int paiext_event_valid(struct perf_event *event)
-{
- u64 cfg = event->attr.config;
-
- if (cfg >= PAI_NNPA_BASE && cfg <= PAI_NNPA_BASE + paiext_cnt) {
- /* Offset NNPA in paiext_cb */
- event->hw.config_base = offsetof(struct paiext_cb, acc);
- return 0;
- }
- return -ENOENT;
-}
-
-/* Might be called on different CPU than the one the event is intended for. */
-static int paiext_event_init(struct perf_event *event)
-{
- struct perf_event_attr *a = &event->attr;
- int rc;
-
- /* PMU pai_ext registered as PERF_TYPE_RAW, check event type */
- if (a->type != PERF_TYPE_RAW && event->pmu->type != a->type)
- return -ENOENT;
- /* PAI extension event must be valid and in supported range */
- rc = paiext_event_valid(event);
- if (rc)
- return rc;
- /* Allow only event NNPA_ALL for sampling. */
- if (a->sample_period && a->config != PAI_NNPA_BASE)
- return -EINVAL;
- /* Prohibit exclude_user event selection */
- if (a->exclude_user)
- return -EINVAL;
- /* Get a page to store last counter values for sampling */
- if (a->sample_period) {
- PAI_SAVE_AREA(event) = get_zeroed_page(GFP_KERNEL);
- if (!PAI_SAVE_AREA(event))
- return -ENOMEM;
- }
-
- if (event->cpu >= 0)
- rc = paiext_alloc_cpu(event, event->cpu);
- else
- rc = paiext_alloc(event);
- if (rc) {
- free_page(PAI_SAVE_AREA(event));
- return rc;
- }
- event->destroy = paiext_event_destroy;
-
- if (a->sample_period) {
- a->sample_period = 1;
- a->freq = 0;
- /* Register for paicrypt_sched_task() to be called */
- event->attach_state |= PERF_ATTACH_SCHED_CB;
- /* Add raw data which are the memory mapped counters */
- a->sample_type |= PERF_SAMPLE_RAW;
- /* Turn off inheritance */
- a->inherit = 0;
- }
-
- return 0;
-}
-
-static u64 paiext_getctr(unsigned long *area, int nr)
-{
- return area[nr];
-}
-
-/* Read the counter values. Return value from location in buffer. For event
- * NNPA_ALL sum up all events.
- */
-static u64 paiext_getdata(struct perf_event *event)
-{
- struct paiext_mapptr *mp = this_cpu_ptr(paiext_root.mapptr);
- struct paiext_map *cpump = mp->mapptr;
- u64 sum = 0;
- int i;
-
- if (event->attr.config != PAI_NNPA_BASE)
- return paiext_getctr(cpump->area,
- event->attr.config - PAI_NNPA_BASE);
-
- for (i = 1; i <= paiext_cnt; i++)
- sum += paiext_getctr(cpump->area, i);
-
- return sum;
-}
-
-static u64 paiext_getall(struct perf_event *event)
-{
- return paiext_getdata(event);
-}
-
-static void paiext_read(struct perf_event *event)
-{
- u64 prev, new, delta;
-
- prev = local64_read(&event->hw.prev_count);
- new = paiext_getall(event);
- local64_set(&event->hw.prev_count, new);
- delta = new - prev;
- local64_add(delta, &event->count);
-}
-
-static void paiext_start(struct perf_event *event, int flags)
-{
- struct paiext_mapptr *mp = this_cpu_ptr(paiext_root.mapptr);
- struct paiext_map *cpump = mp->mapptr;
- u64 sum;
-
- if (!event->attr.sample_period) { /* Counting */
- sum = paiext_getall(event); /* Get current value */
- local64_set(&event->hw.prev_count, sum);
- } else { /* Sampling */
- memcpy((void *)PAI_SAVE_AREA(event), cpump->area,
- PAIE1_CTRBLOCK_SZ);
- /* Enable context switch callback for system-wide sampling */
- if (!(event->attach_state & PERF_ATTACH_TASK)) {
- list_add_tail(PAI_SWLIST(event), &cpump->syswide_list);
- perf_sched_cb_inc(event->pmu);
- } else {
- cpump->event = event;
- }
- }
-}
-
-static int paiext_add(struct perf_event *event, int flags)
-{
- struct paiext_mapptr *mp = this_cpu_ptr(paiext_root.mapptr);
- struct paiext_map *cpump = mp->mapptr;
- struct paiext_cb *pcb = cpump->paiext_cb;
-
- if (++cpump->active_events == 1) {
- get_lowcore()->aicd = virt_to_phys(cpump->paiext_cb);
- pcb->acc = virt_to_phys(cpump->area) | 0x1;
- /* Enable CPU instruction lookup for PAIE1 control block */
- local_ctl_set_bit(0, CR0_PAI_EXTENSION_BIT);
- }
- if (flags & PERF_EF_START)
- paiext_start(event, PERF_EF_RELOAD);
- event->hw.state = 0;
- return 0;
-}
-
-static void paiext_have_sample(struct perf_event *, struct paiext_map *);
-static void paiext_stop(struct perf_event *event, int flags)
-{
- struct paiext_mapptr *mp = this_cpu_ptr(paiext_root.mapptr);
- struct paiext_map *cpump = mp->mapptr;
-
- if (!event->attr.sample_period) { /* Counting */
- paiext_read(event);
- } else { /* Sampling */
- if (!(event->attach_state & PERF_ATTACH_TASK)) {
- list_del(PAI_SWLIST(event));
- perf_sched_cb_dec(event->pmu);
- } else {
- paiext_have_sample(event, cpump);
- cpump->event = NULL;
- }
- }
- event->hw.state = PERF_HES_STOPPED;
-}
-
-static void paiext_del(struct perf_event *event, int flags)
-{
- struct paiext_mapptr *mp = this_cpu_ptr(paiext_root.mapptr);
- struct paiext_map *cpump = mp->mapptr;
- struct paiext_cb *pcb = cpump->paiext_cb;
-
- paiext_stop(event, PERF_EF_UPDATE);
- if (--cpump->active_events == 0) {
- /* Disable CPU instruction lookup for PAIE1 control block */
- local_ctl_clear_bit(0, CR0_PAI_EXTENSION_BIT);
- pcb->acc = 0;
- get_lowcore()->aicd = 0;
- }
-}
-
-/* Create raw data and save it in buffer. Returns number of bytes copied.
- * Saves only positive counter entries of the form
- * 2 bytes: Number of counter
- * 8 bytes: Value of counter
- */
-static size_t paiext_copy(struct pai_userdata *userdata, unsigned long *area,
- unsigned long *area_old)
-{
- int i, outidx = 0;
-
- for (i = 1; i <= paiext_cnt; i++) {
- u64 val = paiext_getctr(area, i);
- u64 val_old = paiext_getctr(area_old, i);
-
- if (val >= val_old)
- val -= val_old;
- else
- val = (~0ULL - val_old) + val + 1;
- if (val) {
- userdata[outidx].num = i;
- userdata[outidx].value = val;
- outidx++;
- }
- }
- return outidx * sizeof(*userdata);
-}
-
-/* Write sample when one or more counters values are nonzero.
- *
- * Note: The function paiext_sched_task() and paiext_push_sample() are not
- * invoked after function paiext_del() has been called because of function
- * perf_sched_cb_dec().
- * The function paiext_sched_task() and paiext_push_sample() are only
- * called when sampling is active. Function perf_sched_cb_inc()
- * has been invoked to install function paiext_sched_task() as call back
- * to run at context switch time (see paiext_add()).
- *
- * This causes function perf_event_context_sched_out() and
- * perf_event_context_sched_in() to check whether the PMU has installed an
- * sched_task() callback. That callback is not active after paiext_del()
- * returns and has deleted the event on that CPU.
- */
-static int paiext_push_sample(size_t rawsize, struct paiext_map *cpump,
- struct perf_event *event)
-{
- struct perf_sample_data data;
- struct perf_raw_record raw;
- struct pt_regs regs;
- int overflow;
-
- /* Setup perf sample */
- memset(®s, 0, sizeof(regs));
- memset(&raw, 0, sizeof(raw));
- memset(&data, 0, sizeof(data));
- perf_sample_data_init(&data, 0, event->hw.last_period);
- if (event->attr.sample_type & PERF_SAMPLE_TID) {
- data.tid_entry.pid = task_tgid_nr(current);
- data.tid_entry.tid = task_pid_nr(current);
- }
- if (event->attr.sample_type & PERF_SAMPLE_TIME)
- data.time = event->clock();
- if (event->attr.sample_type & (PERF_SAMPLE_ID | PERF_SAMPLE_IDENTIFIER))
- data.id = event->id;
- if (event->attr.sample_type & PERF_SAMPLE_CPU)
- data.cpu_entry.cpu = smp_processor_id();
- if (event->attr.sample_type & PERF_SAMPLE_RAW) {
- raw.frag.size = rawsize;
- raw.frag.data = cpump->save;
- perf_sample_save_raw_data(&data, event, &raw);
- }
-
- overflow = perf_event_overflow(event, &data, ®s);
- perf_event_update_userpage(event);
- /* Save NNPA lowcore area after read in event */
- memcpy((void *)PAI_SAVE_AREA(event), cpump->area,
- PAIE1_CTRBLOCK_SZ);
- return overflow;
-}
-
-/* Check if there is data to be saved on schedule out of a task. */
-static void paiext_have_sample(struct perf_event *event,
- struct paiext_map *cpump)
-{
- size_t rawsize;
-
- if (!event)
- return;
- rawsize = paiext_copy(cpump->save, cpump->area,
- (unsigned long *)PAI_SAVE_AREA(event));
- if (rawsize) /* Incremented counters */
- paiext_push_sample(rawsize, cpump, event);
-}
-
-/* Check if there is data to be saved on schedule out of a task. */
-static void paiext_have_samples(void)
-{
- struct paiext_mapptr *mp = this_cpu_ptr(paiext_root.mapptr);
- struct paiext_map *cpump = mp->mapptr;
- struct perf_event *event;
-
- list_for_each_entry(event, &cpump->syswide_list, hw.tp_list)
- paiext_have_sample(event, cpump);
-}
-
-/* Called on schedule-in and schedule-out. No access to event structure,
- * but for sampling only event NNPA_ALL is allowed.
- */
-static void paiext_sched_task(struct perf_event_pmu_context *pmu_ctx,
- struct task_struct *task, bool sched_in)
-{
- /* We started with a clean page on event installation. So read out
- * results on schedule_out and if page was dirty, save old values.
- */
- if (!sched_in)
- paiext_have_samples();
-}
-
-/* Attribute definitions for pai extension1 interface. As with other CPU
- * Measurement Facilities, there is one attribute per mapped counter.
- * The number of mapped counters may vary per machine generation. Use
- * the QUERY PROCESSOR ACTIVITY COUNTER INFORMATION (QPACI) instruction
- * to determine the number of mapped counters. The instructions returns
- * a positive number, which is the highest number of supported counters.
- * All counters less than this number are also supported, there are no
- * holes. A returned number of zero means no support for mapped counters.
- *
- * The identification of the counter is a unique number. The chosen range
- * is 0x1800 + offset in mapped kernel page.
- * All CPU Measurement Facility counters identifiers must be unique and
- * the numbers from 0 to 496 are already used for the CPU Measurement
- * Counter facility. Number 0x1000 to 0x103e are used for PAI cryptography
- * counters.
- * Numbers 0xb0000, 0xbc000 and 0xbd000 are already
- * used for the CPU Measurement Sampling facility.
- */
-PMU_FORMAT_ATTR(event, "config:0-63");
-
-static struct attribute *paiext_format_attr[] = {
- &format_attr_event.attr,
- NULL,
-};
-
-static struct attribute_group paiext_events_group = {
- .name = "events",
- .attrs = NULL, /* Filled in attr_event_init() */
-};
-
-static struct attribute_group paiext_format_group = {
- .name = "format",
- .attrs = paiext_format_attr,
-};
-
-static const struct attribute_group *paiext_attr_groups[] = {
- &paiext_events_group,
- &paiext_format_group,
- NULL,
-};
-
-/* Performance monitoring unit for mapped counters */
-static struct pmu paiext = {
- .task_ctx_nr = perf_hw_context,
- .event_init = paiext_event_init,
- .add = paiext_add,
- .del = paiext_del,
- .start = paiext_start,
- .stop = paiext_stop,
- .read = paiext_read,
- .sched_task = paiext_sched_task,
- .attr_groups = paiext_attr_groups,
-};
-
-/* List of symbolic PAI extension 1 NNPA counter names. */
-static const char * const paiext_ctrnames[] = {
- [0] = "NNPA_ALL",
- [1] = "NNPA_ADD",
- [2] = "NNPA_SUB",
- [3] = "NNPA_MUL",
- [4] = "NNPA_DIV",
- [5] = "NNPA_MIN",
- [6] = "NNPA_MAX",
- [7] = "NNPA_LOG",
- [8] = "NNPA_EXP",
- [9] = "NNPA_IBM_RESERVED_9",
- [10] = "NNPA_RELU",
- [11] = "NNPA_TANH",
- [12] = "NNPA_SIGMOID",
- [13] = "NNPA_SOFTMAX",
- [14] = "NNPA_BATCHNORM",
- [15] = "NNPA_MAXPOOL2D",
- [16] = "NNPA_AVGPOOL2D",
- [17] = "NNPA_LSTMACT",
- [18] = "NNPA_GRUACT",
- [19] = "NNPA_CONVOLUTION",
- [20] = "NNPA_MATMUL_OP",
- [21] = "NNPA_MATMUL_OP_BCAST23",
- [22] = "NNPA_SMALLBATCH",
- [23] = "NNPA_LARGEDIM",
- [24] = "NNPA_SMALLTENSOR",
- [25] = "NNPA_1MFRAME",
- [26] = "NNPA_2GFRAME",
- [27] = "NNPA_ACCESSEXCEPT",
- [28] = "NNPA_TRANSFORM",
- [29] = "NNPA_GELU",
- [30] = "NNPA_MOMENTS",
- [31] = "NNPA_LAYERNORM",
- [32] = "NNPA_MATMUL_OP_BCAST1",
- [33] = "NNPA_SQRT",
- [34] = "NNPA_INVSQRT",
- [35] = "NNPA_NORM",
- [36] = "NNPA_REDUCE",
-};
-
-static void __init attr_event_free(struct attribute **attrs, int num)
-{
- struct perf_pmu_events_attr *pa;
- struct device_attribute *dap;
- int i;
-
- for (i = 0; i < num; i++) {
- dap = container_of(attrs[i], struct device_attribute, attr);
- pa = container_of(dap, struct perf_pmu_events_attr, attr);
- kfree(pa);
- }
- kfree(attrs);
-}
-
-static int __init attr_event_init_one(struct attribute **attrs, int num)
-{
- struct perf_pmu_events_attr *pa;
-
- /* Index larger than array_size, no counter name available */
- if (num >= ARRAY_SIZE(paiext_ctrnames)) {
- attrs[num] = NULL;
- return 0;
- }
-
- pa = kzalloc(sizeof(*pa), GFP_KERNEL);
- if (!pa)
- return -ENOMEM;
-
- sysfs_attr_init(&pa->attr.attr);
- pa->id = PAI_NNPA_BASE + num;
- pa->attr.attr.name = paiext_ctrnames[num];
- pa->attr.attr.mode = 0444;
- pa->attr.show = cpumf_events_sysfs_show;
- pa->attr.store = NULL;
- attrs[num] = &pa->attr.attr;
- return 0;
-}
-
-/* Create PMU sysfs event attributes on the fly. */
-static int __init attr_event_init(void)
-{
- struct attribute **attrs;
- int ret, i;
-
- attrs = kmalloc_array(paiext_cnt + 2, sizeof(*attrs), GFP_KERNEL);
- if (!attrs)
- return -ENOMEM;
- for (i = 0; i <= paiext_cnt; i++) {
- ret = attr_event_init_one(attrs, i);
- if (ret) {
- attr_event_free(attrs, i);
- return ret;
- }
- }
- attrs[i] = NULL;
- paiext_events_group.attrs = attrs;
- return 0;
-}
-
-static int __init paiext_init(void)
-{
- struct qpaci_info_block ib;
- int rc = -ENOMEM;
-
- if (!test_facility(197))
- return 0;
-
- qpaci(&ib);
- paiext_cnt = ib.num_nnpa;
- if (paiext_cnt >= PAI_NNPA_MAXCTR)
- paiext_cnt = PAI_NNPA_MAXCTR;
- if (!paiext_cnt)
- return 0;
-
- rc = attr_event_init();
- if (rc) {
- pr_err("Creation of PMU " KMSG_COMPONENT " /sysfs failed\n");
- return rc;
- }
-
- /* Setup s390dbf facility */
- paiext_dbg = debug_register(KMSG_COMPONENT, 2, 256, 128);
- if (!paiext_dbg) {
- pr_err("Registration of s390dbf " KMSG_COMPONENT " failed\n");
- rc = -ENOMEM;
- goto out_init;
- }
- debug_register_view(paiext_dbg, &debug_sprintf_view);
-
- rc = perf_pmu_register(&paiext, KMSG_COMPONENT, -1);
- if (rc) {
- pr_err("Registration of " KMSG_COMPONENT " PMU failed with "
- "rc=%i\n", rc);
- goto out_pmu;
- }
-
- return 0;
-
-out_pmu:
- debug_unregister_view(paiext_dbg, &debug_sprintf_view);
- debug_unregister(paiext_dbg);
-out_init:
- attr_event_free(paiext_events_group.attrs,
- ARRAY_SIZE(paiext_ctrnames) + 1);
- return rc;
-}
-
-device_initcall(paiext_init);
projects / thirdparty / linux.git / commitdiff
