s390/pai_crypto: Enable per-task and system-wide sampling event

The PMU for PAI crypto counters enforces the following restrictions:

      - No per-task context for PAI crypto sampling event CRYPTO_ALL
      - No multiple system-wide PAI crypto sampling event CRYPTO_ALL

Both restrictions are removed. One or more per-task sampling events
are supported. Also one or more system-wide sampling events are
supported.

Example for per-task context of sampling event CRYPTO_ALL:
       # perf record -e pai_crypto/CRYPTO_ALL/ -- true
Example for system-wide context of sampling event CRYPTO_ALL:
       # perf record -e pai_crypto/CRYPTO_ALL/ -a -- sleep 4

Acked-by: Sumanth Korikkar <sumanthk@linux.ibm.com>
Signed-off-by: Thomas Richter <tmricht@linux.ibm.com>
Signed-off-by: Alexander Gordeev <agordeev@linux.ibm.com>

diff --git a/arch/s390/kernel/perf_pai_crypto.c b/arch/s390/kernel/perf_pai_crypto.c
index 0e296a1482bc95f251ac4bb537ee34f5c054feb4..c0b7f5422e5e766c7272e50ac084fab905b4d5e5 100644 (file)
--- a/arch/s390/kernel/perf_pai_crypto.c
+++ b/arch/s390/kernel/perf_pai_crypto.c
@@ -24,6 +24,8 @@ static debug_info_t *cfm_dbg;
 static unsigned int paicrypt_cnt;      /* Size of the mapped counter sets */
                                        /* extracted with QPACI instruction */
 
+#define PAI_SWLIST(x)          (&(x)->hw.tp_list)
+
 DEFINE_STATIC_KEY_FALSE(pai_key);
 
 struct pai_userdata {
@@ -36,8 +38,8 @@ struct paicrypt_map {
        struct pai_userdata *save;      /* Page to store no-zero counters */
        unsigned int active_events;     /* # of PAI crypto users */
        refcount_t refcnt;              /* Reference count mapped buffers */
-       enum paievt_mode mode;          /* Type of event */
        struct perf_event *event;       /* Perf event for sampling */
+       struct list_head syswide_list;  /* List system-wide sampling events */
 };
 
 struct paicrypt_mapptr {
@@ -90,13 +92,10 @@ static void paicrypt_event_destroy_cpu(struct perf_event *event, int cpu)
        struct paicrypt_map *cpump = mp->mapptr;
 
        mutex_lock(&pai_reserve_mutex);
-       debug_sprintf_event(cfm_dbg, 5, "%s event %#llx cpu %d users %d"
-                           " mode %d refcnt %u\n", __func__,
-                           event->attr.config, event->cpu,
-                           cpump->active_events, cpump->mode,
+       debug_sprintf_event(cfm_dbg, 5, "%s event %#llx cpu %d users %d "
+                           "refcnt %u\n", __func__, event->attr.config,
+                           event->cpu, cpump->active_events,
                            refcount_read(&cpump->refcnt));
-       if (event->attr.sample_period)
-               cpump->mode &= ~PAI_MODE_SAMPLING;
        if (refcount_dec_and_test(&cpump->refcnt)) {
                debug_sprintf_event(cfm_dbg, 4, "%s page %#lx save %p\n",
                                    __func__, (unsigned long)cpump->page,
@@ -172,12 +171,7 @@ static u64 paicrypt_getall(struct perf_event *event)
        return sum;
 }
 
-/* Used to avoid races in checking concurrent access of counting and
- * sampling for crypto events
- *
- * Only one instance of event pai_crypto/CRYPTO_ALL/ for sampling is
- * allowed. Several counting events are allowed in parallel.
- *
+/* Check concurrent access of counting and sampling for crypto events.
  * This function is called in process context and it is save to block.
  * When the event initialization functions fails, no other call back will
  * be invoked.
@@ -186,7 +180,6 @@ static u64 paicrypt_getall(struct perf_event *event)
  */
 static struct paicrypt_map *paicrypt_busy(struct perf_event *event, int cpu)
 {
-       struct perf_event_attr *a = &event->attr;
        struct paicrypt_map *cpump = NULL;
        struct paicrypt_mapptr *mp;
        int rc;
@@ -207,25 +200,9 @@ static struct paicrypt_map *paicrypt_busy(struct perf_event *event, int cpu)
                        rc = -ENOMEM;
                        goto free_root;
                }
+               INIT_LIST_HEAD(&cpump->syswide_list);
        }
 
-       if (a->sample_period) {         /* Sampling requested */
-               if (cpump->mode & PAI_MODE_SAMPLING)
-                       rc = -EBUSY;    /* ... and sampling active */
-               else
-                       cpump->mode |= PAI_MODE_SAMPLING;
-       }
-
-       /*
-        * This error case triggers when there is a conflict:
-        * Either sampling requested and counting already active, or visa
-        * versa. Therefore the struct paicrypto_map for this CPU is
-        * needed or the error could not have occurred. Only adjust root
-        * node refcount.
-        */
-       if (rc)
-               goto free_root;
-
        /* Allocate memory for counter page and counter extraction.
         * Only the first counting event has to allocate a page.
         */
@@ -250,14 +227,14 @@ static struct paicrypt_map *paicrypt_busy(struct perf_event *event, int cpu)
        rc = 0;
        refcount_set(&cpump->refcnt, 1);
        mp->mapptr = cpump;
-       debug_sprintf_event(cfm_dbg, 5, "%s sample_period %#llx users %d"
-                           " mode %d refcnt %u page %#lx save %p rc %d\n",
-                           __func__, a->sample_period, cpump->active_events,
-                           cpump->mode, refcount_read(&cpump->refcnt),
+       debug_sprintf_event(cfm_dbg, 5, "%s users %d refcnt %u page %#lx "
+                           "save %p rc %d\n", __func__, cpump->active_events,
+                           refcount_read(&cpump->refcnt),
                            (unsigned long)cpump->page, cpump->save, rc);
        goto unlock;
 
 free_paicrypt_map:
+       /* Undo memory allocation */
        kfree(cpump);
        mp->mapptr = NULL;
 free_root:
@@ -315,11 +292,7 @@ static int paicrypt_event_init(struct perf_event *event)
        if (a->config < PAI_CRYPTO_BASE ||
            a->config > PAI_CRYPTO_BASE + paicrypt_cnt)
                return -EINVAL;
-       /* Allow only CPU wide operation for sampling */
-       if (a->sample_period &&
-           ((event->attach_state & PERF_ATTACH_TASK) || event->cpu == -1))
-               return -ENOENT;
-       /* Allow only CRYPTO_ALL for sampling. */
+       /* Allow only CRYPTO_ALL for sampling */
        if (a->sample_period && a->config != PAI_CRYPTO_BASE)
                return -EINVAL;
        /* Get a page to store last counter values for sampling */
@@ -382,9 +355,14 @@ static void paicrypt_start(struct perf_event *event, int flags)
                sum = paicrypt_getall(event);   /* Get current value */
                local64_set(&event->hw.prev_count, sum);
        } else {                                /* Sampling */
-               cpump->event = event;
                memcpy((void *)PAI_SAVE_AREA(event), cpump->page, PAGE_SIZE);
-               perf_sched_cb_inc(event->pmu);
+               /* Enable context switch callback for system-wide sampling */
+               if (!(event->attach_state & PERF_ATTACH_TASK)) {
+                       list_add_tail(PAI_SWLIST(event), &cpump->syswide_list);
+                       perf_sched_cb_inc(event->pmu);
+               } else {
+                       cpump->event = event;
+               }
        }
 }
 
@@ -405,6 +383,7 @@ static int paicrypt_add(struct perf_event *event, int flags)
        return 0;
 }
 
+static void paicrypt_have_sample(struct perf_event *, struct paicrypt_map *);
 static void paicrypt_stop(struct perf_event *event, int flags)
 {
        struct paicrypt_mapptr *mp = this_cpu_ptr(paicrypt_root.mapptr);
@@ -413,8 +392,13 @@ static void paicrypt_stop(struct perf_event *event, int flags)
        if (!event->attr.sample_period) {       /* Counting */
                paicrypt_read(event);
        } else {                                /* Sampling */
-               perf_sched_cb_dec(event->pmu);
-               cpump->event = NULL;
+               if (!(event->attach_state & PERF_ATTACH_TASK)) {
+                       perf_sched_cb_dec(event->pmu);
+                       list_del(PAI_SWLIST(event));
+               } else {
+                       paicrypt_have_sample(event, cpump);
+                       cpump->event = NULL;
+               }
        }
        event->hw.state = PERF_HES_STOPPED;
 }
@@ -507,23 +491,30 @@ static int paicrypt_push_sample(size_t rawsize, struct paicrypt_map *cpump,
 }
 
 /* Check if there is data to be saved on schedule out of a task. */
-static int paicrypt_have_sample(void)
+static void paicrypt_have_sample(struct perf_event *event,
+                                struct paicrypt_map *cpump)
 {
-       struct paicrypt_mapptr *mp = this_cpu_ptr(paicrypt_root.mapptr);
-       struct paicrypt_map *cpump = mp->mapptr;
-       struct perf_event *event = cpump->event;
        size_t rawsize;
-       int rc = 0;
 
        if (!event)             /* No event active */
-               return 0;
+               return;
        rawsize = paicrypt_copy(cpump->save, cpump->page,
                                (unsigned long *)PAI_SAVE_AREA(event),
-                               cpump->event->attr.exclude_user,
-                               cpump->event->attr.exclude_kernel);
+                               event->attr.exclude_user,
+                               event->attr.exclude_kernel);
        if (rawsize)                    /* No incremented counters */
-               rc = paicrypt_push_sample(rawsize, cpump, event);
-       return rc;
+               paicrypt_push_sample(rawsize, cpump, event);
+}
+
+/* Check if there is data to be saved on schedule out of a task. */
+static void paicrypt_have_samples(void)
+{
+       struct paicrypt_mapptr *mp = this_cpu_ptr(paicrypt_root.mapptr);
+       struct paicrypt_map *cpump = mp->mapptr;
+       struct perf_event *event;
+
+       list_for_each_entry(event, &cpump->syswide_list, hw.tp_list)
+               paicrypt_have_sample(event, cpump);
 }
 
 /* Called on schedule-in and schedule-out. No access to event structure,
@@ -535,7 +526,7 @@ static void paicrypt_sched_task(struct perf_event_pmu_context *pmu_ctx, bool sch
         * results on schedule_out and if page was dirty, save old values.
         */
        if (!sched_in)
-               paicrypt_have_sample();
+               paicrypt_have_samples();
 }
 
 /* Attribute definitions for paicrypt interface. As with other CPU