perf/amd/ibs: Add support for OP Load Latency Filtering

IBS Op PMU on Zen5 uarch added new Load Latency filtering capability. It's
advertised by CPUID_Fn8000001B_EAX bit 12. When enabled, IBS HW will raise
interrupt only for sample that had an IbsDcMissLat value greater than N
cycles, where N is a programmable value defined as multiples of 128 (i.e.
128, 256, 384 etc.) from 128-2048 cycles. Similar to L3MissOnly, IBS HW
internally drops the sample and restarts if the sample does not meet the
filtering criteria.

Add support for LdLat filtering in IBS Op PMU. Since hardware supports
threshold in multiple of 128, add a software filter on top to support
latency threshold with the granularity of 1 cycle between [128-2048].

Example usage:
  # perf record -a -e ibs_op/ldlat=128/ -- sleep 5

Signed-off-by: Ravi Bangoria <ravi.bangoria@amd.com>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Link: https://lkml.kernel.org/r/20250205060547.1337-2-ravi.bangoria@amd.com

diff --git a/arch/x86/events/amd/ibs.c b/arch/x86/events/amd/ibs.c
index 7978d7910adc3911b7a90b44775a4e843ff9d014..85b29b3f305b2b97f99d38d2eb1aecbfe9e3e9d1 100644 (file)
--- a/arch/x86/events/amd/ibs.c
+++ b/arch/x86/events/amd/ibs.c
@@ -268,6 +268,14 @@ static int validate_group(struct perf_event *event)
        return 0;
 }
 
+static bool perf_ibs_ldlat_event(struct perf_ibs *perf_ibs,
+                                struct perf_event *event)
+{
+       return perf_ibs == &perf_ibs_op &&
+              (ibs_caps & IBS_CAPS_OPLDLAT) &&
+              (event->attr.config1 & 0xFFF);
+}
+
 static int perf_ibs_init(struct perf_event *event)
 {
        struct hw_perf_event *hwc = &event->hw;
@@ -339,6 +347,17 @@ static int perf_ibs_init(struct perf_event *event)
                        return -EINVAL;
        }
 
+       if (perf_ibs_ldlat_event(perf_ibs, event)) {
+               u64 ldlat = event->attr.config1 & 0xFFF;
+
+               if (ldlat < 128 || ldlat > 2048)
+                       return -EINVAL;
+               ldlat >>= 7;
+
+               config |= (ldlat - 1) << 59;
+               config |= IBS_OP_L3MISSONLY | IBS_OP_LDLAT_EN;
+       }
+
        /*
         * If we modify hwc->sample_period, we also need to update
         * hwc->last_period and hwc->period_left.
@@ -607,7 +626,9 @@ PMU_FORMAT_ATTR(cnt_ctl,    "config:19");
 PMU_FORMAT_ATTR(swfilt,                "config2:0");
 PMU_EVENT_ATTR_STRING(l3missonly, fetch_l3missonly, "config:59");
 PMU_EVENT_ATTR_STRING(l3missonly, op_l3missonly, "config:16");
+PMU_EVENT_ATTR_STRING(ldlat, ibs_op_ldlat_format, "config1:0-11");
 PMU_EVENT_ATTR_STRING(zen4_ibs_extensions, zen4_ibs_extensions, "1");
+PMU_EVENT_ATTR_STRING(ldlat, ibs_op_ldlat_cap, "1");
 
 static umode_t
 zen4_ibs_extensions_is_visible(struct kobject *kobj, struct attribute *attr, int i)
@@ -615,6 +636,12 @@ zen4_ibs_extensions_is_visible(struct kobject *kobj, struct attribute *attr, int
        return ibs_caps & IBS_CAPS_ZEN4 ? attr->mode : 0;
 }
 
+static umode_t
+ibs_op_ldlat_is_visible(struct kobject *kobj, struct attribute *attr, int i)
+{
+       return ibs_caps & IBS_CAPS_OPLDLAT ? attr->mode : 0;
+}
+
 static struct attribute *fetch_attrs[] = {
        &format_attr_rand_en.attr,
        &format_attr_swfilt.attr,
@@ -631,6 +658,11 @@ static struct attribute *zen4_ibs_extensions_attrs[] = {
        NULL,
 };
 
+static struct attribute *ibs_op_ldlat_cap_attrs[] = {
+       &ibs_op_ldlat_cap.attr.attr,
+       NULL,
+};
+
 static struct attribute_group group_fetch_formats = {
        .name = "format",
        .attrs = fetch_attrs,
@@ -648,6 +680,12 @@ static struct attribute_group group_zen4_ibs_extensions = {
        .is_visible = zen4_ibs_extensions_is_visible,
 };
 
+static struct attribute_group group_ibs_op_ldlat_cap = {
+       .name = "caps",
+       .attrs = ibs_op_ldlat_cap_attrs,
+       .is_visible = ibs_op_ldlat_is_visible,
+};
+
 static const struct attribute_group *fetch_attr_groups[] = {
        &group_fetch_formats,
        &empty_caps_group,
@@ -686,6 +724,11 @@ static struct attribute_group group_op_formats = {
        .attrs = op_attrs,
 };
 
+static struct attribute *ibs_op_ldlat_format_attrs[] = {
+       &ibs_op_ldlat_format.attr.attr,
+       NULL,
+};
+
 static struct attribute_group group_cnt_ctl = {
        .name = "format",
        .attrs = cnt_ctl_attrs,
@@ -704,10 +747,18 @@ static const struct attribute_group *op_attr_groups[] = {
        NULL,
 };
 
+static struct attribute_group group_ibs_op_ldlat_format = {
+       .name = "format",
+       .attrs = ibs_op_ldlat_format_attrs,
+       .is_visible = ibs_op_ldlat_is_visible,
+};
+
 static const struct attribute_group *op_attr_update[] = {
        &group_cnt_ctl,
        &group_op_l3missonly,
        &group_zen4_ibs_extensions,
+       &group_ibs_op_ldlat_cap,
+       &group_ibs_op_ldlat_format,
        NULL,
 };
 
@@ -1060,15 +1111,25 @@ static void perf_ibs_parse_ld_st_data(__u64 sample_type,
        }
 }
 
-static int perf_ibs_get_offset_max(struct perf_ibs *perf_ibs, u64 sample_type,
+static bool perf_ibs_is_mem_sample_type(struct perf_ibs *perf_ibs,
+                                       struct perf_event *event)
+{
+       u64 sample_type = event->attr.sample_type;
+
+       return perf_ibs == &perf_ibs_op &&
+              sample_type & (PERF_SAMPLE_DATA_SRC |
+                             PERF_SAMPLE_WEIGHT_TYPE |
+                             PERF_SAMPLE_ADDR |
+                             PERF_SAMPLE_PHYS_ADDR);
+}
+
+static int perf_ibs_get_offset_max(struct perf_ibs *perf_ibs,
+                                  struct perf_event *event,
                                   int check_rip)
 {
-       if (sample_type & PERF_SAMPLE_RAW ||
-           (perf_ibs == &perf_ibs_op &&
-            (sample_type & PERF_SAMPLE_DATA_SRC ||
-             sample_type & PERF_SAMPLE_WEIGHT_TYPE ||
-             sample_type & PERF_SAMPLE_ADDR ||
-             sample_type & PERF_SAMPLE_PHYS_ADDR)))
+       if (event->attr.sample_type & PERF_SAMPLE_RAW ||
+           perf_ibs_is_mem_sample_type(perf_ibs, event) ||
+           perf_ibs_ldlat_event(perf_ibs, event))
                return perf_ibs->offset_max;
        else if (check_rip)
                return 3;
@@ -1123,7 +1184,7 @@ fail:
        offset = 1;
        check_rip = (perf_ibs == &perf_ibs_op && (ibs_caps & IBS_CAPS_RIPINVALIDCHK));
 
-       offset_max = perf_ibs_get_offset_max(perf_ibs, event->attr.sample_type, check_rip);
+       offset_max = perf_ibs_get_offset_max(perf_ibs, event, check_rip);
 
        do {
                rdmsrl(msr + offset, *buf++);
@@ -1132,6 +1193,22 @@ fail:
                                       perf_ibs->offset_max,
                                       offset + 1);
        } while (offset < offset_max);
+
+       if (perf_ibs_ldlat_event(perf_ibs, event)) {
+               union ibs_op_data3 op_data3;
+
+               op_data3.val = ibs_data.regs[ibs_op_msr_idx(MSR_AMD64_IBSOPDATA3)];
+               /*
+                * Opening event is errored out if load latency threshold is
+                * outside of [128, 2048] range. Since the event has reached
+                * interrupt handler, we can safely assume the threshold is
+                * within [128, 2048] range.
+                */
+               if (!op_data3.ld_op || !op_data3.dc_miss ||
+                   op_data3.dc_miss_lat <= (event->attr.config1 & 0xFFF))
+                       goto out;
+       }
+
        /*
         * Read IbsBrTarget, IbsOpData4, and IbsExtdCtl separately
         * depending on their availability.

diff --git a/arch/x86/include/asm/amd-ibs.h b/arch/x86/include/asm/amd-ibs.h
index cb2a5e113daa75d50ae8dd1ed338bd1c77db9273..77f3a589a99ac58fdd0e41291adcf1e48a44e2c2 100644 (file)
--- a/arch/x86/include/asm/amd-ibs.h
+++ b/arch/x86/include/asm/amd-ibs.h
@@ -64,7 +64,8 @@ union ibs_op_ctl {
                        opmaxcnt_ext:7, /* 20-26: upper 7 bits of periodic op maximum count */
                        reserved0:5,    /* 27-31: reserved */
                        opcurcnt:27,    /* 32-58: periodic op counter current count */
-                       reserved1:5;    /* 59-63: reserved */
+                       ldlat_thrsh:4,  /* 59-62: Load Latency threshold */
+                       ldlat_en:1;     /* 63: Load Latency enabled */
        };
 };
 

diff --git a/arch/x86/include/asm/perf_event.h b/arch/x86/include/asm/perf_event.h
index 73b104049f8cf9753138ac3adc519e3d8ed9b98d..a60efe47498dec4033215a2f560a5a2b09fe58af 100644 (file)
--- a/arch/x86/include/asm/perf_event.h
+++ b/arch/x86/include/asm/perf_event.h
@@ -502,6 +502,7 @@ struct pebs_cntr_header {
 #define IBS_CAPS_FETCHCTLEXTD          (1U<<9)
 #define IBS_CAPS_OPDATA4               (1U<<10)
 #define IBS_CAPS_ZEN4                  (1U<<11)
+#define IBS_CAPS_OPLDLAT               (1U<<12)
 
 #define IBS_CAPS_DEFAULT               (IBS_CAPS_AVAIL         \
                                         | IBS_CAPS_FETCHSAM    \
@@ -527,6 +528,8 @@ struct pebs_cntr_header {
  * The lower 7 bits of the current count are random bits
  * preloaded by hardware and ignored in software
  */
+#define IBS_OP_LDLAT_EN                (1ULL<<63)
+#define IBS_OP_LDLAT_THRSH     (0xFULL<<59)
 #define IBS_OP_CUR_CNT         (0xFFF80ULL<<32)
 #define IBS_OP_CUR_CNT_RAND    (0x0007FULL<<32)
 #define IBS_OP_CUR_CNT_EXT_MASK        (0x7FULL<<52)