#include "util/lock-contention.h"
#include <linux/zalloc.h>
#include <linux/string.h>
+#include <api/fs/fs.h>
#include <bpf/bpf.h>
#include <bpf/btf.h>
#include <inttypes.h>
static void check_slab_cache_iter(struct lock_contention *con)
{
- struct btf *btf = btf__load_vmlinux_btf();
s32 ret;
hashmap__init(&slab_hash, slab_cache_hash, slab_cache_equal, /*ctx=*/NULL);
- if (btf == NULL) {
+ con->btf = btf__load_vmlinux_btf();
+ if (con->btf == NULL) {
pr_debug("BTF loading failed: %s\n", strerror(errno));
return;
}
- ret = btf__find_by_name_kind(btf, "bpf_iter__kmem_cache", BTF_KIND_STRUCT);
+ ret = btf__find_by_name_kind(con->btf, "bpf_iter__kmem_cache", BTF_KIND_STRUCT);
if (ret < 0) {
bpf_program__set_autoload(skel->progs.slab_cache_iter, false);
pr_debug("slab cache iterator is not available: %d\n", ret);
- goto out;
+ return;
}
has_slab_iter = true;
bpf_map__set_max_entries(skel->maps.slab_caches, con->map_nr_entries);
-out:
- btf__free(btf);
}
static void run_slab_cache_iter(void)
hashmap__clear(&slab_hash);
}
+static void init_numa_data(struct lock_contention *con)
+{
+ struct symbol *sym;
+ struct map *kmap;
+ char *buf = NULL, *p;
+ size_t len;
+ long last = -1;
+ int ret;
+
+ /*
+ * 'struct zone' is embedded in 'struct pglist_data' as an array.
+ * As we may not have full information of the struct zone in the
+ * (fake) vmlinux.h, let's get the actual size from BTF.
+ */
+ ret = btf__find_by_name_kind(con->btf, "zone", BTF_KIND_STRUCT);
+ if (ret < 0) {
+ pr_debug("cannot get type of struct zone: %d\n", ret);
+ return;
+ }
+
+ ret = btf__resolve_size(con->btf, ret);
+ if (ret < 0) {
+ pr_debug("cannot get size of struct zone: %d\n", ret);
+ return;
+ }
+ skel->rodata->sizeof_zone = ret;
+
+ /* UMA system doesn't have 'node_data[]' - just use contig_page_data. */
+ sym = machine__find_kernel_symbol_by_name(con->machine,
+ "contig_page_data",
+ &kmap);
+ if (sym) {
+ skel->rodata->contig_page_data_addr = map__unmap_ip(kmap, sym->start);
+ map__put(kmap);
+ return;
+ }
+
+ /*
+ * The 'node_data' is an array of pointers to struct pglist_data.
+ * It needs to follow the pointer for each node in BPF to get the
+ * address of struct pglist_data and its zones.
+ */
+ sym = machine__find_kernel_symbol_by_name(con->machine,
+ "node_data",
+ &kmap);
+ if (sym == NULL)
+ return;
+
+ skel->rodata->node_data_addr = map__unmap_ip(kmap, sym->start);
+ map__put(kmap);
+
+ /* get the number of online nodes using the last node number + 1 */
+ ret = sysfs__read_str("devices/system/node/online", &buf, &len);
+ if (ret < 0) {
+ pr_debug("failed to read online node: %d\n", ret);
+ return;
+ }
+
+ p = buf;
+ while (p && *p) {
+ last = strtol(p, &p, 0);
+
+ if (p && (*p == ',' || *p == '-' || *p == '\n'))
+ p++;
+ }
+ skel->rodata->nr_nodes = last + 1;
+ free(buf);
+}
+
int lock_contention_prepare(struct lock_contention *con)
{
int i, fd;
bpf_map__set_max_entries(skel->maps.slab_filter, nslabs);
+ init_numa_data(con);
+
if (lock_contention_bpf__load(skel) < 0) {
pr_err("Failed to load lock-contention BPF skeleton\n");
return -1;
return "rq_lock";
}
+ if (!bpf_map_lookup_elem(lock_fd, &key->lock_addr_or_cgroup, &flags)) {
+ if (flags == LOCK_CLASS_ZONE_LOCK)
+ return "zone_lock";
+ }
+
/* look slab_hash for dynamic locks in a slab object */
if (hashmap__find(&slab_hash, flags & LCB_F_SLAB_ID_MASK, &slab_data)) {
snprintf(name_buf, sizeof(name_buf), "&%s", slab_data->name);
}
exit_slab_cache_iter();
+ btf__free(con->btf);
return 0;
}
/* for collect_lock_syms(). 4096 was rejected by the verifier */
#define MAX_CPUS 1024
+/* for collect_zone_lock(). It should be more than the actual zones. */
+#define MAX_ZONES 10
+
/* lock contention flags from include/trace/events/lock.h */
#define LCB_F_SPIN (1U << 0)
#define LCB_F_READ (1U << 1)
extern struct rq runqueues __ksym;
+const volatile __u64 contig_page_data_addr;
+const volatile __u64 node_data_addr;
+const volatile int nr_nodes;
+const volatile int sizeof_zone;
+
struct rq___old {
raw_spinlock_t lock;
} __attribute__((preserve_access_index));
raw_spinlock_t __lock;
} __attribute__((preserve_access_index));
+static void collect_zone_lock(void)
+{
+ __u64 nr_zones, zone_off;
+ __u64 lock_addr, lock_off;
+ __u32 lock_flag = LOCK_CLASS_ZONE_LOCK;
+
+ zone_off = offsetof(struct pglist_data, node_zones);
+ lock_off = offsetof(struct zone, lock);
+
+ if (contig_page_data_addr) {
+ struct pglist_data *contig_page_data;
+
+ contig_page_data = (void *)(long)contig_page_data_addr;
+ nr_zones = BPF_CORE_READ(contig_page_data, nr_zones);
+
+ for (int i = 0; i < MAX_ZONES; i++) {
+ __u64 zone_addr;
+
+ if (i >= nr_zones)
+ break;
+
+ zone_addr = contig_page_data_addr + (sizeof_zone * i) + zone_off;
+ lock_addr = zone_addr + lock_off;
+
+ bpf_map_update_elem(&lock_syms, &lock_addr, &lock_flag, BPF_ANY);
+ }
+ } else if (nr_nodes > 0) {
+ struct pglist_data **node_data = (void *)(long)node_data_addr;
+
+ for (int i = 0; i < nr_nodes; i++) {
+ struct pglist_data *pgdat = NULL;
+ int err;
+
+ err = bpf_core_read(&pgdat, sizeof(pgdat), &node_data[i]);
+ if (err < 0 || pgdat == NULL)
+ break;
+
+ nr_zones = BPF_CORE_READ(pgdat, nr_zones);
+ for (int k = 0; k < MAX_ZONES; k++) {
+ __u64 zone_addr;
+
+ if (k >= nr_zones)
+ break;
+
+ zone_addr = (__u64)(void *)pgdat + (sizeof_zone * k) + zone_off;
+ lock_addr = zone_addr + lock_off;
+
+ bpf_map_update_elem(&lock_syms, &lock_addr, &lock_flag, BPF_ANY);
+ }
+ }
+ }
+}
+
SEC("raw_tp/bpf_test_finish")
int BPF_PROG(collect_lock_syms)
{
lock_flag = LOCK_CLASS_RQLOCK;
bpf_map_update_elem(&lock_syms, &lock_addr, &lock_flag, BPF_ANY);
}
+
+ collect_zone_lock();
+
return 0;
}
projects / thirdparty / linux.git / commitdiff
