#include <linux/stringify.h>
#include <linux/bug.h>
#include <linux/sched.h>
+#include <linux/slab.h>
+#include <linux/hashtable.h>
/*
* Per-cpu variables require a unique name although static in some
/*
* Type for the target id, default to int but can be overridden.
+ * A long type can work as hash table key (PER_OBJ) but will be downgraded to
+ * int in the event tracepoint.
+ * Unused for implicit monitors.
*/
#ifndef da_id_type
#define da_id_type int
}
/*
- * da_get_task - return the task associated to the monitor
+ * da_get_target - return the task associated to the monitor
*/
-static inline struct task_struct *da_get_task(struct da_monitor *da_mon)
+static inline struct task_struct *da_get_target(struct da_monitor *da_mon)
{
return container_of(da_mon, struct task_struct, rv[task_mon_slot].da_mon);
}
*/
static inline da_id_type da_get_id(struct da_monitor *da_mon)
{
- return da_get_task(da_mon)->pid;
+ return da_get_target(da_mon)->pid;
}
static void da_monitor_reset_all(void)
da_monitor_reset_all();
}
+
+#elif RV_MON_TYPE == RV_MON_PER_OBJ
+/*
+ * Functions to define, init and get a per-object monitor.
+ */
+
+struct da_monitor_storage {
+ da_id_type id;
+ monitor_target target;
+ union rv_task_monitor rv;
+ struct hlist_node node;
+ struct rcu_head rcu;
+};
+
+#ifndef DA_MONITOR_HT_BITS
+#define DA_MONITOR_HT_BITS 10
+#endif
+static DEFINE_HASHTABLE(da_monitor_ht, DA_MONITOR_HT_BITS);
+
+/*
+ * da_create_empty_storage - pre-allocate an empty storage
+ */
+static inline struct da_monitor_storage *da_create_empty_storage(da_id_type id)
+{
+ struct da_monitor_storage *mon_storage;
+
+ mon_storage = kmalloc_nolock(sizeof(struct da_monitor_storage),
+ __GFP_ZERO, NUMA_NO_NODE);
+ if (!mon_storage)
+ return NULL;
+
+ hash_add_rcu(da_monitor_ht, &mon_storage->node, id);
+ mon_storage->id = id;
+ return mon_storage;
+}
+
+/*
+ * da_create_storage - create the per-object storage
+ *
+ * The caller is responsible to synchronise writers, either with locks or
+ * implicitly. For instance, if da_create_storage is only called from a single
+ * event for target (e.g. sched_switch), it's safe to call this without locks.
+ */
+static inline struct da_monitor *da_create_storage(da_id_type id,
+ monitor_target target,
+ struct da_monitor *da_mon)
+{
+ struct da_monitor_storage *mon_storage;
+
+ if (da_mon)
+ return da_mon;
+
+ mon_storage = da_create_empty_storage(id);
+ if (!mon_storage)
+ return NULL;
+
+ mon_storage->target = target;
+ return &mon_storage->rv.da_mon;
+}
+
+/*
+ * __da_get_mon_storage - get the monitor storage from the hash table
+ */
+static inline struct da_monitor_storage *__da_get_mon_storage(da_id_type id)
+{
+ struct da_monitor_storage *mon_storage;
+
+ lockdep_assert_in_rcu_read_lock();
+ hash_for_each_possible_rcu(da_monitor_ht, mon_storage, node, id) {
+ if (mon_storage->id == id)
+ return mon_storage;
+ }
+
+ return NULL;
+}
+
+/*
+ * da_get_monitor - return the monitor for target
+ */
+static struct da_monitor *da_get_monitor(da_id_type id, monitor_target target)
+{
+ struct da_monitor_storage *mon_storage;
+
+ mon_storage = __da_get_mon_storage(id);
+ return mon_storage ? &mon_storage->rv.da_mon : NULL;
+}
+
+/*
+ * da_get_target - return the object associated to the monitor
+ */
+static inline monitor_target da_get_target(struct da_monitor *da_mon)
+{
+ return container_of(da_mon, struct da_monitor_storage, rv.da_mon)->target;
+}
+
+/*
+ * da_get_id - return the id associated to the monitor
+ */
+static inline da_id_type da_get_id(struct da_monitor *da_mon)
+{
+ return container_of(da_mon, struct da_monitor_storage, rv.da_mon)->id;
+}
+
+/*
+ * da_create_or_get - create the per-object storage if not already there
+ *
+ * This needs a lookup so should be guarded by RCU, the condition is checked
+ * directly in da_create_storage()
+ */
+static inline void da_create_or_get(da_id_type id, monitor_target target)
+{
+ guard(rcu)();
+ da_create_storage(id, target, da_get_monitor(id, target));
+}
+
+/*
+ * da_fill_empty_storage - store the target in a pre-allocated storage
+ *
+ * Can be used as a substitute of da_create_storage when starting a monitor in
+ * an environment where allocation is unsafe.
+ */
+static inline struct da_monitor *da_fill_empty_storage(da_id_type id,
+ monitor_target target,
+ struct da_monitor *da_mon)
+{
+ if (unlikely(da_mon && !da_get_target(da_mon)))
+ container_of(da_mon, struct da_monitor_storage, rv.da_mon)->target = target;
+ return da_mon;
+}
+
+/*
+ * da_get_target_by_id - return the object associated to the id
+ */
+static inline monitor_target da_get_target_by_id(da_id_type id)
+{
+ struct da_monitor_storage *mon_storage;
+
+ guard(rcu)();
+ mon_storage = __da_get_mon_storage(id);
+
+ if (unlikely(!mon_storage))
+ return NULL;
+ return mon_storage->target;
+}
+
+/*
+ * da_destroy_storage - destroy the per-object storage
+ *
+ * The caller is responsible to synchronise writers, either with locks or
+ * implicitly. For instance, if da_destroy_storage is called at sched_exit and
+ * da_create_storage can never occur after that, it's safe to call this without
+ * locks.
+ * This function includes an RCU read-side critical section to synchronise
+ * against da_monitor_destroy().
+ */
+static inline void da_destroy_storage(da_id_type id)
+{
+ struct da_monitor_storage *mon_storage;
+
+ guard(rcu)();
+ mon_storage = __da_get_mon_storage(id);
+
+ if (!mon_storage)
+ return;
+ da_monitor_reset_hook(&mon_storage->rv.da_mon);
+ hash_del_rcu(&mon_storage->node);
+ kfree_rcu(mon_storage, rcu);
+}
+
+static void da_monitor_reset_all(void)
+{
+ struct da_monitor_storage *mon_storage;
+ int bkt;
+
+ rcu_read_lock();
+ hash_for_each_rcu(da_monitor_ht, bkt, mon_storage, node)
+ da_monitor_reset(&mon_storage->rv.da_mon);
+ rcu_read_unlock();
+}
+
+static inline int da_monitor_init(void)
+{
+ hash_init(da_monitor_ht);
+ return 0;
+}
+
+static inline void da_monitor_destroy(void)
+{
+ struct da_monitor_storage *mon_storage;
+ struct hlist_node *tmp;
+ int bkt;
+
+ /*
+ * This function is called after all probes are disabled, we need only
+ * worry about concurrency against old events.
+ */
+ synchronize_rcu();
+ hash_for_each_safe(da_monitor_ht, bkt, tmp, mon_storage, node) {
+ da_monitor_reset_hook(&mon_storage->rv.da_mon);
+ hash_del_rcu(&mon_storage->node);
+ kfree(mon_storage);
+ }
+}
+
+/*
+ * Allow the per-object monitors to run allocation manually, necessary if the
+ * start condition is in a context problematic for allocation (e.g. scheduling).
+ * In such case, if the storage was pre-allocated without a target, set it now.
+ */
+#ifdef DA_SKIP_AUTO_ALLOC
+#define da_prepare_storage da_fill_empty_storage
+#else
+#define da_prepare_storage da_create_storage
+#endif /* DA_SKIP_AUTO_ALLOC */
+
#endif /* RV_MON_TYPE */
#if RV_MON_TYPE == RV_MON_GLOBAL || RV_MON_TYPE == RV_MON_PER_CPU
return 0;
}
-#elif RV_MON_TYPE == RV_MON_PER_TASK
+#elif RV_MON_TYPE == RV_MON_PER_TASK || RV_MON_TYPE == RV_MON_PER_OBJ
/*
- * Trace events for per_task monitors, report the PID of the task.
+ * Trace events for per_task/per_object monitors, report the target id.
*/
static inline void da_trace_event(struct da_monitor *da_mon,
{
return __da_handle_start_run_event(da_get_monitor(tsk), event, tsk->pid);
}
+
+#elif RV_MON_TYPE == RV_MON_PER_OBJ
+/*
+ * Handle event for per object.
+ */
+
+/*
+ * da_handle_event - handle an event
+ */
+static inline void da_handle_event(da_id_type id, monitor_target target, enum events event)
+{
+ struct da_monitor *da_mon;
+
+ guard(rcu)();
+ da_mon = da_get_monitor(id, target);
+ if (likely(da_mon))
+ __da_handle_event(da_mon, event, id);
+}
+
+/*
+ * da_handle_start_event - start monitoring or handle event
+ *
+ * This function is used to notify the monitor that the system is returning
+ * to the initial state, so the monitor can start monitoring in the next event.
+ * Thus:
+ *
+ * If the monitor already started, handle the event.
+ * If the monitor did not start yet, start the monitor but skip the event.
+ */
+static inline bool da_handle_start_event(da_id_type id, monitor_target target,
+ enum events event)
+{
+ struct da_monitor *da_mon;
+
+ guard(rcu)();
+ da_mon = da_get_monitor(id, target);
+ da_mon = da_prepare_storage(id, target, da_mon);
+ if (unlikely(!da_mon))
+ return 0;
+ return __da_handle_start_event(da_mon, event, id);
+}
+
+/*
+ * da_handle_start_run_event - start monitoring and handle event
+ *
+ * This function is used to notify the monitor that the system is in the
+ * initial state, so the monitor can start monitoring and handling event.
+ */
+static inline bool da_handle_start_run_event(da_id_type id, monitor_target target,
+ enum events event)
+{
+ struct da_monitor *da_mon;
+
+ guard(rcu)();
+ da_mon = da_get_monitor(id, target);
+ da_mon = da_prepare_storage(id, target, da_mon);
+ if (unlikely(!da_mon))
+ return 0;
+ return __da_handle_start_run_event(da_mon, event, id);
+}
+
+static inline void da_reset(da_id_type id, monitor_target target)
+{
+ struct da_monitor *da_mon;
+
+ guard(rcu)();
+ da_mon = da_get_monitor(id, target);
+ if (likely(da_mon))
+ da_monitor_reset(da_mon);
+}
#endif /* RV_MON_TYPE */
#endif
projects / thirdparty / linux.git / commitdiff
