#endif
}
+static inline bool can_fill_pool(void)
+{
+ /*
+ * On !RT enabled kernels there are no restrictions and spinlock_t and
+ * raw_spinlock_t are the same types.
+ */
+ if (!IS_ENABLED(CONFIG_PREEMPT_RT))
+ return true;
+
+ /*
+ * On RT enabled kernels, the task must not be blocked on a lock as
+ * that could corrupt the PI state when blocking on a lock in the
+ * allocation path.
+ */
+ if (debug_objects_is_pi_blocked_on())
+ return false;
+
+ /*
+ * On RT enabled kernels the pool refill should happen in preemptible
+ * context.
+ */
+ if (preemptible())
+ return true;
+
+ /*
+ * Though during system boot before scheduling is set up, preemption is
+ * disabled and the pool can get exhausted. Before scheduling is active
+ * a task cannot be blocked on a sleeping lock, but it might hold a lock
+ * and if interrupted then hard interrupt context might run into a lock
+ * inversion. So exclude hard interrupt context from allocations before
+ * scheduling is active.
+ */
+ return system_state < SYSTEM_SCHEDULING && !in_hardirq();
+}
+
static void debug_objects_fill_pool(void)
{
if (!static_branch_likely(&obj_cache_enabled))
if (likely(!pool_should_refill(&pool_global)))
return;
- /*
- * On RT enabled kernels the pool refill must happen in preemptible
- * context and not enqueued on an rt_mutex -- for !RT kernels we rely
- * on the fact that spinlock_t and raw_spinlock_t are basically the
- * same type and this lock-type inversion works just fine.
- */
- if (!IS_ENABLED(CONFIG_PREEMPT_RT) || system_state < SYSTEM_SCHEDULING ||
- (preemptible() && !debug_objects_is_pi_blocked_on())) {
+ if (can_fill_pool()) {
/*
* Annotate away the spinlock_t inside raw_spinlock_t warning
* by temporarily raising the wait-type to LD_WAIT_CONFIG, matching
- * the preemptible() condition above.
+ * the preemptible() condition in can_fill_pool().
*/
static DEFINE_WAIT_OVERRIDE_MAP(fill_pool_map, LD_WAIT_CONFIG);
lock_map_acquire_try(&fill_pool_map);