1 // SPDX-License-Identifier: GPL-2.0
3 * padata.c - generic interface to process data streams in parallel
5 * See Documentation/core-api/padata.rst for more information.
7 * Copyright (C) 2008, 2009 secunet Security Networks AG
8 * Copyright (C) 2008, 2009 Steffen Klassert <steffen.klassert@secunet.com>
10 * Copyright (c) 2020 Oracle and/or its affiliates.
11 * Author: Daniel Jordan <daniel.m.jordan@oracle.com>
14 #include <linux/completion.h>
15 #include <linux/export.h>
16 #include <linux/cpumask.h>
17 #include <linux/err.h>
18 #include <linux/cpu.h>
19 #include <linux/padata.h>
20 #include <linux/mutex.h>
21 #include <linux/sched.h>
22 #include <linux/slab.h>
23 #include <linux/sysfs.h>
24 #include <linux/rcupdate.h>
26 #define PADATA_WORK_ONSTACK 1 /* Work's memory is on stack */
29 struct work_struct pw_work
;
30 struct list_head pw_list
; /* padata_free_works linkage */
34 static DEFINE_SPINLOCK(padata_works_lock
);
35 static struct padata_work
*padata_works
;
36 static LIST_HEAD(padata_free_works
);
38 struct padata_mt_job_state
{
40 struct completion completion
;
41 struct padata_mt_job
*job
;
44 unsigned long chunk_size
;
47 static void padata_free_pd(struct parallel_data
*pd
);
48 static void __init
padata_mt_helper(struct work_struct
*work
);
50 static int padata_index_to_cpu(struct parallel_data
*pd
, int cpu_index
)
54 target_cpu
= cpumask_first(pd
->cpumask
.pcpu
);
55 for (cpu
= 0; cpu
< cpu_index
; cpu
++)
56 target_cpu
= cpumask_next(target_cpu
, pd
->cpumask
.pcpu
);
61 static int padata_cpu_hash(struct parallel_data
*pd
, unsigned int seq_nr
)
64 * Hash the sequence numbers to the cpus by taking
65 * seq_nr mod. number of cpus in use.
67 int cpu_index
= seq_nr
% cpumask_weight(pd
->cpumask
.pcpu
);
69 return padata_index_to_cpu(pd
, cpu_index
);
72 static struct padata_work
*padata_work_alloc(void)
74 struct padata_work
*pw
;
76 lockdep_assert_held(&padata_works_lock
);
78 if (list_empty(&padata_free_works
))
79 return NULL
; /* No more work items allowed to be queued. */
81 pw
= list_first_entry(&padata_free_works
, struct padata_work
, pw_list
);
82 list_del(&pw
->pw_list
);
87 * This function is marked __ref because this function may be optimized in such
88 * a way that it directly refers to work_fn's address, which causes modpost to
89 * complain when work_fn is marked __init. This scenario was observed with clang
90 * LTO, where padata_work_init() was optimized to refer directly to
91 * padata_mt_helper() because the calls to padata_work_init() with other work_fn
92 * values were eliminated or inlined.
94 static void __ref
padata_work_init(struct padata_work
*pw
, work_func_t work_fn
,
95 void *data
, int flags
)
97 if (flags
& PADATA_WORK_ONSTACK
)
98 INIT_WORK_ONSTACK(&pw
->pw_work
, work_fn
);
100 INIT_WORK(&pw
->pw_work
, work_fn
);
104 static int __init
padata_work_alloc_mt(int nworks
, void *data
,
105 struct list_head
*head
)
109 spin_lock(&padata_works_lock
);
110 /* Start at 1 because the current task participates in the job. */
111 for (i
= 1; i
< nworks
; ++i
) {
112 struct padata_work
*pw
= padata_work_alloc();
116 padata_work_init(pw
, padata_mt_helper
, data
, 0);
117 list_add(&pw
->pw_list
, head
);
119 spin_unlock(&padata_works_lock
);
124 static void padata_work_free(struct padata_work
*pw
)
126 lockdep_assert_held(&padata_works_lock
);
127 list_add(&pw
->pw_list
, &padata_free_works
);
130 static void __init
padata_works_free(struct list_head
*works
)
132 struct padata_work
*cur
, *next
;
134 if (list_empty(works
))
137 spin_lock(&padata_works_lock
);
138 list_for_each_entry_safe(cur
, next
, works
, pw_list
) {
139 list_del(&cur
->pw_list
);
140 padata_work_free(cur
);
142 spin_unlock(&padata_works_lock
);
145 static void padata_parallel_worker(struct work_struct
*parallel_work
)
147 struct padata_work
*pw
= container_of(parallel_work
, struct padata_work
,
149 struct padata_priv
*padata
= pw
->pw_data
;
152 padata
->parallel(padata
);
153 spin_lock(&padata_works_lock
);
154 padata_work_free(pw
);
155 spin_unlock(&padata_works_lock
);
160 * padata_do_parallel - padata parallelization function
163 * @padata: object to be parallelized
164 * @cb_cpu: pointer to the CPU that the serialization callback function should
165 * run on. If it's not in the serial cpumask of @pinst
166 * (i.e. cpumask.cbcpu), this function selects a fallback CPU and if
167 * none found, returns -EINVAL.
169 * The parallelization callback function will run with BHs off.
170 * Note: Every object which is parallelized by padata_do_parallel
171 * must be seen by padata_do_serial.
173 * Return: 0 on success or else negative error code.
175 int padata_do_parallel(struct padata_shell
*ps
,
176 struct padata_priv
*padata
, int *cb_cpu
)
178 struct padata_instance
*pinst
= ps
->pinst
;
179 int i
, cpu
, cpu_index
, err
;
180 struct parallel_data
*pd
;
181 struct padata_work
*pw
;
185 pd
= rcu_dereference_bh(ps
->pd
);
188 if (!(pinst
->flags
& PADATA_INIT
) || pinst
->flags
& PADATA_INVALID
)
191 if (!cpumask_test_cpu(*cb_cpu
, pd
->cpumask
.cbcpu
)) {
192 if (cpumask_empty(pd
->cpumask
.cbcpu
))
195 /* Select an alternate fallback CPU and notify the caller. */
196 cpu_index
= *cb_cpu
% cpumask_weight(pd
->cpumask
.cbcpu
);
198 cpu
= cpumask_first(pd
->cpumask
.cbcpu
);
199 for (i
= 0; i
< cpu_index
; i
++)
200 cpu
= cpumask_next(cpu
, pd
->cpumask
.cbcpu
);
206 if ((pinst
->flags
& PADATA_RESET
))
209 refcount_inc(&pd
->refcnt
);
211 padata
->cb_cpu
= *cb_cpu
;
213 spin_lock(&padata_works_lock
);
214 padata
->seq_nr
= ++pd
->seq_nr
;
215 pw
= padata_work_alloc();
216 spin_unlock(&padata_works_lock
);
219 /* Maximum works limit exceeded, run in the current task. */
220 padata
->parallel(padata
);
223 rcu_read_unlock_bh();
226 padata_work_init(pw
, padata_parallel_worker
, padata
, 0);
227 queue_work(pinst
->parallel_wq
, &pw
->pw_work
);
232 rcu_read_unlock_bh();
236 EXPORT_SYMBOL(padata_do_parallel
);
239 * padata_find_next - Find the next object that needs serialization.
242 * * A pointer to the control struct of the next object that needs
243 * serialization, if present in one of the percpu reorder queues.
244 * * NULL, if the next object that needs serialization will
245 * be parallel processed by another cpu and is not yet present in
246 * the cpu's reorder queue.
248 static struct padata_priv
*padata_find_next(struct parallel_data
*pd
,
251 struct padata_priv
*padata
;
252 struct padata_list
*reorder
;
255 reorder
= per_cpu_ptr(pd
->reorder_list
, cpu
);
257 spin_lock(&reorder
->lock
);
258 if (list_empty(&reorder
->list
)) {
259 spin_unlock(&reorder
->lock
);
263 padata
= list_entry(reorder
->list
.next
, struct padata_priv
, list
);
266 * Checks the rare case where two or more parallel jobs have hashed to
267 * the same CPU and one of the later ones finishes first.
269 if (padata
->seq_nr
!= pd
->processed
) {
270 spin_unlock(&reorder
->lock
);
275 list_del_init(&padata
->list
);
277 pd
->cpu
= cpumask_next_wrap(cpu
, pd
->cpumask
.pcpu
, -1, false);
280 spin_unlock(&reorder
->lock
);
284 static void padata_reorder(struct parallel_data
*pd
)
286 struct padata_instance
*pinst
= pd
->ps
->pinst
;
288 struct padata_priv
*padata
;
289 struct padata_serial_queue
*squeue
;
290 struct padata_list
*reorder
;
293 * We need to ensure that only one cpu can work on dequeueing of
294 * the reorder queue the time. Calculating in which percpu reorder
295 * queue the next object will arrive takes some time. A spinlock
296 * would be highly contended. Also it is not clear in which order
297 * the objects arrive to the reorder queues. So a cpu could wait to
298 * get the lock just to notice that there is nothing to do at the
299 * moment. Therefore we use a trylock and let the holder of the lock
300 * care for all the objects enqueued during the holdtime of the lock.
302 if (!spin_trylock_bh(&pd
->lock
))
306 padata
= padata_find_next(pd
, true);
309 * If the next object that needs serialization is parallel
310 * processed by another cpu and is still on it's way to the
311 * cpu's reorder queue, nothing to do for now.
316 cb_cpu
= padata
->cb_cpu
;
317 squeue
= per_cpu_ptr(pd
->squeue
, cb_cpu
);
319 spin_lock(&squeue
->serial
.lock
);
320 list_add_tail(&padata
->list
, &squeue
->serial
.list
);
321 spin_unlock(&squeue
->serial
.lock
);
323 queue_work_on(cb_cpu
, pinst
->serial_wq
, &squeue
->work
);
326 spin_unlock_bh(&pd
->lock
);
329 * The next object that needs serialization might have arrived to
330 * the reorder queues in the meantime.
332 * Ensure reorder queue is read after pd->lock is dropped so we see
333 * new objects from another task in padata_do_serial. Pairs with
334 * smp_mb in padata_do_serial.
338 reorder
= per_cpu_ptr(pd
->reorder_list
, pd
->cpu
);
339 if (!list_empty(&reorder
->list
) && padata_find_next(pd
, false))
340 queue_work(pinst
->serial_wq
, &pd
->reorder_work
);
343 static void invoke_padata_reorder(struct work_struct
*work
)
345 struct parallel_data
*pd
;
348 pd
= container_of(work
, struct parallel_data
, reorder_work
);
353 static void padata_serial_worker(struct work_struct
*serial_work
)
355 struct padata_serial_queue
*squeue
;
356 struct parallel_data
*pd
;
357 LIST_HEAD(local_list
);
361 squeue
= container_of(serial_work
, struct padata_serial_queue
, work
);
364 spin_lock(&squeue
->serial
.lock
);
365 list_replace_init(&squeue
->serial
.list
, &local_list
);
366 spin_unlock(&squeue
->serial
.lock
);
370 while (!list_empty(&local_list
)) {
371 struct padata_priv
*padata
;
373 padata
= list_entry(local_list
.next
,
374 struct padata_priv
, list
);
376 list_del_init(&padata
->list
);
378 padata
->serial(padata
);
383 if (refcount_sub_and_test(cnt
, &pd
->refcnt
))
388 * padata_do_serial - padata serialization function
390 * @padata: object to be serialized.
392 * padata_do_serial must be called for every parallelized object.
393 * The serialization callback function will run with BHs off.
395 void padata_do_serial(struct padata_priv
*padata
)
397 struct parallel_data
*pd
= padata
->pd
;
398 int hashed_cpu
= padata_cpu_hash(pd
, padata
->seq_nr
);
399 struct padata_list
*reorder
= per_cpu_ptr(pd
->reorder_list
, hashed_cpu
);
400 struct padata_priv
*cur
;
401 struct list_head
*pos
;
403 spin_lock(&reorder
->lock
);
404 /* Sort in ascending order of sequence number. */
405 list_for_each_prev(pos
, &reorder
->list
) {
406 cur
= list_entry(pos
, struct padata_priv
, list
);
407 if (cur
->seq_nr
< padata
->seq_nr
)
410 list_add(&padata
->list
, pos
);
411 spin_unlock(&reorder
->lock
);
414 * Ensure the addition to the reorder list is ordered correctly
415 * with the trylock of pd->lock in padata_reorder. Pairs with smp_mb
422 EXPORT_SYMBOL(padata_do_serial
);
424 static int padata_setup_cpumasks(struct padata_instance
*pinst
)
426 struct workqueue_attrs
*attrs
;
429 attrs
= alloc_workqueue_attrs();
433 /* Restrict parallel_wq workers to pd->cpumask.pcpu. */
434 cpumask_copy(attrs
->cpumask
, pinst
->cpumask
.pcpu
);
435 err
= apply_workqueue_attrs(pinst
->parallel_wq
, attrs
);
436 free_workqueue_attrs(attrs
);
441 static void __init
padata_mt_helper(struct work_struct
*w
)
443 struct padata_work
*pw
= container_of(w
, struct padata_work
, pw_work
);
444 struct padata_mt_job_state
*ps
= pw
->pw_data
;
445 struct padata_mt_job
*job
= ps
->job
;
448 spin_lock(&ps
->lock
);
450 while (job
->size
> 0) {
451 unsigned long start
, size
, end
;
454 /* So end is chunk size aligned if enough work remains. */
455 size
= roundup(start
+ 1, ps
->chunk_size
) - start
;
456 size
= min(size
, job
->size
);
462 spin_unlock(&ps
->lock
);
463 job
->thread_fn(start
, end
, job
->fn_arg
);
464 spin_lock(&ps
->lock
);
468 done
= (ps
->nworks_fini
== ps
->nworks
);
469 spin_unlock(&ps
->lock
);
472 complete(&ps
->completion
);
476 * padata_do_multithreaded - run a multithreaded job
477 * @job: Description of the job.
479 * See the definition of struct padata_mt_job for more details.
481 void __init
padata_do_multithreaded(struct padata_mt_job
*job
)
483 /* In case threads finish at different times. */
484 static const unsigned long load_balance_factor
= 4;
485 struct padata_work my_work
, *pw
;
486 struct padata_mt_job_state ps
;
493 /* Ensure at least one thread when size < min_chunk. */
494 nworks
= max(job
->size
/ max(job
->min_chunk
, job
->align
), 1ul);
495 nworks
= min(nworks
, job
->max_threads
);
498 /* Single thread, no coordination needed, cut to the chase. */
499 job
->thread_fn(job
->start
, job
->start
+ job
->size
, job
->fn_arg
);
503 spin_lock_init(&ps
.lock
);
504 init_completion(&ps
.completion
);
506 ps
.nworks
= padata_work_alloc_mt(nworks
, &ps
, &works
);
510 * Chunk size is the amount of work a helper does per call to the
511 * thread function. Load balance large jobs between threads by
512 * increasing the number of chunks, guarantee at least the minimum
513 * chunk size from the caller, and honor the caller's alignment.
515 ps
.chunk_size
= job
->size
/ (ps
.nworks
* load_balance_factor
);
516 ps
.chunk_size
= max(ps
.chunk_size
, job
->min_chunk
);
517 ps
.chunk_size
= roundup(ps
.chunk_size
, job
->align
);
519 list_for_each_entry(pw
, &works
, pw_list
)
520 queue_work(system_unbound_wq
, &pw
->pw_work
);
522 /* Use the current thread, which saves starting a workqueue worker. */
523 padata_work_init(&my_work
, padata_mt_helper
, &ps
, PADATA_WORK_ONSTACK
);
524 padata_mt_helper(&my_work
.pw_work
);
526 /* Wait for all the helpers to finish. */
527 wait_for_completion(&ps
.completion
);
529 destroy_work_on_stack(&my_work
.pw_work
);
530 padata_works_free(&works
);
533 static void __padata_list_init(struct padata_list
*pd_list
)
535 INIT_LIST_HEAD(&pd_list
->list
);
536 spin_lock_init(&pd_list
->lock
);
539 /* Initialize all percpu queues used by serial workers */
540 static void padata_init_squeues(struct parallel_data
*pd
)
543 struct padata_serial_queue
*squeue
;
545 for_each_cpu(cpu
, pd
->cpumask
.cbcpu
) {
546 squeue
= per_cpu_ptr(pd
->squeue
, cpu
);
548 __padata_list_init(&squeue
->serial
);
549 INIT_WORK(&squeue
->work
, padata_serial_worker
);
553 /* Initialize per-CPU reorder lists */
554 static void padata_init_reorder_list(struct parallel_data
*pd
)
557 struct padata_list
*list
;
559 for_each_cpu(cpu
, pd
->cpumask
.pcpu
) {
560 list
= per_cpu_ptr(pd
->reorder_list
, cpu
);
561 __padata_list_init(list
);
565 /* Allocate and initialize the internal cpumask dependend resources. */
566 static struct parallel_data
*padata_alloc_pd(struct padata_shell
*ps
)
568 struct padata_instance
*pinst
= ps
->pinst
;
569 struct parallel_data
*pd
;
571 pd
= kzalloc(sizeof(struct parallel_data
), GFP_KERNEL
);
575 pd
->reorder_list
= alloc_percpu(struct padata_list
);
576 if (!pd
->reorder_list
)
579 pd
->squeue
= alloc_percpu(struct padata_serial_queue
);
581 goto err_free_reorder_list
;
585 if (!alloc_cpumask_var(&pd
->cpumask
.pcpu
, GFP_KERNEL
))
586 goto err_free_squeue
;
587 if (!alloc_cpumask_var(&pd
->cpumask
.cbcpu
, GFP_KERNEL
))
590 cpumask_and(pd
->cpumask
.pcpu
, pinst
->cpumask
.pcpu
, cpu_online_mask
);
591 cpumask_and(pd
->cpumask
.cbcpu
, pinst
->cpumask
.cbcpu
, cpu_online_mask
);
593 padata_init_reorder_list(pd
);
594 padata_init_squeues(pd
);
596 refcount_set(&pd
->refcnt
, 1);
597 spin_lock_init(&pd
->lock
);
598 pd
->cpu
= cpumask_first(pd
->cpumask
.pcpu
);
599 INIT_WORK(&pd
->reorder_work
, invoke_padata_reorder
);
604 free_cpumask_var(pd
->cpumask
.pcpu
);
606 free_percpu(pd
->squeue
);
607 err_free_reorder_list
:
608 free_percpu(pd
->reorder_list
);
615 static void padata_free_pd(struct parallel_data
*pd
)
617 free_cpumask_var(pd
->cpumask
.pcpu
);
618 free_cpumask_var(pd
->cpumask
.cbcpu
);
619 free_percpu(pd
->reorder_list
);
620 free_percpu(pd
->squeue
);
624 static void __padata_start(struct padata_instance
*pinst
)
626 pinst
->flags
|= PADATA_INIT
;
629 static void __padata_stop(struct padata_instance
*pinst
)
631 if (!(pinst
->flags
& PADATA_INIT
))
634 pinst
->flags
&= ~PADATA_INIT
;
639 /* Replace the internal control structure with a new one. */
640 static int padata_replace_one(struct padata_shell
*ps
)
642 struct parallel_data
*pd_new
;
644 pd_new
= padata_alloc_pd(ps
);
648 ps
->opd
= rcu_dereference_protected(ps
->pd
, 1);
649 rcu_assign_pointer(ps
->pd
, pd_new
);
654 static int padata_replace(struct padata_instance
*pinst
)
656 struct padata_shell
*ps
;
659 pinst
->flags
|= PADATA_RESET
;
661 list_for_each_entry(ps
, &pinst
->pslist
, list
) {
662 err
= padata_replace_one(ps
);
669 list_for_each_entry_continue_reverse(ps
, &pinst
->pslist
, list
)
670 if (refcount_dec_and_test(&ps
->opd
->refcnt
))
671 padata_free_pd(ps
->opd
);
673 pinst
->flags
&= ~PADATA_RESET
;
678 /* If cpumask contains no active cpu, we mark the instance as invalid. */
679 static bool padata_validate_cpumask(struct padata_instance
*pinst
,
680 const struct cpumask
*cpumask
)
682 if (!cpumask_intersects(cpumask
, cpu_online_mask
)) {
683 pinst
->flags
|= PADATA_INVALID
;
687 pinst
->flags
&= ~PADATA_INVALID
;
691 static int __padata_set_cpumasks(struct padata_instance
*pinst
,
692 cpumask_var_t pcpumask
,
693 cpumask_var_t cbcpumask
)
698 valid
= padata_validate_cpumask(pinst
, pcpumask
);
700 __padata_stop(pinst
);
704 valid
= padata_validate_cpumask(pinst
, cbcpumask
);
706 __padata_stop(pinst
);
709 cpumask_copy(pinst
->cpumask
.pcpu
, pcpumask
);
710 cpumask_copy(pinst
->cpumask
.cbcpu
, cbcpumask
);
712 err
= padata_setup_cpumasks(pinst
) ?: padata_replace(pinst
);
715 __padata_start(pinst
);
721 * padata_set_cpumask - Sets specified by @cpumask_type cpumask to the value
722 * equivalent to @cpumask.
723 * @pinst: padata instance
724 * @cpumask_type: PADATA_CPU_SERIAL or PADATA_CPU_PARALLEL corresponding
725 * to parallel and serial cpumasks respectively.
726 * @cpumask: the cpumask to use
728 * Return: 0 on success or negative error code
730 int padata_set_cpumask(struct padata_instance
*pinst
, int cpumask_type
,
731 cpumask_var_t cpumask
)
733 struct cpumask
*serial_mask
, *parallel_mask
;
737 mutex_lock(&pinst
->lock
);
739 switch (cpumask_type
) {
740 case PADATA_CPU_PARALLEL
:
741 serial_mask
= pinst
->cpumask
.cbcpu
;
742 parallel_mask
= cpumask
;
744 case PADATA_CPU_SERIAL
:
745 parallel_mask
= pinst
->cpumask
.pcpu
;
746 serial_mask
= cpumask
;
752 err
= __padata_set_cpumasks(pinst
, parallel_mask
, serial_mask
);
755 mutex_unlock(&pinst
->lock
);
760 EXPORT_SYMBOL(padata_set_cpumask
);
762 #ifdef CONFIG_HOTPLUG_CPU
764 static int __padata_add_cpu(struct padata_instance
*pinst
, int cpu
)
768 if (cpumask_test_cpu(cpu
, cpu_online_mask
)) {
769 err
= padata_replace(pinst
);
771 if (padata_validate_cpumask(pinst
, pinst
->cpumask
.pcpu
) &&
772 padata_validate_cpumask(pinst
, pinst
->cpumask
.cbcpu
))
773 __padata_start(pinst
);
779 static int __padata_remove_cpu(struct padata_instance
*pinst
, int cpu
)
783 if (!cpumask_test_cpu(cpu
, cpu_online_mask
)) {
784 if (!padata_validate_cpumask(pinst
, pinst
->cpumask
.pcpu
) ||
785 !padata_validate_cpumask(pinst
, pinst
->cpumask
.cbcpu
))
786 __padata_stop(pinst
);
788 err
= padata_replace(pinst
);
794 static inline int pinst_has_cpu(struct padata_instance
*pinst
, int cpu
)
796 return cpumask_test_cpu(cpu
, pinst
->cpumask
.pcpu
) ||
797 cpumask_test_cpu(cpu
, pinst
->cpumask
.cbcpu
);
800 static int padata_cpu_online(unsigned int cpu
, struct hlist_node
*node
)
802 struct padata_instance
*pinst
;
805 pinst
= hlist_entry_safe(node
, struct padata_instance
, cpu_online_node
);
806 if (!pinst_has_cpu(pinst
, cpu
))
809 mutex_lock(&pinst
->lock
);
810 ret
= __padata_add_cpu(pinst
, cpu
);
811 mutex_unlock(&pinst
->lock
);
815 static int padata_cpu_dead(unsigned int cpu
, struct hlist_node
*node
)
817 struct padata_instance
*pinst
;
820 pinst
= hlist_entry_safe(node
, struct padata_instance
, cpu_dead_node
);
821 if (!pinst_has_cpu(pinst
, cpu
))
824 mutex_lock(&pinst
->lock
);
825 ret
= __padata_remove_cpu(pinst
, cpu
);
826 mutex_unlock(&pinst
->lock
);
830 static enum cpuhp_state hp_online
;
833 static void __padata_free(struct padata_instance
*pinst
)
835 #ifdef CONFIG_HOTPLUG_CPU
836 cpuhp_state_remove_instance_nocalls(CPUHP_PADATA_DEAD
,
837 &pinst
->cpu_dead_node
);
838 cpuhp_state_remove_instance_nocalls(hp_online
, &pinst
->cpu_online_node
);
841 WARN_ON(!list_empty(&pinst
->pslist
));
843 free_cpumask_var(pinst
->cpumask
.pcpu
);
844 free_cpumask_var(pinst
->cpumask
.cbcpu
);
845 destroy_workqueue(pinst
->serial_wq
);
846 destroy_workqueue(pinst
->parallel_wq
);
850 #define kobj2pinst(_kobj) \
851 container_of(_kobj, struct padata_instance, kobj)
852 #define attr2pentry(_attr) \
853 container_of(_attr, struct padata_sysfs_entry, attr)
855 static void padata_sysfs_release(struct kobject
*kobj
)
857 struct padata_instance
*pinst
= kobj2pinst(kobj
);
858 __padata_free(pinst
);
861 struct padata_sysfs_entry
{
862 struct attribute attr
;
863 ssize_t (*show
)(struct padata_instance
*, struct attribute
*, char *);
864 ssize_t (*store
)(struct padata_instance
*, struct attribute
*,
865 const char *, size_t);
868 static ssize_t
show_cpumask(struct padata_instance
*pinst
,
869 struct attribute
*attr
, char *buf
)
871 struct cpumask
*cpumask
;
874 mutex_lock(&pinst
->lock
);
875 if (!strcmp(attr
->name
, "serial_cpumask"))
876 cpumask
= pinst
->cpumask
.cbcpu
;
878 cpumask
= pinst
->cpumask
.pcpu
;
880 len
= snprintf(buf
, PAGE_SIZE
, "%*pb\n",
881 nr_cpu_ids
, cpumask_bits(cpumask
));
882 mutex_unlock(&pinst
->lock
);
883 return len
< PAGE_SIZE
? len
: -EINVAL
;
886 static ssize_t
store_cpumask(struct padata_instance
*pinst
,
887 struct attribute
*attr
,
888 const char *buf
, size_t count
)
890 cpumask_var_t new_cpumask
;
894 if (!alloc_cpumask_var(&new_cpumask
, GFP_KERNEL
))
897 ret
= bitmap_parse(buf
, count
, cpumask_bits(new_cpumask
),
902 mask_type
= !strcmp(attr
->name
, "serial_cpumask") ?
903 PADATA_CPU_SERIAL
: PADATA_CPU_PARALLEL
;
904 ret
= padata_set_cpumask(pinst
, mask_type
, new_cpumask
);
909 free_cpumask_var(new_cpumask
);
913 #define PADATA_ATTR_RW(_name, _show_name, _store_name) \
914 static struct padata_sysfs_entry _name##_attr = \
915 __ATTR(_name, 0644, _show_name, _store_name)
916 #define PADATA_ATTR_RO(_name, _show_name) \
917 static struct padata_sysfs_entry _name##_attr = \
918 __ATTR(_name, 0400, _show_name, NULL)
920 PADATA_ATTR_RW(serial_cpumask
, show_cpumask
, store_cpumask
);
921 PADATA_ATTR_RW(parallel_cpumask
, show_cpumask
, store_cpumask
);
924 * Padata sysfs provides the following objects:
925 * serial_cpumask [RW] - cpumask for serial workers
926 * parallel_cpumask [RW] - cpumask for parallel workers
928 static struct attribute
*padata_default_attrs
[] = {
929 &serial_cpumask_attr
.attr
,
930 ¶llel_cpumask_attr
.attr
,
933 ATTRIBUTE_GROUPS(padata_default
);
935 static ssize_t
padata_sysfs_show(struct kobject
*kobj
,
936 struct attribute
*attr
, char *buf
)
938 struct padata_instance
*pinst
;
939 struct padata_sysfs_entry
*pentry
;
942 pinst
= kobj2pinst(kobj
);
943 pentry
= attr2pentry(attr
);
945 ret
= pentry
->show(pinst
, attr
, buf
);
950 static ssize_t
padata_sysfs_store(struct kobject
*kobj
, struct attribute
*attr
,
951 const char *buf
, size_t count
)
953 struct padata_instance
*pinst
;
954 struct padata_sysfs_entry
*pentry
;
957 pinst
= kobj2pinst(kobj
);
958 pentry
= attr2pentry(attr
);
960 ret
= pentry
->store(pinst
, attr
, buf
, count
);
965 static const struct sysfs_ops padata_sysfs_ops
= {
966 .show
= padata_sysfs_show
,
967 .store
= padata_sysfs_store
,
970 static const struct kobj_type padata_attr_type
= {
971 .sysfs_ops
= &padata_sysfs_ops
,
972 .default_groups
= padata_default_groups
,
973 .release
= padata_sysfs_release
,
977 * padata_alloc - allocate and initialize a padata instance
978 * @name: used to identify the instance
980 * Return: new instance on success, NULL on error
982 struct padata_instance
*padata_alloc(const char *name
)
984 struct padata_instance
*pinst
;
986 pinst
= kzalloc(sizeof(struct padata_instance
), GFP_KERNEL
);
990 pinst
->parallel_wq
= alloc_workqueue("%s_parallel", WQ_UNBOUND
, 0,
992 if (!pinst
->parallel_wq
)
997 pinst
->serial_wq
= alloc_workqueue("%s_serial", WQ_MEM_RECLAIM
|
998 WQ_CPU_INTENSIVE
, 1, name
);
999 if (!pinst
->serial_wq
)
1002 if (!alloc_cpumask_var(&pinst
->cpumask
.pcpu
, GFP_KERNEL
))
1003 goto err_free_serial_wq
;
1004 if (!alloc_cpumask_var(&pinst
->cpumask
.cbcpu
, GFP_KERNEL
)) {
1005 free_cpumask_var(pinst
->cpumask
.pcpu
);
1006 goto err_free_serial_wq
;
1009 INIT_LIST_HEAD(&pinst
->pslist
);
1011 cpumask_copy(pinst
->cpumask
.pcpu
, cpu_possible_mask
);
1012 cpumask_copy(pinst
->cpumask
.cbcpu
, cpu_possible_mask
);
1014 if (padata_setup_cpumasks(pinst
))
1015 goto err_free_masks
;
1017 __padata_start(pinst
);
1019 kobject_init(&pinst
->kobj
, &padata_attr_type
);
1020 mutex_init(&pinst
->lock
);
1022 #ifdef CONFIG_HOTPLUG_CPU
1023 cpuhp_state_add_instance_nocalls_cpuslocked(hp_online
,
1024 &pinst
->cpu_online_node
);
1025 cpuhp_state_add_instance_nocalls_cpuslocked(CPUHP_PADATA_DEAD
,
1026 &pinst
->cpu_dead_node
);
1034 free_cpumask_var(pinst
->cpumask
.pcpu
);
1035 free_cpumask_var(pinst
->cpumask
.cbcpu
);
1037 destroy_workqueue(pinst
->serial_wq
);
1040 destroy_workqueue(pinst
->parallel_wq
);
1046 EXPORT_SYMBOL(padata_alloc
);
1049 * padata_free - free a padata instance
1051 * @pinst: padata instance to free
1053 void padata_free(struct padata_instance
*pinst
)
1055 kobject_put(&pinst
->kobj
);
1057 EXPORT_SYMBOL(padata_free
);
1060 * padata_alloc_shell - Allocate and initialize padata shell.
1062 * @pinst: Parent padata_instance object.
1064 * Return: new shell on success, NULL on error
1066 struct padata_shell
*padata_alloc_shell(struct padata_instance
*pinst
)
1068 struct parallel_data
*pd
;
1069 struct padata_shell
*ps
;
1071 ps
= kzalloc(sizeof(*ps
), GFP_KERNEL
);
1078 pd
= padata_alloc_pd(ps
);
1084 mutex_lock(&pinst
->lock
);
1085 RCU_INIT_POINTER(ps
->pd
, pd
);
1086 list_add(&ps
->list
, &pinst
->pslist
);
1087 mutex_unlock(&pinst
->lock
);
1096 EXPORT_SYMBOL(padata_alloc_shell
);
1099 * padata_free_shell - free a padata shell
1101 * @ps: padata shell to free
1103 void padata_free_shell(struct padata_shell
*ps
)
1105 struct parallel_data
*pd
;
1110 mutex_lock(&ps
->pinst
->lock
);
1111 list_del(&ps
->list
);
1112 pd
= rcu_dereference_protected(ps
->pd
, 1);
1113 if (refcount_dec_and_test(&pd
->refcnt
))
1115 mutex_unlock(&ps
->pinst
->lock
);
1119 EXPORT_SYMBOL(padata_free_shell
);
1121 void __init
padata_init(void)
1123 unsigned int i
, possible_cpus
;
1124 #ifdef CONFIG_HOTPLUG_CPU
1127 ret
= cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN
, "padata:online",
1128 padata_cpu_online
, NULL
);
1133 ret
= cpuhp_setup_state_multi(CPUHP_PADATA_DEAD
, "padata:dead",
1134 NULL
, padata_cpu_dead
);
1136 goto remove_online_state
;
1139 possible_cpus
= num_possible_cpus();
1140 padata_works
= kmalloc_array(possible_cpus
, sizeof(struct padata_work
),
1143 goto remove_dead_state
;
1145 for (i
= 0; i
< possible_cpus
; ++i
)
1146 list_add(&padata_works
[i
].pw_list
, &padata_free_works
);
1151 #ifdef CONFIG_HOTPLUG_CPU
1152 cpuhp_remove_multi_state(CPUHP_PADATA_DEAD
);
1153 remove_online_state
:
1154 cpuhp_remove_multi_state(hp_online
);
1157 pr_warn("padata: initialization failed\n");