2 * Copyright (C) 2002 Sistina Software (UK) Limited.
3 * Copyright (C) 2006 Red Hat GmbH
5 * This file is released under the GPL.
7 * Kcopyd provides a simple interface for copying an area of one
8 * block-device to one or more other block-devices, with an asynchronous
9 * completion notification.
12 #include <linux/types.h>
13 #include <linux/atomic.h>
14 #include <linux/blkdev.h>
16 #include <linux/init.h>
17 #include <linux/list.h>
18 #include <linux/mempool.h>
19 #include <linux/module.h>
20 #include <linux/pagemap.h>
21 #include <linux/slab.h>
22 #include <linux/vmalloc.h>
23 #include <linux/workqueue.h>
24 #include <linux/mutex.h>
25 #include <linux/delay.h>
26 #include <linux/device-mapper.h>
27 #include <linux/dm-kcopyd.h>
31 #define SUB_JOB_SIZE 128
34 #define RESERVE_PAGES (DIV_ROUND_UP(SUB_JOB_SIZE << SECTOR_SHIFT, PAGE_SIZE))
36 /*-----------------------------------------------------------------
37 * Each kcopyd client has its own little pool of preallocated
38 * pages for kcopyd io.
39 *---------------------------------------------------------------*/
40 struct dm_kcopyd_client
{
41 struct page_list
*pages
;
42 unsigned nr_reserved_pages
;
43 unsigned nr_free_pages
;
45 struct dm_io_client
*io_client
;
47 wait_queue_head_t destroyq
;
51 struct workqueue_struct
*kcopyd_wq
;
52 struct work_struct kcopyd_work
;
54 struct dm_kcopyd_throttle
*throttle
;
59 * We maintain four lists of jobs:
61 * i) jobs waiting for pages
62 * ii) jobs that have pages, and are waiting for the io to be issued.
63 * iii) jobs that don't need to do any IO and just run a callback
64 * iv) jobs that have completed.
66 * All four of these are protected by job_lock.
69 struct list_head callback_jobs
;
70 struct list_head complete_jobs
;
71 struct list_head io_jobs
;
72 struct list_head pages_jobs
;
75 static struct page_list zero_page_list
;
77 static DEFINE_SPINLOCK(throttle_spinlock
);
80 * IO/IDLE accounting slowly decays after (1 << ACCOUNT_INTERVAL_SHIFT) period.
81 * When total_period >= (1 << ACCOUNT_INTERVAL_SHIFT) the counters are divided
84 #define ACCOUNT_INTERVAL_SHIFT SHIFT_HZ
87 * Sleep this number of milliseconds.
89 * The value was decided experimentally.
90 * Smaller values seem to cause an increased copy rate above the limit.
91 * The reason for this is unknown but possibly due to jiffies rounding errors
92 * or read/write cache inside the disk.
94 #define SLEEP_MSEC 100
97 * Maximum number of sleep events. There is a theoretical livelock if more
98 * kcopyd clients do work simultaneously which this limit avoids.
100 #define MAX_SLEEPS 10
102 static void io_job_start(struct dm_kcopyd_throttle
*t
)
104 unsigned throttle
, now
, difference
;
111 spin_lock_irq(&throttle_spinlock
);
113 throttle
= READ_ONCE(t
->throttle
);
115 if (likely(throttle
>= 100))
119 difference
= now
- t
->last_jiffies
;
120 t
->last_jiffies
= now
;
122 t
->io_period
+= difference
;
123 t
->total_period
+= difference
;
126 * Maintain sane values if we got a temporary overflow.
128 if (unlikely(t
->io_period
> t
->total_period
))
129 t
->io_period
= t
->total_period
;
131 if (unlikely(t
->total_period
>= (1 << ACCOUNT_INTERVAL_SHIFT
))) {
132 int shift
= fls(t
->total_period
>> ACCOUNT_INTERVAL_SHIFT
);
133 t
->total_period
>>= shift
;
134 t
->io_period
>>= shift
;
137 skew
= t
->io_period
- throttle
* t
->total_period
/ 100;
139 if (unlikely(skew
> 0) && slept
< MAX_SLEEPS
) {
141 spin_unlock_irq(&throttle_spinlock
);
149 spin_unlock_irq(&throttle_spinlock
);
152 static void io_job_finish(struct dm_kcopyd_throttle
*t
)
159 spin_lock_irqsave(&throttle_spinlock
, flags
);
163 if (likely(READ_ONCE(t
->throttle
) >= 100))
166 if (!t
->num_io_jobs
) {
167 unsigned now
, difference
;
170 difference
= now
- t
->last_jiffies
;
171 t
->last_jiffies
= now
;
173 t
->io_period
+= difference
;
174 t
->total_period
+= difference
;
177 * Maintain sane values if we got a temporary overflow.
179 if (unlikely(t
->io_period
> t
->total_period
))
180 t
->io_period
= t
->total_period
;
184 spin_unlock_irqrestore(&throttle_spinlock
, flags
);
188 static void wake(struct dm_kcopyd_client
*kc
)
190 queue_work(kc
->kcopyd_wq
, &kc
->kcopyd_work
);
194 * Obtain one page for the use of kcopyd.
196 static struct page_list
*alloc_pl(gfp_t gfp
)
198 struct page_list
*pl
;
200 pl
= kmalloc(sizeof(*pl
), gfp
);
204 pl
->page
= alloc_page(gfp
);
213 static void free_pl(struct page_list
*pl
)
215 __free_page(pl
->page
);
220 * Add the provided pages to a client's free page list, releasing
221 * back to the system any beyond the reserved_pages limit.
223 static void kcopyd_put_pages(struct dm_kcopyd_client
*kc
, struct page_list
*pl
)
225 struct page_list
*next
;
230 if (kc
->nr_free_pages
>= kc
->nr_reserved_pages
)
233 pl
->next
= kc
->pages
;
242 static int kcopyd_get_pages(struct dm_kcopyd_client
*kc
,
243 unsigned int nr
, struct page_list
**pages
)
245 struct page_list
*pl
;
250 pl
= alloc_pl(__GFP_NOWARN
| __GFP_NORETRY
| __GFP_KSWAPD_RECLAIM
);
252 /* Use reserved pages */
256 kc
->pages
= pl
->next
;
267 kcopyd_put_pages(kc
, *pages
);
272 * These three functions resize the page pool.
274 static void drop_pages(struct page_list
*pl
)
276 struct page_list
*next
;
286 * Allocate and reserve nr_pages for the use of a specific client.
288 static int client_reserve_pages(struct dm_kcopyd_client
*kc
, unsigned nr_pages
)
291 struct page_list
*pl
= NULL
, *next
;
293 for (i
= 0; i
< nr_pages
; i
++) {
294 next
= alloc_pl(GFP_KERNEL
);
304 kc
->nr_reserved_pages
+= nr_pages
;
305 kcopyd_put_pages(kc
, pl
);
310 static void client_free_pages(struct dm_kcopyd_client
*kc
)
312 BUG_ON(kc
->nr_free_pages
!= kc
->nr_reserved_pages
);
313 drop_pages(kc
->pages
);
315 kc
->nr_free_pages
= kc
->nr_reserved_pages
= 0;
318 /*-----------------------------------------------------------------
319 * kcopyd_jobs need to be allocated by the *clients* of kcopyd,
320 * for this reason we use a mempool to prevent the client from
321 * ever having to do io (which could cause a deadlock).
322 *---------------------------------------------------------------*/
324 struct dm_kcopyd_client
*kc
;
325 struct list_head list
;
329 * Error state of the job.
332 unsigned long write_err
;
335 * Either READ or WRITE
338 struct dm_io_region source
;
341 * The destinations for the transfer.
343 unsigned int num_dests
;
344 struct dm_io_region dests
[DM_KCOPYD_MAX_REGIONS
];
346 struct page_list
*pages
;
349 * Set this to ensure you are notified when the job has
350 * completed. 'context' is for callback to use.
352 dm_kcopyd_notify_fn fn
;
356 * These fields are only used if the job has been split
357 * into more manageable parts.
362 sector_t write_offset
;
364 struct kcopyd_job
*master_job
;
367 static struct kmem_cache
*_job_cache
;
369 int __init
dm_kcopyd_init(void)
371 _job_cache
= kmem_cache_create("kcopyd_job",
372 sizeof(struct kcopyd_job
) * (SPLIT_COUNT
+ 1),
373 __alignof__(struct kcopyd_job
), 0, NULL
);
377 zero_page_list
.next
= &zero_page_list
;
378 zero_page_list
.page
= ZERO_PAGE(0);
383 void dm_kcopyd_exit(void)
385 kmem_cache_destroy(_job_cache
);
390 * Functions to push and pop a job onto the head of a given job
393 static struct kcopyd_job
*pop_io_job(struct list_head
*jobs
,
394 struct dm_kcopyd_client
*kc
)
396 struct kcopyd_job
*job
;
399 * For I/O jobs, pop any read, any write without sequential write
400 * constraint and sequential writes that are at the right position.
402 list_for_each_entry(job
, jobs
, list
) {
403 if (job
->rw
== READ
|| !test_bit(DM_KCOPYD_WRITE_SEQ
, &job
->flags
)) {
404 list_del(&job
->list
);
408 if (job
->write_offset
== job
->master_job
->write_offset
) {
409 job
->master_job
->write_offset
+= job
->source
.count
;
410 list_del(&job
->list
);
418 static struct kcopyd_job
*pop(struct list_head
*jobs
,
419 struct dm_kcopyd_client
*kc
)
421 struct kcopyd_job
*job
= NULL
;
424 spin_lock_irqsave(&kc
->job_lock
, flags
);
426 if (!list_empty(jobs
)) {
427 if (jobs
== &kc
->io_jobs
)
428 job
= pop_io_job(jobs
, kc
);
430 job
= list_entry(jobs
->next
, struct kcopyd_job
, list
);
431 list_del(&job
->list
);
434 spin_unlock_irqrestore(&kc
->job_lock
, flags
);
439 static void push(struct list_head
*jobs
, struct kcopyd_job
*job
)
442 struct dm_kcopyd_client
*kc
= job
->kc
;
444 spin_lock_irqsave(&kc
->job_lock
, flags
);
445 list_add_tail(&job
->list
, jobs
);
446 spin_unlock_irqrestore(&kc
->job_lock
, flags
);
450 static void push_head(struct list_head
*jobs
, struct kcopyd_job
*job
)
453 struct dm_kcopyd_client
*kc
= job
->kc
;
455 spin_lock_irqsave(&kc
->job_lock
, flags
);
456 list_add(&job
->list
, jobs
);
457 spin_unlock_irqrestore(&kc
->job_lock
, flags
);
461 * These three functions process 1 item from the corresponding
467 * > 0: can't process yet.
469 static int run_complete_job(struct kcopyd_job
*job
)
471 void *context
= job
->context
;
472 int read_err
= job
->read_err
;
473 unsigned long write_err
= job
->write_err
;
474 dm_kcopyd_notify_fn fn
= job
->fn
;
475 struct dm_kcopyd_client
*kc
= job
->kc
;
477 if (job
->pages
&& job
->pages
!= &zero_page_list
)
478 kcopyd_put_pages(kc
, job
->pages
);
480 * If this is the master job, the sub jobs have already
481 * completed so we can free everything.
483 if (job
->master_job
== job
) {
484 mutex_destroy(&job
->lock
);
485 mempool_free(job
, &kc
->job_pool
);
487 fn(read_err
, write_err
, context
);
489 if (atomic_dec_and_test(&kc
->nr_jobs
))
490 wake_up(&kc
->destroyq
);
497 static void complete_io(unsigned long error
, void *context
)
499 struct kcopyd_job
*job
= (struct kcopyd_job
*) context
;
500 struct dm_kcopyd_client
*kc
= job
->kc
;
502 io_job_finish(kc
->throttle
);
505 if (op_is_write(job
->rw
))
506 job
->write_err
|= error
;
510 if (!test_bit(DM_KCOPYD_IGNORE_ERROR
, &job
->flags
)) {
511 push(&kc
->complete_jobs
, job
);
517 if (op_is_write(job
->rw
))
518 push(&kc
->complete_jobs
, job
);
522 push(&kc
->io_jobs
, job
);
529 * Request io on as many buffer heads as we can currently get for
532 static int run_io_job(struct kcopyd_job
*job
)
535 struct dm_io_request io_req
= {
538 .mem
.type
= DM_IO_PAGE_LIST
,
539 .mem
.ptr
.pl
= job
->pages
,
541 .notify
.fn
= complete_io
,
542 .notify
.context
= job
,
543 .client
= job
->kc
->io_client
,
547 * If we need to write sequentially and some reads or writes failed,
548 * no point in continuing.
550 if (test_bit(DM_KCOPYD_WRITE_SEQ
, &job
->flags
) &&
551 job
->master_job
->write_err
)
554 io_job_start(job
->kc
->throttle
);
557 r
= dm_io(&io_req
, 1, &job
->source
, NULL
);
559 r
= dm_io(&io_req
, job
->num_dests
, job
->dests
, NULL
);
564 static int run_pages_job(struct kcopyd_job
*job
)
567 unsigned nr_pages
= dm_div_up(job
->dests
[0].count
, PAGE_SIZE
>> 9);
569 r
= kcopyd_get_pages(job
->kc
, nr_pages
, &job
->pages
);
571 /* this job is ready for io */
572 push(&job
->kc
->io_jobs
, job
);
577 /* can't complete now */
584 * Run through a list for as long as possible. Returns the count
585 * of successful jobs.
587 static int process_jobs(struct list_head
*jobs
, struct dm_kcopyd_client
*kc
,
588 int (*fn
) (struct kcopyd_job
*))
590 struct kcopyd_job
*job
;
593 while ((job
= pop(jobs
, kc
))) {
598 /* error this rogue job */
599 if (op_is_write(job
->rw
))
600 job
->write_err
= (unsigned long) -1L;
603 push(&kc
->complete_jobs
, job
);
609 * We couldn't service this job ATM, so
610 * push this job back onto the list.
612 push_head(jobs
, job
);
623 * kcopyd does this every time it's woken up.
625 static void do_work(struct work_struct
*work
)
627 struct dm_kcopyd_client
*kc
= container_of(work
,
628 struct dm_kcopyd_client
, kcopyd_work
);
629 struct blk_plug plug
;
633 * The order that these are called is *very* important.
634 * complete jobs can free some pages for pages jobs.
635 * Pages jobs when successful will jump onto the io jobs
636 * list. io jobs call wake when they complete and it all
639 spin_lock_irqsave(&kc
->job_lock
, flags
);
640 list_splice_tail_init(&kc
->callback_jobs
, &kc
->complete_jobs
);
641 spin_unlock_irqrestore(&kc
->job_lock
, flags
);
643 blk_start_plug(&plug
);
644 process_jobs(&kc
->complete_jobs
, kc
, run_complete_job
);
645 process_jobs(&kc
->pages_jobs
, kc
, run_pages_job
);
646 process_jobs(&kc
->io_jobs
, kc
, run_io_job
);
647 blk_finish_plug(&plug
);
651 * If we are copying a small region we just dispatch a single job
652 * to do the copy, otherwise the io has to be split up into many
655 static void dispatch_job(struct kcopyd_job
*job
)
657 struct dm_kcopyd_client
*kc
= job
->kc
;
658 atomic_inc(&kc
->nr_jobs
);
659 if (unlikely(!job
->source
.count
))
660 push(&kc
->callback_jobs
, job
);
661 else if (job
->pages
== &zero_page_list
)
662 push(&kc
->io_jobs
, job
);
664 push(&kc
->pages_jobs
, job
);
668 static void segment_complete(int read_err
, unsigned long write_err
,
671 /* FIXME: tidy this function */
672 sector_t progress
= 0;
674 struct kcopyd_job
*sub_job
= (struct kcopyd_job
*) context
;
675 struct kcopyd_job
*job
= sub_job
->master_job
;
676 struct dm_kcopyd_client
*kc
= job
->kc
;
678 mutex_lock(&job
->lock
);
680 /* update the error */
685 job
->write_err
|= write_err
;
688 * Only dispatch more work if there hasn't been an error.
690 if ((!job
->read_err
&& !job
->write_err
) ||
691 test_bit(DM_KCOPYD_IGNORE_ERROR
, &job
->flags
)) {
692 /* get the next chunk of work */
693 progress
= job
->progress
;
694 count
= job
->source
.count
- progress
;
696 if (count
> SUB_JOB_SIZE
)
697 count
= SUB_JOB_SIZE
;
699 job
->progress
+= count
;
702 mutex_unlock(&job
->lock
);
708 sub_job
->write_offset
= progress
;
709 sub_job
->source
.sector
+= progress
;
710 sub_job
->source
.count
= count
;
712 for (i
= 0; i
< job
->num_dests
; i
++) {
713 sub_job
->dests
[i
].sector
+= progress
;
714 sub_job
->dests
[i
].count
= count
;
717 sub_job
->fn
= segment_complete
;
718 sub_job
->context
= sub_job
;
719 dispatch_job(sub_job
);
721 } else if (atomic_dec_and_test(&job
->sub_jobs
)) {
724 * Queue the completion callback to the kcopyd thread.
726 * Some callers assume that all the completions are called
727 * from a single thread and don't race with each other.
729 * We must not call the callback directly here because this
730 * code may not be executing in the thread.
732 push(&kc
->complete_jobs
, job
);
738 * Create some sub jobs to share the work between them.
740 static void split_job(struct kcopyd_job
*master_job
)
744 atomic_inc(&master_job
->kc
->nr_jobs
);
746 atomic_set(&master_job
->sub_jobs
, SPLIT_COUNT
);
747 for (i
= 0; i
< SPLIT_COUNT
; i
++) {
748 master_job
[i
+ 1].master_job
= master_job
;
749 segment_complete(0, 0u, &master_job
[i
+ 1]);
753 void dm_kcopyd_copy(struct dm_kcopyd_client
*kc
, struct dm_io_region
*from
,
754 unsigned int num_dests
, struct dm_io_region
*dests
,
755 unsigned int flags
, dm_kcopyd_notify_fn fn
, void *context
)
757 struct kcopyd_job
*job
;
761 * Allocate an array of jobs consisting of one master job
762 * followed by SPLIT_COUNT sub jobs.
764 job
= mempool_alloc(&kc
->job_pool
, GFP_NOIO
);
765 mutex_init(&job
->lock
);
768 * set up for the read.
775 job
->num_dests
= num_dests
;
776 memcpy(&job
->dests
, dests
, sizeof(*dests
) * num_dests
);
779 * If one of the destination is a host-managed zoned block device,
780 * we need to write sequentially. If one of the destination is a
781 * host-aware device, then leave it to the caller to choose what to do.
783 if (!test_bit(DM_KCOPYD_WRITE_SEQ
, &job
->flags
)) {
784 for (i
= 0; i
< job
->num_dests
; i
++) {
785 if (bdev_zoned_model(dests
[i
].bdev
) == BLK_ZONED_HM
) {
786 set_bit(DM_KCOPYD_WRITE_SEQ
, &job
->flags
);
793 * If we need to write sequentially, errors cannot be ignored.
795 if (test_bit(DM_KCOPYD_WRITE_SEQ
, &job
->flags
) &&
796 test_bit(DM_KCOPYD_IGNORE_ERROR
, &job
->flags
))
797 clear_bit(DM_KCOPYD_IGNORE_ERROR
, &job
->flags
);
804 memset(&job
->source
, 0, sizeof job
->source
);
805 job
->source
.count
= job
->dests
[0].count
;
806 job
->pages
= &zero_page_list
;
809 * Use WRITE ZEROES to optimize zeroing if all dests support it.
811 job
->rw
= REQ_OP_WRITE_ZEROES
;
812 for (i
= 0; i
< job
->num_dests
; i
++)
813 if (!bdev_write_zeroes_sectors(job
->dests
[i
].bdev
)) {
820 job
->context
= context
;
821 job
->master_job
= job
;
822 job
->write_offset
= 0;
824 if (job
->source
.count
<= SUB_JOB_SIZE
)
831 EXPORT_SYMBOL(dm_kcopyd_copy
);
833 void dm_kcopyd_zero(struct dm_kcopyd_client
*kc
,
834 unsigned num_dests
, struct dm_io_region
*dests
,
835 unsigned flags
, dm_kcopyd_notify_fn fn
, void *context
)
837 dm_kcopyd_copy(kc
, NULL
, num_dests
, dests
, flags
, fn
, context
);
839 EXPORT_SYMBOL(dm_kcopyd_zero
);
841 void *dm_kcopyd_prepare_callback(struct dm_kcopyd_client
*kc
,
842 dm_kcopyd_notify_fn fn
, void *context
)
844 struct kcopyd_job
*job
;
846 job
= mempool_alloc(&kc
->job_pool
, GFP_NOIO
);
848 memset(job
, 0, sizeof(struct kcopyd_job
));
851 job
->context
= context
;
852 job
->master_job
= job
;
854 atomic_inc(&kc
->nr_jobs
);
858 EXPORT_SYMBOL(dm_kcopyd_prepare_callback
);
860 void dm_kcopyd_do_callback(void *j
, int read_err
, unsigned long write_err
)
862 struct kcopyd_job
*job
= j
;
863 struct dm_kcopyd_client
*kc
= job
->kc
;
865 job
->read_err
= read_err
;
866 job
->write_err
= write_err
;
868 push(&kc
->callback_jobs
, job
);
871 EXPORT_SYMBOL(dm_kcopyd_do_callback
);
874 * Cancels a kcopyd job, eg. someone might be deactivating a
878 int kcopyd_cancel(struct kcopyd_job
*job
, int block
)
885 /*-----------------------------------------------------------------
887 *---------------------------------------------------------------*/
888 struct dm_kcopyd_client
*dm_kcopyd_client_create(struct dm_kcopyd_throttle
*throttle
)
891 struct dm_kcopyd_client
*kc
;
893 kc
= kzalloc(sizeof(*kc
), GFP_KERNEL
);
895 return ERR_PTR(-ENOMEM
);
897 spin_lock_init(&kc
->job_lock
);
898 INIT_LIST_HEAD(&kc
->callback_jobs
);
899 INIT_LIST_HEAD(&kc
->complete_jobs
);
900 INIT_LIST_HEAD(&kc
->io_jobs
);
901 INIT_LIST_HEAD(&kc
->pages_jobs
);
902 kc
->throttle
= throttle
;
904 r
= mempool_init_slab_pool(&kc
->job_pool
, MIN_JOBS
, _job_cache
);
908 INIT_WORK(&kc
->kcopyd_work
, do_work
);
909 kc
->kcopyd_wq
= alloc_workqueue("kcopyd", WQ_MEM_RECLAIM
, 0);
910 if (!kc
->kcopyd_wq
) {
916 kc
->nr_reserved_pages
= kc
->nr_free_pages
= 0;
917 r
= client_reserve_pages(kc
, RESERVE_PAGES
);
919 goto bad_client_pages
;
921 kc
->io_client
= dm_io_client_create();
922 if (IS_ERR(kc
->io_client
)) {
923 r
= PTR_ERR(kc
->io_client
);
927 init_waitqueue_head(&kc
->destroyq
);
928 atomic_set(&kc
->nr_jobs
, 0);
933 client_free_pages(kc
);
935 destroy_workqueue(kc
->kcopyd_wq
);
937 mempool_exit(&kc
->job_pool
);
943 EXPORT_SYMBOL(dm_kcopyd_client_create
);
945 void dm_kcopyd_client_destroy(struct dm_kcopyd_client
*kc
)
947 /* Wait for completion of all jobs submitted by this client. */
948 wait_event(kc
->destroyq
, !atomic_read(&kc
->nr_jobs
));
950 BUG_ON(!list_empty(&kc
->callback_jobs
));
951 BUG_ON(!list_empty(&kc
->complete_jobs
));
952 BUG_ON(!list_empty(&kc
->io_jobs
));
953 BUG_ON(!list_empty(&kc
->pages_jobs
));
954 destroy_workqueue(kc
->kcopyd_wq
);
955 dm_io_client_destroy(kc
->io_client
);
956 client_free_pages(kc
);
957 mempool_exit(&kc
->job_pool
);
960 EXPORT_SYMBOL(dm_kcopyd_client_destroy
);