1 // SPDX-License-Identifier: GPL-2.0
3 * Main bcache entry point - handle a read or a write request and decide what to
4 * do with it; the make_request functions are called by the block layer.
6 * Copyright 2010, 2011 Kent Overstreet <kent.overstreet@gmail.com>
7 * Copyright 2012 Google, Inc.
14 #include "writeback.h"
16 #include <linux/module.h>
17 #include <linux/hash.h>
18 #include <linux/random.h>
19 #include <linux/backing-dev.h>
21 #include <trace/events/bcache.h>
23 #define CUTOFF_CACHE_ADD 95
24 #define CUTOFF_CACHE_READA 90
26 struct kmem_cache
*bch_search_cache
;
28 static void bch_data_insert_start(struct closure
*cl
);
30 static unsigned int cache_mode(struct cached_dev
*dc
)
32 return BDEV_CACHE_MODE(&dc
->sb
);
35 static bool verify(struct cached_dev
*dc
)
40 static void bio_csum(struct bio
*bio
, struct bkey
*k
)
43 struct bvec_iter iter
;
46 bio_for_each_segment(bv
, bio
, iter
) {
47 void *d
= kmap(bv
.bv_page
) + bv
.bv_offset
;
49 csum
= bch_crc64_update(csum
, d
, bv
.bv_len
);
53 k
->ptr
[KEY_PTRS(k
)] = csum
& (~0ULL >> 1);
56 /* Insert data into cache */
58 static void bch_data_insert_keys(struct closure
*cl
)
60 struct data_insert_op
*op
= container_of(cl
, struct data_insert_op
, cl
);
61 atomic_t
*journal_ref
= NULL
;
62 struct bkey
*replace_key
= op
->replace
? &op
->replace_key
: NULL
;
66 journal_ref
= bch_journal(op
->c
, &op
->insert_keys
,
67 op
->flush_journal
? cl
: NULL
);
69 ret
= bch_btree_insert(op
->c
, &op
->insert_keys
,
70 journal_ref
, replace_key
);
72 op
->replace_collision
= true;
74 op
->status
= BLK_STS_RESOURCE
;
75 op
->insert_data_done
= true;
79 atomic_dec_bug(journal_ref
);
81 if (!op
->insert_data_done
) {
82 continue_at(cl
, bch_data_insert_start
, op
->wq
);
86 bch_keylist_free(&op
->insert_keys
);
90 static int bch_keylist_realloc(struct keylist
*l
, unsigned int u64s
,
93 size_t oldsize
= bch_keylist_nkeys(l
);
94 size_t newsize
= oldsize
+ u64s
;
97 * The journalling code doesn't handle the case where the keys to insert
98 * is bigger than an empty write: If we just return -ENOMEM here,
99 * bch_data_insert_keys() will insert the keys created so far
100 * and finish the rest when the keylist is empty.
102 if (newsize
* sizeof(uint64_t) > block_bytes(c
) - sizeof(struct jset
))
105 return __bch_keylist_realloc(l
, u64s
);
108 static void bch_data_invalidate(struct closure
*cl
)
110 struct data_insert_op
*op
= container_of(cl
, struct data_insert_op
, cl
);
111 struct bio
*bio
= op
->bio
;
113 pr_debug("invalidating %i sectors from %llu\n",
114 bio_sectors(bio
), (uint64_t) bio
->bi_iter
.bi_sector
);
116 while (bio_sectors(bio
)) {
117 unsigned int sectors
= min(bio_sectors(bio
),
118 1U << (KEY_SIZE_BITS
- 1));
120 if (bch_keylist_realloc(&op
->insert_keys
, 2, op
->c
))
123 bio
->bi_iter
.bi_sector
+= sectors
;
124 bio
->bi_iter
.bi_size
-= sectors
<< 9;
126 bch_keylist_add(&op
->insert_keys
,
128 bio
->bi_iter
.bi_sector
,
132 op
->insert_data_done
= true;
133 /* get in bch_data_insert() */
136 continue_at(cl
, bch_data_insert_keys
, op
->wq
);
139 static void bch_data_insert_error(struct closure
*cl
)
141 struct data_insert_op
*op
= container_of(cl
, struct data_insert_op
, cl
);
144 * Our data write just errored, which means we've got a bunch of keys to
145 * insert that point to data that wasn't successfully written.
147 * We don't have to insert those keys but we still have to invalidate
148 * that region of the cache - so, if we just strip off all the pointers
149 * from the keys we'll accomplish just that.
152 struct bkey
*src
= op
->insert_keys
.keys
, *dst
= op
->insert_keys
.keys
;
154 while (src
!= op
->insert_keys
.top
) {
155 struct bkey
*n
= bkey_next(src
);
157 SET_KEY_PTRS(src
, 0);
158 memmove(dst
, src
, bkey_bytes(src
));
160 dst
= bkey_next(dst
);
164 op
->insert_keys
.top
= dst
;
166 bch_data_insert_keys(cl
);
169 static void bch_data_insert_endio(struct bio
*bio
)
171 struct closure
*cl
= bio
->bi_private
;
172 struct data_insert_op
*op
= container_of(cl
, struct data_insert_op
, cl
);
174 if (bio
->bi_status
) {
175 /* TODO: We could try to recover from this. */
177 op
->status
= bio
->bi_status
;
178 else if (!op
->replace
)
179 set_closure_fn(cl
, bch_data_insert_error
, op
->wq
);
181 set_closure_fn(cl
, NULL
, NULL
);
184 bch_bbio_endio(op
->c
, bio
, bio
->bi_status
, "writing data to cache");
187 static void bch_data_insert_start(struct closure
*cl
)
189 struct data_insert_op
*op
= container_of(cl
, struct data_insert_op
, cl
);
190 struct bio
*bio
= op
->bio
, *n
;
193 return bch_data_invalidate(cl
);
195 if (atomic_sub_return(bio_sectors(bio
), &op
->c
->sectors_to_gc
) < 0)
199 * Journal writes are marked REQ_PREFLUSH; if the original write was a
200 * flush, it'll wait on the journal write.
202 bio
->bi_opf
&= ~(REQ_PREFLUSH
|REQ_FUA
);
207 struct bio_set
*split
= &op
->c
->bio_split
;
209 /* 1 for the device pointer and 1 for the chksum */
210 if (bch_keylist_realloc(&op
->insert_keys
,
211 3 + (op
->csum
? 1 : 0),
213 continue_at(cl
, bch_data_insert_keys
, op
->wq
);
217 k
= op
->insert_keys
.top
;
219 SET_KEY_INODE(k
, op
->inode
);
220 SET_KEY_OFFSET(k
, bio
->bi_iter
.bi_sector
);
222 if (!bch_alloc_sectors(op
->c
, k
, bio_sectors(bio
),
223 op
->write_point
, op
->write_prio
,
227 n
= bio_next_split(bio
, KEY_SIZE(k
), GFP_NOIO
, split
);
229 n
->bi_end_io
= bch_data_insert_endio
;
233 SET_KEY_DIRTY(k
, true);
235 for (i
= 0; i
< KEY_PTRS(k
); i
++)
236 SET_GC_MARK(PTR_BUCKET(op
->c
, k
, i
),
240 SET_KEY_CSUM(k
, op
->csum
);
244 trace_bcache_cache_insert(k
);
245 bch_keylist_push(&op
->insert_keys
);
247 bio_set_op_attrs(n
, REQ_OP_WRITE
, 0);
248 bch_submit_bbio(n
, op
->c
, k
, 0);
251 op
->insert_data_done
= true;
252 continue_at(cl
, bch_data_insert_keys
, op
->wq
);
255 /* bch_alloc_sectors() blocks if s->writeback = true */
256 BUG_ON(op
->writeback
);
259 * But if it's not a writeback write we'd rather just bail out if
260 * there aren't any buckets ready to write to - it might take awhile and
261 * we might be starving btree writes for gc or something.
266 * Writethrough write: We can't complete the write until we've
267 * updated the index. But we don't want to delay the write while
268 * we wait for buckets to be freed up, so just invalidate the
272 return bch_data_invalidate(cl
);
275 * From a cache miss, we can just insert the keys for the data
276 * we have written or bail out if we didn't do anything.
278 op
->insert_data_done
= true;
281 if (!bch_keylist_empty(&op
->insert_keys
))
282 continue_at(cl
, bch_data_insert_keys
, op
->wq
);
289 * bch_data_insert - stick some data in the cache
290 * @cl: closure pointer.
292 * This is the starting point for any data to end up in a cache device; it could
293 * be from a normal write, or a writeback write, or a write to a flash only
294 * volume - it's also used by the moving garbage collector to compact data in
295 * mostly empty buckets.
297 * It first writes the data to the cache, creating a list of keys to be inserted
298 * (if the data had to be fragmented there will be multiple keys); after the
299 * data is written it calls bch_journal, and after the keys have been added to
300 * the next journal write they're inserted into the btree.
302 * It inserts the data in op->bio; bi_sector is used for the key offset,
303 * and op->inode is used for the key inode.
305 * If op->bypass is true, instead of inserting the data it invalidates the
306 * region of the cache represented by op->bio and op->inode.
308 void bch_data_insert(struct closure
*cl
)
310 struct data_insert_op
*op
= container_of(cl
, struct data_insert_op
, cl
);
312 trace_bcache_write(op
->c
, op
->inode
, op
->bio
,
313 op
->writeback
, op
->bypass
);
315 bch_keylist_init(&op
->insert_keys
);
317 bch_data_insert_start(cl
);
321 * Congested? Return 0 (not congested) or the limit (in sectors)
322 * beyond which we should bypass the cache due to congestion.
324 unsigned int bch_get_congested(const struct cache_set
*c
)
328 if (!c
->congested_read_threshold_us
&&
329 !c
->congested_write_threshold_us
)
332 i
= (local_clock_us() - c
->congested_last_us
) / 1024;
336 i
+= atomic_read(&c
->congested
);
343 i
= fract_exp_two(i
, 6);
345 i
-= hweight32(get_random_u32());
347 return i
> 0 ? i
: 1;
350 static void add_sequential(struct task_struct
*t
)
352 ewma_add(t
->sequential_io_avg
,
353 t
->sequential_io
, 8, 0);
355 t
->sequential_io
= 0;
358 static struct hlist_head
*iohash(struct cached_dev
*dc
, uint64_t k
)
360 return &dc
->io_hash
[hash_64(k
, RECENT_IO_BITS
)];
363 static bool check_should_bypass(struct cached_dev
*dc
, struct bio
*bio
)
365 struct cache_set
*c
= dc
->disk
.c
;
366 unsigned int mode
= cache_mode(dc
);
367 unsigned int sectors
, congested
;
368 struct task_struct
*task
= current
;
371 if (test_bit(BCACHE_DEV_DETACHING
, &dc
->disk
.flags
) ||
372 c
->gc_stats
.in_use
> CUTOFF_CACHE_ADD
||
373 (bio_op(bio
) == REQ_OP_DISCARD
))
376 if (mode
== CACHE_MODE_NONE
||
377 (mode
== CACHE_MODE_WRITEAROUND
&&
378 op_is_write(bio_op(bio
))))
382 * If the bio is for read-ahead or background IO, bypass it or
383 * not depends on the following situations,
384 * - If the IO is for meta data, always cache it and no bypass
385 * - If the IO is not meta data, check dc->cache_reada_policy,
386 * BCH_CACHE_READA_ALL: cache it and not bypass
387 * BCH_CACHE_READA_META_ONLY: not cache it and bypass
388 * That is, read-ahead request for metadata always get cached
389 * (eg, for gfs2 or xfs).
391 if ((bio
->bi_opf
& (REQ_RAHEAD
|REQ_BACKGROUND
))) {
392 if (!(bio
->bi_opf
& (REQ_META
|REQ_PRIO
)) &&
393 (dc
->cache_readahead_policy
!= BCH_CACHE_READA_ALL
))
397 if (bio
->bi_iter
.bi_sector
& (c
->sb
.block_size
- 1) ||
398 bio_sectors(bio
) & (c
->sb
.block_size
- 1)) {
399 pr_debug("skipping unaligned io\n");
403 if (bypass_torture_test(dc
)) {
404 if ((get_random_int() & 3) == 3)
410 congested
= bch_get_congested(c
);
411 if (!congested
&& !dc
->sequential_cutoff
)
414 spin_lock(&dc
->io_lock
);
416 hlist_for_each_entry(i
, iohash(dc
, bio
->bi_iter
.bi_sector
), hash
)
417 if (i
->last
== bio
->bi_iter
.bi_sector
&&
418 time_before(jiffies
, i
->jiffies
))
421 i
= list_first_entry(&dc
->io_lru
, struct io
, lru
);
423 add_sequential(task
);
426 if (i
->sequential
+ bio
->bi_iter
.bi_size
> i
->sequential
)
427 i
->sequential
+= bio
->bi_iter
.bi_size
;
429 i
->last
= bio_end_sector(bio
);
430 i
->jiffies
= jiffies
+ msecs_to_jiffies(5000);
431 task
->sequential_io
= i
->sequential
;
434 hlist_add_head(&i
->hash
, iohash(dc
, i
->last
));
435 list_move_tail(&i
->lru
, &dc
->io_lru
);
437 spin_unlock(&dc
->io_lock
);
439 sectors
= max(task
->sequential_io
,
440 task
->sequential_io_avg
) >> 9;
442 if (dc
->sequential_cutoff
&&
443 sectors
>= dc
->sequential_cutoff
>> 9) {
444 trace_bcache_bypass_sequential(bio
);
448 if (congested
&& sectors
>= congested
) {
449 trace_bcache_bypass_congested(bio
);
454 bch_rescale_priorities(c
, bio_sectors(bio
));
457 bch_mark_sectors_bypassed(c
, dc
, bio_sectors(bio
));
464 /* Stack frame for bio_complete */
468 struct bio
*orig_bio
;
469 struct bio
*cache_miss
;
470 struct bcache_device
*d
;
472 unsigned int insert_bio_sectors
;
473 unsigned int recoverable
:1;
474 unsigned int write
:1;
475 unsigned int read_dirty_data
:1;
476 unsigned int cache_missed
:1;
478 unsigned long start_time
;
481 struct data_insert_op iop
;
484 static void bch_cache_read_endio(struct bio
*bio
)
486 struct bbio
*b
= container_of(bio
, struct bbio
, bio
);
487 struct closure
*cl
= bio
->bi_private
;
488 struct search
*s
= container_of(cl
, struct search
, cl
);
491 * If the bucket was reused while our bio was in flight, we might have
492 * read the wrong data. Set s->error but not error so it doesn't get
493 * counted against the cache device, but we'll still reread the data
494 * from the backing device.
498 s
->iop
.status
= bio
->bi_status
;
499 else if (!KEY_DIRTY(&b
->key
) &&
500 ptr_stale(s
->iop
.c
, &b
->key
, 0)) {
501 atomic_long_inc(&s
->iop
.c
->cache_read_races
);
502 s
->iop
.status
= BLK_STS_IOERR
;
505 bch_bbio_endio(s
->iop
.c
, bio
, bio
->bi_status
, "reading from cache");
509 * Read from a single key, handling the initial cache miss if the key starts in
510 * the middle of the bio
512 static int cache_lookup_fn(struct btree_op
*op
, struct btree
*b
, struct bkey
*k
)
514 struct search
*s
= container_of(op
, struct search
, op
);
515 struct bio
*n
, *bio
= &s
->bio
.bio
;
516 struct bkey
*bio_key
;
519 if (bkey_cmp(k
, &KEY(s
->iop
.inode
, bio
->bi_iter
.bi_sector
, 0)) <= 0)
522 if (KEY_INODE(k
) != s
->iop
.inode
||
523 KEY_START(k
) > bio
->bi_iter
.bi_sector
) {
524 unsigned int bio_sectors
= bio_sectors(bio
);
525 unsigned int sectors
= KEY_INODE(k
) == s
->iop
.inode
526 ? min_t(uint64_t, INT_MAX
,
527 KEY_START(k
) - bio
->bi_iter
.bi_sector
)
529 int ret
= s
->d
->cache_miss(b
, s
, bio
, sectors
);
531 if (ret
!= MAP_CONTINUE
)
534 /* if this was a complete miss we shouldn't get here */
535 BUG_ON(bio_sectors
<= sectors
);
541 /* XXX: figure out best pointer - for multiple cache devices */
544 PTR_BUCKET(b
->c
, k
, ptr
)->prio
= INITIAL_PRIO
;
547 s
->read_dirty_data
= true;
549 n
= bio_next_split(bio
, min_t(uint64_t, INT_MAX
,
550 KEY_OFFSET(k
) - bio
->bi_iter
.bi_sector
),
551 GFP_NOIO
, &s
->d
->bio_split
);
553 bio_key
= &container_of(n
, struct bbio
, bio
)->key
;
554 bch_bkey_copy_single_ptr(bio_key
, k
, ptr
);
556 bch_cut_front(&KEY(s
->iop
.inode
, n
->bi_iter
.bi_sector
, 0), bio_key
);
557 bch_cut_back(&KEY(s
->iop
.inode
, bio_end_sector(n
), 0), bio_key
);
559 n
->bi_end_io
= bch_cache_read_endio
;
560 n
->bi_private
= &s
->cl
;
563 * The bucket we're reading from might be reused while our bio
564 * is in flight, and we could then end up reading the wrong
567 * We guard against this by checking (in cache_read_endio()) if
568 * the pointer is stale again; if so, we treat it as an error
569 * and reread from the backing device (but we don't pass that
570 * error up anywhere).
573 __bch_submit_bbio(n
, b
->c
);
574 return n
== bio
? MAP_DONE
: MAP_CONTINUE
;
577 static void cache_lookup(struct closure
*cl
)
579 struct search
*s
= container_of(cl
, struct search
, iop
.cl
);
580 struct bio
*bio
= &s
->bio
.bio
;
581 struct cached_dev
*dc
;
584 bch_btree_op_init(&s
->op
, -1);
586 ret
= bch_btree_map_keys(&s
->op
, s
->iop
.c
,
587 &KEY(s
->iop
.inode
, bio
->bi_iter
.bi_sector
, 0),
588 cache_lookup_fn
, MAP_END_KEY
);
589 if (ret
== -EAGAIN
) {
590 continue_at(cl
, cache_lookup
, bcache_wq
);
595 * We might meet err when searching the btree, If that happens, we will
596 * get negative ret, in this scenario we should not recover data from
597 * backing device (when cache device is dirty) because we don't know
598 * whether bkeys the read request covered are all clean.
600 * And after that happened, s->iop.status is still its initial value
601 * before we submit s->bio.bio
604 BUG_ON(ret
== -EINTR
);
605 if (s
->d
&& s
->d
->c
&&
606 !UUID_FLASH_ONLY(&s
->d
->c
->uuids
[s
->d
->id
])) {
607 dc
= container_of(s
->d
, struct cached_dev
, disk
);
608 if (dc
&& atomic_read(&dc
->has_dirty
))
609 s
->recoverable
= false;
612 s
->iop
.status
= BLK_STS_IOERR
;
618 /* Common code for the make_request functions */
620 static void request_endio(struct bio
*bio
)
622 struct closure
*cl
= bio
->bi_private
;
624 if (bio
->bi_status
) {
625 struct search
*s
= container_of(cl
, struct search
, cl
);
627 s
->iop
.status
= bio
->bi_status
;
628 /* Only cache read errors are recoverable */
629 s
->recoverable
= false;
636 static void backing_request_endio(struct bio
*bio
)
638 struct closure
*cl
= bio
->bi_private
;
640 if (bio
->bi_status
) {
641 struct search
*s
= container_of(cl
, struct search
, cl
);
642 struct cached_dev
*dc
= container_of(s
->d
,
643 struct cached_dev
, disk
);
645 * If a bio has REQ_PREFLUSH for writeback mode, it is
646 * speically assembled in cached_dev_write() for a non-zero
647 * write request which has REQ_PREFLUSH. we don't set
648 * s->iop.status by this failure, the status will be decided
649 * by result of bch_data_insert() operation.
651 if (unlikely(s
->iop
.writeback
&&
652 bio
->bi_opf
& REQ_PREFLUSH
)) {
653 pr_err("Can't flush %s: returned bi_status %i\n",
654 dc
->backing_dev_name
, bio
->bi_status
);
656 /* set to orig_bio->bi_status in bio_complete() */
657 s
->iop
.status
= bio
->bi_status
;
659 s
->recoverable
= false;
660 /* should count I/O error for backing device here */
661 bch_count_backing_io_errors(dc
, bio
);
668 static void bio_complete(struct search
*s
)
671 bio_end_io_acct(s
->orig_bio
, s
->start_time
);
672 trace_bcache_request_end(s
->d
, s
->orig_bio
);
673 s
->orig_bio
->bi_status
= s
->iop
.status
;
674 bio_endio(s
->orig_bio
);
679 static void do_bio_hook(struct search
*s
,
680 struct bio
*orig_bio
,
681 bio_end_io_t
*end_io_fn
)
683 struct bio
*bio
= &s
->bio
.bio
;
685 bio_init(bio
, NULL
, 0);
686 __bio_clone_fast(bio
, orig_bio
);
688 * bi_end_io can be set separately somewhere else, e.g. the
690 * - cache_bio->bi_end_io from cached_dev_cache_miss()
691 * - n->bi_end_io from cache_lookup_fn()
693 bio
->bi_end_io
= end_io_fn
;
694 bio
->bi_private
= &s
->cl
;
699 static void search_free(struct closure
*cl
)
701 struct search
*s
= container_of(cl
, struct search
, cl
);
703 atomic_dec(&s
->iop
.c
->search_inflight
);
709 closure_debug_destroy(cl
);
710 mempool_free(s
, &s
->iop
.c
->search
);
713 static inline struct search
*search_alloc(struct bio
*bio
,
714 struct bcache_device
*d
)
718 s
= mempool_alloc(&d
->c
->search
, GFP_NOIO
);
720 closure_init(&s
->cl
, NULL
);
721 do_bio_hook(s
, bio
, request_endio
);
722 atomic_inc(&d
->c
->search_inflight
);
725 s
->cache_miss
= NULL
;
729 s
->write
= op_is_write(bio_op(bio
));
730 s
->read_dirty_data
= 0;
731 s
->start_time
= bio_start_io_acct(bio
);
735 s
->iop
.inode
= d
->id
;
736 s
->iop
.write_point
= hash_long((unsigned long) current
, 16);
737 s
->iop
.write_prio
= 0;
740 s
->iop
.flush_journal
= op_is_flush(bio
->bi_opf
);
741 s
->iop
.wq
= bcache_wq
;
748 static void cached_dev_bio_complete(struct closure
*cl
)
750 struct search
*s
= container_of(cl
, struct search
, cl
);
751 struct cached_dev
*dc
= container_of(s
->d
, struct cached_dev
, disk
);
759 static void cached_dev_read_error_done(struct closure
*cl
)
761 struct search
*s
= container_of(cl
, struct search
, cl
);
763 if (s
->iop
.replace_collision
)
764 bch_mark_cache_miss_collision(s
->iop
.c
, s
->d
);
767 bio_free_pages(s
->iop
.bio
);
769 cached_dev_bio_complete(cl
);
772 static void cached_dev_read_error(struct closure
*cl
)
774 struct search
*s
= container_of(cl
, struct search
, cl
);
775 struct bio
*bio
= &s
->bio
.bio
;
778 * If read request hit dirty data (s->read_dirty_data is true),
779 * then recovery a failed read request from cached device may
780 * get a stale data back. So read failure recovery is only
781 * permitted when read request hit clean data in cache device,
782 * or when cache read race happened.
784 if (s
->recoverable
&& !s
->read_dirty_data
) {
785 /* Retry from the backing device: */
786 trace_bcache_read_retry(s
->orig_bio
);
789 do_bio_hook(s
, s
->orig_bio
, backing_request_endio
);
791 /* XXX: invalidate cache */
793 /* I/O request sent to backing device */
794 closure_bio_submit(s
->iop
.c
, bio
, cl
);
797 continue_at(cl
, cached_dev_read_error_done
, NULL
);
800 static void cached_dev_cache_miss_done(struct closure
*cl
)
802 struct search
*s
= container_of(cl
, struct search
, cl
);
803 struct bcache_device
*d
= s
->d
;
805 if (s
->iop
.replace_collision
)
806 bch_mark_cache_miss_collision(s
->iop
.c
, s
->d
);
809 bio_free_pages(s
->iop
.bio
);
811 cached_dev_bio_complete(cl
);
815 static void cached_dev_read_done(struct closure
*cl
)
817 struct search
*s
= container_of(cl
, struct search
, cl
);
818 struct cached_dev
*dc
= container_of(s
->d
, struct cached_dev
, disk
);
821 * We had a cache miss; cache_bio now contains data ready to be inserted
824 * First, we copy the data we just read from cache_bio's bounce buffers
825 * to the buffers the original bio pointed to:
829 bio_reset(s
->iop
.bio
);
830 s
->iop
.bio
->bi_iter
.bi_sector
=
831 s
->cache_miss
->bi_iter
.bi_sector
;
832 bio_copy_dev(s
->iop
.bio
, s
->cache_miss
);
833 s
->iop
.bio
->bi_iter
.bi_size
= s
->insert_bio_sectors
<< 9;
834 bch_bio_map(s
->iop
.bio
, NULL
);
836 bio_copy_data(s
->cache_miss
, s
->iop
.bio
);
838 bio_put(s
->cache_miss
);
839 s
->cache_miss
= NULL
;
842 if (verify(dc
) && s
->recoverable
&& !s
->read_dirty_data
)
843 bch_data_verify(dc
, s
->orig_bio
);
845 closure_get(&dc
->disk
.cl
);
849 !test_bit(CACHE_SET_STOPPING
, &s
->iop
.c
->flags
)) {
850 BUG_ON(!s
->iop
.replace
);
851 closure_call(&s
->iop
.cl
, bch_data_insert
, NULL
, cl
);
854 continue_at(cl
, cached_dev_cache_miss_done
, NULL
);
857 static void cached_dev_read_done_bh(struct closure
*cl
)
859 struct search
*s
= container_of(cl
, struct search
, cl
);
860 struct cached_dev
*dc
= container_of(s
->d
, struct cached_dev
, disk
);
862 bch_mark_cache_accounting(s
->iop
.c
, s
->d
,
863 !s
->cache_missed
, s
->iop
.bypass
);
864 trace_bcache_read(s
->orig_bio
, !s
->cache_missed
, s
->iop
.bypass
);
867 continue_at_nobarrier(cl
, cached_dev_read_error
, bcache_wq
);
868 else if (s
->iop
.bio
|| verify(dc
))
869 continue_at_nobarrier(cl
, cached_dev_read_done
, bcache_wq
);
871 continue_at_nobarrier(cl
, cached_dev_bio_complete
, NULL
);
874 static int cached_dev_cache_miss(struct btree
*b
, struct search
*s
,
875 struct bio
*bio
, unsigned int sectors
)
877 int ret
= MAP_CONTINUE
;
878 unsigned int reada
= 0;
879 struct cached_dev
*dc
= container_of(s
->d
, struct cached_dev
, disk
);
880 struct bio
*miss
, *cache_bio
;
884 if (s
->cache_miss
|| s
->iop
.bypass
) {
885 miss
= bio_next_split(bio
, sectors
, GFP_NOIO
, &s
->d
->bio_split
);
886 ret
= miss
== bio
? MAP_DONE
: MAP_CONTINUE
;
890 if (!(bio
->bi_opf
& REQ_RAHEAD
) &&
891 !(bio
->bi_opf
& (REQ_META
|REQ_PRIO
)) &&
892 s
->iop
.c
->gc_stats
.in_use
< CUTOFF_CACHE_READA
)
893 reada
= min_t(sector_t
, dc
->readahead
>> 9,
894 get_capacity(bio
->bi_disk
) - bio_end_sector(bio
));
896 s
->insert_bio_sectors
= min(sectors
, bio_sectors(bio
) + reada
);
898 s
->iop
.replace_key
= KEY(s
->iop
.inode
,
899 bio
->bi_iter
.bi_sector
+ s
->insert_bio_sectors
,
900 s
->insert_bio_sectors
);
902 ret
= bch_btree_insert_check_key(b
, &s
->op
, &s
->iop
.replace_key
);
906 s
->iop
.replace
= true;
908 miss
= bio_next_split(bio
, sectors
, GFP_NOIO
, &s
->d
->bio_split
);
910 /* btree_search_recurse()'s btree iterator is no good anymore */
911 ret
= miss
== bio
? MAP_DONE
: -EINTR
;
913 cache_bio
= bio_alloc_bioset(GFP_NOWAIT
,
914 DIV_ROUND_UP(s
->insert_bio_sectors
, PAGE_SECTORS
),
915 &dc
->disk
.bio_split
);
919 cache_bio
->bi_iter
.bi_sector
= miss
->bi_iter
.bi_sector
;
920 bio_copy_dev(cache_bio
, miss
);
921 cache_bio
->bi_iter
.bi_size
= s
->insert_bio_sectors
<< 9;
923 cache_bio
->bi_end_io
= backing_request_endio
;
924 cache_bio
->bi_private
= &s
->cl
;
926 bch_bio_map(cache_bio
, NULL
);
927 if (bch_bio_alloc_pages(cache_bio
, __GFP_NOWARN
|GFP_NOIO
))
931 bch_mark_cache_readahead(s
->iop
.c
, s
->d
);
933 s
->cache_miss
= miss
;
934 s
->iop
.bio
= cache_bio
;
936 /* I/O request sent to backing device */
937 closure_bio_submit(s
->iop
.c
, cache_bio
, &s
->cl
);
943 miss
->bi_end_io
= backing_request_endio
;
944 miss
->bi_private
= &s
->cl
;
945 /* I/O request sent to backing device */
946 closure_bio_submit(s
->iop
.c
, miss
, &s
->cl
);
950 static void cached_dev_read(struct cached_dev
*dc
, struct search
*s
)
952 struct closure
*cl
= &s
->cl
;
954 closure_call(&s
->iop
.cl
, cache_lookup
, NULL
, cl
);
955 continue_at(cl
, cached_dev_read_done_bh
, NULL
);
960 static void cached_dev_write_complete(struct closure
*cl
)
962 struct search
*s
= container_of(cl
, struct search
, cl
);
963 struct cached_dev
*dc
= container_of(s
->d
, struct cached_dev
, disk
);
965 up_read_non_owner(&dc
->writeback_lock
);
966 cached_dev_bio_complete(cl
);
969 static void cached_dev_write(struct cached_dev
*dc
, struct search
*s
)
971 struct closure
*cl
= &s
->cl
;
972 struct bio
*bio
= &s
->bio
.bio
;
973 struct bkey start
= KEY(dc
->disk
.id
, bio
->bi_iter
.bi_sector
, 0);
974 struct bkey end
= KEY(dc
->disk
.id
, bio_end_sector(bio
), 0);
976 bch_keybuf_check_overlapping(&s
->iop
.c
->moving_gc_keys
, &start
, &end
);
978 down_read_non_owner(&dc
->writeback_lock
);
979 if (bch_keybuf_check_overlapping(&dc
->writeback_keys
, &start
, &end
)) {
981 * We overlap with some dirty data undergoing background
982 * writeback, force this write to writeback
984 s
->iop
.bypass
= false;
985 s
->iop
.writeback
= true;
989 * Discards aren't _required_ to do anything, so skipping if
990 * check_overlapping returned true is ok
992 * But check_overlapping drops dirty keys for which io hasn't started,
993 * so we still want to call it.
995 if (bio_op(bio
) == REQ_OP_DISCARD
)
996 s
->iop
.bypass
= true;
998 if (should_writeback(dc
, s
->orig_bio
,
1001 s
->iop
.bypass
= false;
1002 s
->iop
.writeback
= true;
1005 if (s
->iop
.bypass
) {
1006 s
->iop
.bio
= s
->orig_bio
;
1007 bio_get(s
->iop
.bio
);
1009 if (bio_op(bio
) == REQ_OP_DISCARD
&&
1010 !blk_queue_discard(bdev_get_queue(dc
->bdev
)))
1013 /* I/O request sent to backing device */
1014 bio
->bi_end_io
= backing_request_endio
;
1015 closure_bio_submit(s
->iop
.c
, bio
, cl
);
1017 } else if (s
->iop
.writeback
) {
1018 bch_writeback_add(dc
);
1021 if (bio
->bi_opf
& REQ_PREFLUSH
) {
1023 * Also need to send a flush to the backing
1028 flush
= bio_alloc_bioset(GFP_NOIO
, 0,
1029 &dc
->disk
.bio_split
);
1031 s
->iop
.status
= BLK_STS_RESOURCE
;
1034 bio_copy_dev(flush
, bio
);
1035 flush
->bi_end_io
= backing_request_endio
;
1036 flush
->bi_private
= cl
;
1037 flush
->bi_opf
= REQ_OP_WRITE
| REQ_PREFLUSH
;
1038 /* I/O request sent to backing device */
1039 closure_bio_submit(s
->iop
.c
, flush
, cl
);
1042 s
->iop
.bio
= bio_clone_fast(bio
, GFP_NOIO
, &dc
->disk
.bio_split
);
1043 /* I/O request sent to backing device */
1044 bio
->bi_end_io
= backing_request_endio
;
1045 closure_bio_submit(s
->iop
.c
, bio
, cl
);
1049 closure_call(&s
->iop
.cl
, bch_data_insert
, NULL
, cl
);
1050 continue_at(cl
, cached_dev_write_complete
, NULL
);
1053 static void cached_dev_nodata(struct closure
*cl
)
1055 struct search
*s
= container_of(cl
, struct search
, cl
);
1056 struct bio
*bio
= &s
->bio
.bio
;
1058 if (s
->iop
.flush_journal
)
1059 bch_journal_meta(s
->iop
.c
, cl
);
1061 /* If it's a flush, we send the flush to the backing device too */
1062 bio
->bi_end_io
= backing_request_endio
;
1063 closure_bio_submit(s
->iop
.c
, bio
, cl
);
1065 continue_at(cl
, cached_dev_bio_complete
, NULL
);
1068 struct detached_dev_io_private
{
1069 struct bcache_device
*d
;
1070 unsigned long start_time
;
1071 bio_end_io_t
*bi_end_io
;
1075 static void detached_dev_end_io(struct bio
*bio
)
1077 struct detached_dev_io_private
*ddip
;
1079 ddip
= bio
->bi_private
;
1080 bio
->bi_end_io
= ddip
->bi_end_io
;
1081 bio
->bi_private
= ddip
->bi_private
;
1083 bio_end_io_acct(bio
, ddip
->start_time
);
1085 if (bio
->bi_status
) {
1086 struct cached_dev
*dc
= container_of(ddip
->d
,
1087 struct cached_dev
, disk
);
1088 /* should count I/O error for backing device here */
1089 bch_count_backing_io_errors(dc
, bio
);
1093 bio
->bi_end_io(bio
);
1096 static void detached_dev_do_request(struct bcache_device
*d
, struct bio
*bio
)
1098 struct detached_dev_io_private
*ddip
;
1099 struct cached_dev
*dc
= container_of(d
, struct cached_dev
, disk
);
1102 * no need to call closure_get(&dc->disk.cl),
1103 * because upper layer had already opened bcache device,
1104 * which would call closure_get(&dc->disk.cl)
1106 ddip
= kzalloc(sizeof(struct detached_dev_io_private
), GFP_NOIO
);
1108 ddip
->start_time
= bio_start_io_acct(bio
);
1109 ddip
->bi_end_io
= bio
->bi_end_io
;
1110 ddip
->bi_private
= bio
->bi_private
;
1111 bio
->bi_end_io
= detached_dev_end_io
;
1112 bio
->bi_private
= ddip
;
1114 if ((bio_op(bio
) == REQ_OP_DISCARD
) &&
1115 !blk_queue_discard(bdev_get_queue(dc
->bdev
)))
1116 bio
->bi_end_io(bio
);
1118 generic_make_request(bio
);
1121 static void quit_max_writeback_rate(struct cache_set
*c
,
1122 struct cached_dev
*this_dc
)
1125 struct bcache_device
*d
;
1126 struct cached_dev
*dc
;
1129 * mutex bch_register_lock may compete with other parallel requesters,
1130 * or attach/detach operations on other backing device. Waiting to
1131 * the mutex lock may increase I/O request latency for seconds or more.
1132 * To avoid such situation, if mutext_trylock() failed, only writeback
1133 * rate of current cached device is set to 1, and __update_write_back()
1134 * will decide writeback rate of other cached devices (remember now
1135 * c->idle_counter is 0 already).
1137 if (mutex_trylock(&bch_register_lock
)) {
1138 for (i
= 0; i
< c
->devices_max_used
; i
++) {
1142 if (UUID_FLASH_ONLY(&c
->uuids
[i
]))
1146 dc
= container_of(d
, struct cached_dev
, disk
);
1148 * set writeback rate to default minimum value,
1149 * then let update_writeback_rate() to decide the
1152 atomic_long_set(&dc
->writeback_rate
.rate
, 1);
1154 mutex_unlock(&bch_register_lock
);
1156 atomic_long_set(&this_dc
->writeback_rate
.rate
, 1);
1159 /* Cached devices - read & write stuff */
1161 blk_qc_t
cached_dev_make_request(struct request_queue
*q
, struct bio
*bio
)
1164 struct bcache_device
*d
= bio
->bi_disk
->private_data
;
1165 struct cached_dev
*dc
= container_of(d
, struct cached_dev
, disk
);
1166 int rw
= bio_data_dir(bio
);
1168 if (unlikely((d
->c
&& test_bit(CACHE_SET_IO_DISABLE
, &d
->c
->flags
)) ||
1170 bio
->bi_status
= BLK_STS_IOERR
;
1172 return BLK_QC_T_NONE
;
1176 if (atomic_read(&d
->c
->idle_counter
))
1177 atomic_set(&d
->c
->idle_counter
, 0);
1179 * If at_max_writeback_rate of cache set is true and new I/O
1180 * comes, quit max writeback rate of all cached devices
1181 * attached to this cache set, and set at_max_writeback_rate
1184 if (unlikely(atomic_read(&d
->c
->at_max_writeback_rate
) == 1)) {
1185 atomic_set(&d
->c
->at_max_writeback_rate
, 0);
1186 quit_max_writeback_rate(d
->c
, dc
);
1190 bio_set_dev(bio
, dc
->bdev
);
1191 bio
->bi_iter
.bi_sector
+= dc
->sb
.data_offset
;
1193 if (cached_dev_get(dc
)) {
1194 s
= search_alloc(bio
, d
);
1195 trace_bcache_request_start(s
->d
, bio
);
1197 if (!bio
->bi_iter
.bi_size
) {
1199 * can't call bch_journal_meta from under
1200 * generic_make_request
1202 continue_at_nobarrier(&s
->cl
,
1206 s
->iop
.bypass
= check_should_bypass(dc
, bio
);
1209 cached_dev_write(dc
, s
);
1211 cached_dev_read(dc
, s
);
1214 /* I/O request sent to backing device */
1215 detached_dev_do_request(d
, bio
);
1217 return BLK_QC_T_NONE
;
1220 static int cached_dev_ioctl(struct bcache_device
*d
, fmode_t mode
,
1221 unsigned int cmd
, unsigned long arg
)
1223 struct cached_dev
*dc
= container_of(d
, struct cached_dev
, disk
);
1228 return __blkdev_driver_ioctl(dc
->bdev
, mode
, cmd
, arg
);
1231 static int cached_dev_congested(void *data
, int bits
)
1233 struct bcache_device
*d
= data
;
1234 struct cached_dev
*dc
= container_of(d
, struct cached_dev
, disk
);
1235 struct request_queue
*q
= bdev_get_queue(dc
->bdev
);
1238 if (bdi_congested(q
->backing_dev_info
, bits
))
1241 if (cached_dev_get(dc
)) {
1245 for_each_cache(ca
, d
->c
, i
) {
1246 q
= bdev_get_queue(ca
->bdev
);
1247 ret
|= bdi_congested(q
->backing_dev_info
, bits
);
1256 void bch_cached_dev_request_init(struct cached_dev
*dc
)
1258 struct gendisk
*g
= dc
->disk
.disk
;
1260 g
->queue
->backing_dev_info
->congested_fn
= cached_dev_congested
;
1261 dc
->disk
.cache_miss
= cached_dev_cache_miss
;
1262 dc
->disk
.ioctl
= cached_dev_ioctl
;
1265 /* Flash backed devices */
1267 static int flash_dev_cache_miss(struct btree
*b
, struct search
*s
,
1268 struct bio
*bio
, unsigned int sectors
)
1270 unsigned int bytes
= min(sectors
, bio_sectors(bio
)) << 9;
1272 swap(bio
->bi_iter
.bi_size
, bytes
);
1274 swap(bio
->bi_iter
.bi_size
, bytes
);
1276 bio_advance(bio
, bytes
);
1278 if (!bio
->bi_iter
.bi_size
)
1281 return MAP_CONTINUE
;
1284 static void flash_dev_nodata(struct closure
*cl
)
1286 struct search
*s
= container_of(cl
, struct search
, cl
);
1288 if (s
->iop
.flush_journal
)
1289 bch_journal_meta(s
->iop
.c
, cl
);
1291 continue_at(cl
, search_free
, NULL
);
1294 blk_qc_t
flash_dev_make_request(struct request_queue
*q
, struct bio
*bio
)
1298 struct bcache_device
*d
= bio
->bi_disk
->private_data
;
1300 if (unlikely(d
->c
&& test_bit(CACHE_SET_IO_DISABLE
, &d
->c
->flags
))) {
1301 bio
->bi_status
= BLK_STS_IOERR
;
1303 return BLK_QC_T_NONE
;
1306 s
= search_alloc(bio
, d
);
1310 trace_bcache_request_start(s
->d
, bio
);
1312 if (!bio
->bi_iter
.bi_size
) {
1314 * can't call bch_journal_meta from under
1315 * generic_make_request
1317 continue_at_nobarrier(&s
->cl
,
1320 return BLK_QC_T_NONE
;
1321 } else if (bio_data_dir(bio
)) {
1322 bch_keybuf_check_overlapping(&s
->iop
.c
->moving_gc_keys
,
1323 &KEY(d
->id
, bio
->bi_iter
.bi_sector
, 0),
1324 &KEY(d
->id
, bio_end_sector(bio
), 0));
1326 s
->iop
.bypass
= (bio_op(bio
) == REQ_OP_DISCARD
) != 0;
1327 s
->iop
.writeback
= true;
1330 closure_call(&s
->iop
.cl
, bch_data_insert
, NULL
, cl
);
1332 closure_call(&s
->iop
.cl
, cache_lookup
, NULL
, cl
);
1335 continue_at(cl
, search_free
, NULL
);
1336 return BLK_QC_T_NONE
;
1339 static int flash_dev_ioctl(struct bcache_device
*d
, fmode_t mode
,
1340 unsigned int cmd
, unsigned long arg
)
1345 static int flash_dev_congested(void *data
, int bits
)
1347 struct bcache_device
*d
= data
;
1348 struct request_queue
*q
;
1353 for_each_cache(ca
, d
->c
, i
) {
1354 q
= bdev_get_queue(ca
->bdev
);
1355 ret
|= bdi_congested(q
->backing_dev_info
, bits
);
1361 void bch_flash_dev_request_init(struct bcache_device
*d
)
1363 struct gendisk
*g
= d
->disk
;
1365 g
->queue
->backing_dev_info
->congested_fn
= flash_dev_congested
;
1366 d
->cache_miss
= flash_dev_cache_miss
;
1367 d
->ioctl
= flash_dev_ioctl
;
1370 void bch_request_exit(void)
1372 kmem_cache_destroy(bch_search_cache
);
1375 int __init
bch_request_init(void)
1377 bch_search_cache
= KMEM_CACHE(search
, 0);
1378 if (!bch_search_cache
)