1 // SPDX-License-Identifier: GPL-2.0
3 * Some low level IO code, and hacks for various block layer limitations
5 * Copyright 2010, 2011 Kent Overstreet <kent.overstreet@gmail.com>
6 * Copyright 2012 Google, Inc.
13 #include <linux/blkdev.h>
15 /* Bios with headers */
17 void bch_bbio_free(struct bio
*bio
, struct cache_set
*c
)
19 struct bbio
*b
= container_of(bio
, struct bbio
, bio
);
20 mempool_free(b
, c
->bio_meta
);
23 struct bio
*bch_bbio_alloc(struct cache_set
*c
)
25 struct bbio
*b
= mempool_alloc(c
->bio_meta
, GFP_NOIO
);
26 struct bio
*bio
= &b
->bio
;
28 bio_init(bio
, bio
->bi_inline_vecs
, bucket_pages(c
));
33 void __bch_submit_bbio(struct bio
*bio
, struct cache_set
*c
)
35 struct bbio
*b
= container_of(bio
, struct bbio
, bio
);
37 bio
->bi_iter
.bi_sector
= PTR_OFFSET(&b
->key
, 0);
38 bio_set_dev(bio
, PTR_CACHE(c
, &b
->key
, 0)->bdev
);
40 b
->submit_time_us
= local_clock_us();
41 closure_bio_submit(bio
, bio
->bi_private
);
44 void bch_submit_bbio(struct bio
*bio
, struct cache_set
*c
,
45 struct bkey
*k
, unsigned ptr
)
47 struct bbio
*b
= container_of(bio
, struct bbio
, bio
);
48 bch_bkey_copy_single_ptr(&b
->key
, k
, ptr
);
49 __bch_submit_bbio(bio
, c
);
54 void bch_count_io_errors(struct cache
*ca
, blk_status_t error
, const char *m
)
57 * The halflife of an error is:
58 * log2(1/2)/log2(127/128) * refresh ~= 88 * refresh
61 if (ca
->set
->error_decay
) {
62 unsigned count
= atomic_inc_return(&ca
->io_count
);
64 while (count
> ca
->set
->error_decay
) {
67 unsigned new = count
- ca
->set
->error_decay
;
70 * First we subtract refresh from count; each time we
71 * succesfully do so, we rescale the errors once:
74 count
= atomic_cmpxchg(&ca
->io_count
, old
, new);
79 errors
= atomic_read(&ca
->io_errors
);
82 new = ((uint64_t) errors
* 127) / 128;
83 errors
= atomic_cmpxchg(&ca
->io_errors
,
85 } while (old
!= errors
);
91 char buf
[BDEVNAME_SIZE
];
92 unsigned errors
= atomic_add_return(1 << IO_ERROR_SHIFT
,
94 errors
>>= IO_ERROR_SHIFT
;
96 if (errors
< ca
->set
->error_limit
)
97 pr_err("%s: IO error on %s, recovering",
98 bdevname(ca
->bdev
, buf
), m
);
100 bch_cache_set_error(ca
->set
,
101 "%s: too many IO errors %s",
102 bdevname(ca
->bdev
, buf
), m
);
106 void bch_bbio_count_io_errors(struct cache_set
*c
, struct bio
*bio
,
107 blk_status_t error
, const char *m
)
109 struct bbio
*b
= container_of(bio
, struct bbio
, bio
);
110 struct cache
*ca
= PTR_CACHE(c
, &b
->key
, 0);
112 unsigned threshold
= op_is_write(bio_op(bio
))
113 ? c
->congested_write_threshold_us
114 : c
->congested_read_threshold_us
;
117 unsigned t
= local_clock_us();
119 int us
= t
- b
->submit_time_us
;
120 int congested
= atomic_read(&c
->congested
);
122 if (us
> (int) threshold
) {
124 c
->congested_last_us
= t
;
126 ms
= min(ms
, CONGESTED_MAX
+ congested
);
127 atomic_sub(ms
, &c
->congested
);
128 } else if (congested
< 0)
129 atomic_inc(&c
->congested
);
132 bch_count_io_errors(ca
, error
, m
);
135 void bch_bbio_endio(struct cache_set
*c
, struct bio
*bio
,
136 blk_status_t error
, const char *m
)
138 struct closure
*cl
= bio
->bi_private
;
140 bch_bbio_count_io_errors(c
, bio
, error
, m
);