From 6baf2730ccaf0bbbe87f10fb34692441942b59ba Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Thu, 7 Nov 2019 15:00:08 -0500 Subject: [PATCH] bcachefs: Inline fast path of bch2_increment_clock() Shaving more cycles. Signed-off-by: Kent Overstreet Signed-off-by: Kent Overstreet --- fs/bcachefs/clock.c | 7 +++---- fs/bcachefs/clock.h | 13 ++++++++++++- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/fs/bcachefs/clock.c b/fs/bcachefs/clock.c index e4486fcbea193..e227753563abd 100644 --- a/fs/bcachefs/clock.c +++ b/fs/bcachefs/clock.c @@ -135,17 +135,16 @@ static struct io_timer *get_expired_timer(struct io_clock *clock, return ret; } -void bch2_increment_clock(struct bch_fs *c, unsigned sectors, int rw) +void __bch2_increment_clock(struct io_clock *clock) { - struct io_clock *clock = &c->io_clock[rw]; struct io_timer *timer; unsigned long now; + unsigned sectors; /* Buffer up one megabyte worth of IO in the percpu counter */ preempt_disable(); - if (likely(this_cpu_add_return(*clock->pcpu_buf, sectors) < - IO_CLOCK_PCPU_SECTORS)) { + if (this_cpu_read(*clock->pcpu_buf) < IO_CLOCK_PCPU_SECTORS) { preempt_enable(); return; } diff --git a/fs/bcachefs/clock.h b/fs/bcachefs/clock.h index 5cb043c579d8b..bfbbca8a207b0 100644 --- a/fs/bcachefs/clock.h +++ b/fs/bcachefs/clock.h @@ -6,7 +6,18 @@ void bch2_io_timer_add(struct io_clock *, struct io_timer *); void bch2_io_timer_del(struct io_clock *, struct io_timer *); void bch2_kthread_io_clock_wait(struct io_clock *, unsigned long, unsigned long); -void bch2_increment_clock(struct bch_fs *, unsigned, int); + +void __bch2_increment_clock(struct io_clock *); + +static inline void bch2_increment_clock(struct bch_fs *c, unsigned sectors, + int rw) +{ + struct io_clock *clock = &c->io_clock[rw]; + + if (unlikely(this_cpu_add_return(*clock->pcpu_buf, sectors) >= + IO_CLOCK_PCPU_SECTORS)) + __bch2_increment_clock(clock); +} void bch2_io_clock_schedule_timeout(struct io_clock *, unsigned long); -- 2.47.2