bcachefs: Increase JOURNAL_BUF_NR

Increase journal pipelining.

Signed-off-by: Kent Overstreet <kent.overstreet@linux.dev>

diff --git a/fs/bcachefs/bcachefs.h b/fs/bcachefs/bcachefs.h
index 13acfbf3852aef2ed6600625aecdfb1a844e032a..9791bfe088958d3c843cd83e06a55e2b76f3b201 100644 (file)
--- a/fs/bcachefs/bcachefs.h
+++ b/fs/bcachefs/bcachefs.h
@@ -456,6 +456,7 @@ BCH_DEBUG_PARAMS_DEBUG()
        x(blocked_journal_low_on_space)         \
        x(blocked_journal_low_on_pin)           \
        x(blocked_journal_max_in_flight)        \
+       x(blocked_journal_max_open)             \
        x(blocked_key_cache_flush)              \
        x(blocked_allocate)                     \
        x(blocked_allocate_open_bucket)         \

diff --git a/fs/bcachefs/journal.c b/fs/bcachefs/journal.c
index d47a4dfa03e3b848b4cfb4f8b0c72e087d677f97..40d3ad5a1e5c5afe11d86e49878f444679080d1b 100644 (file)
--- a/fs/bcachefs/journal.c
+++ b/fs/bcachefs/journal.c
@@ -56,7 +56,12 @@ static void bch2_journal_buf_to_text(struct printbuf *out, struct journal *j, u6
        prt_printf(out, "seq:\t%llu\n", seq);
        printbuf_indent_add(out, 2);
 
-       prt_printf(out, "refcount:\t%u\n", journal_state_count(s, i));
+       if (!buf->write_started)
+               prt_printf(out, "refcount:\t%u\n", journal_state_count(s, i & JOURNAL_STATE_BUF_MASK));
+
+       struct closure *cl = &buf->io;
+       int r = atomic_read(&cl->remaining);
+       prt_printf(out, "io:\t%pS r %i\n", cl->fn, r & CLOSURE_REMAINING_MASK);
 
        if (buf->data) {
                prt_printf(out, "size:\t");
@@ -200,7 +205,8 @@ void bch2_journal_do_writes(struct journal *j)
                if (w->write_started)
                        continue;
 
-               if (!journal_state_count(j->reservations, idx)) {
+               if (!journal_state_seq_count(j, j->reservations, seq)) {
+                       j->seq_write_started = seq;
                        w->write_started = true;
                        closure_call(&w->io, bch2_journal_write, j->wq, NULL);
                }
@@ -396,6 +402,9 @@ static int journal_entry_open(struct journal *j)
        if (nr_unwritten_journal_entries(j) == ARRAY_SIZE(j->buf))
                return JOURNAL_ERR_max_in_flight;
 
+       if (atomic64_read(&j->seq) - j->seq_write_started == JOURNAL_STATE_BUF_NR)
+               return JOURNAL_ERR_max_open;
+
        if (journal_cur_seq(j) >= JOURNAL_SEQ_MAX) {
                bch_err(c, "cannot start: journal seq overflow");
                if (bch2_fs_emergency_read_only_locked(c))
@@ -477,7 +486,7 @@ static int journal_entry_open(struct journal *j)
 
                new.idx++;
                BUG_ON(journal_state_count(new, new.idx));
-               BUG_ON(new.idx != (journal_cur_seq(j) & JOURNAL_BUF_MASK));
+               BUG_ON(new.idx != (journal_cur_seq(j) & JOURNAL_STATE_BUF_MASK));
 
                journal_state_inc(&new);
 
@@ -638,6 +647,23 @@ out:
                count_event(c, journal_entry_full);
        }
 
+       if (ret == JOURNAL_ERR_max_open &&
+           track_event_change(&c->times[BCH_TIME_blocked_journal_max_open], true) &&
+           trace_journal_entry_full_enabled()) {
+               struct printbuf buf = PRINTBUF;
+
+               bch2_printbuf_make_room(&buf, 4096);
+
+               spin_lock(&j->lock);
+               prt_printf(&buf, "seq %llu\n", journal_cur_seq(j));
+               bch2_journal_bufs_to_text(&buf, j);
+               spin_unlock(&j->lock);
+
+               trace_journal_entry_full(c, buf.buf);
+               printbuf_exit(&buf);
+               count_event(c, journal_entry_full);
+       }
+
        /*
         * Journal is full - can't rely on reclaim from work item due to
         * freezing:
@@ -1041,7 +1067,7 @@ static struct journal_buf *__bch2_next_write_buffer_flush_journal_buf(struct jou
                                *blocked = true;
                        }
 
-                       ret = journal_state_count(s, idx) > open
+                       ret = journal_state_count(s, idx & JOURNAL_STATE_BUF_MASK) > open
                                ? ERR_PTR(-EAGAIN)
                                : buf;
                        break;
@@ -1398,6 +1424,7 @@ int bch2_fs_journal_start(struct journal *j, u64 cur_seq)
        j->replay_journal_seq_end = cur_seq;
        j->last_seq_ondisk      = last_seq;
        j->flushed_seq_ondisk   = cur_seq - 1;
+       j->seq_write_started    = cur_seq - 1;
        j->seq_ondisk           = cur_seq - 1;
        j->pin.front            = last_seq;
        j->pin.back             = cur_seq;

diff --git a/fs/bcachefs/journal.h b/fs/bcachefs/journal.h
index e514d664b8aee6dc37f03fe3cf6bf52a69db631c..1c460ded2a11f4db66a5d880f189494d3df30357 100644 (file)
--- a/fs/bcachefs/journal.h
+++ b/fs/bcachefs/journal.h
@@ -121,11 +121,6 @@ static inline void journal_wake(struct journal *j)
        closure_wake_up(&j->async_wait);
 }
 
-static inline struct journal_buf *journal_cur_buf(struct journal *j)
-{
-       return j->buf + j->reservations.idx;
-}
-
 /* Sequence number of oldest dirty journal entry */
 
 static inline u64 journal_last_seq(struct journal *j)
@@ -143,6 +138,15 @@ static inline u64 journal_last_unwritten_seq(struct journal *j)
        return j->seq_ondisk + 1;
 }
 
+static inline struct journal_buf *journal_cur_buf(struct journal *j)
+{
+       unsigned idx = (journal_cur_seq(j) &
+                       JOURNAL_BUF_MASK &
+                       ~JOURNAL_STATE_BUF_MASK) + j->reservations.idx;
+
+       return j->buf + idx;
+}
+
 static inline int journal_state_count(union journal_res_state s, int idx)
 {
        switch (idx) {
@@ -154,6 +158,15 @@ static inline int journal_state_count(union journal_res_state s, int idx)
        BUG();
 }
 
+static inline int journal_state_seq_count(struct journal *j,
+                                         union journal_res_state s, u64 seq)
+{
+       if (journal_cur_seq(j) - seq <= JOURNAL_STATE_BUF_NR)
+               return journal_state_count(s, seq & JOURNAL_STATE_BUF_MASK);
+       else
+               return 0;
+}
+
 static inline void journal_state_inc(union journal_res_state *s)
 {
        s->buf0_count += s->idx == 0;
@@ -269,7 +282,7 @@ void bch2_journal_buf_put_final(struct journal *, u64);
 
 static inline void __bch2_journal_buf_put(struct journal *j, u64 seq)
 {
-       unsigned idx = seq & JOURNAL_BUF_MASK;
+       unsigned idx = seq & JOURNAL_STATE_BUF_MASK;
        union journal_res_state s;
 
        s = journal_state_buf_put(j, idx);
@@ -279,7 +292,7 @@ static inline void __bch2_journal_buf_put(struct journal *j, u64 seq)
 
 static inline void bch2_journal_buf_put(struct journal *j, u64 seq)
 {
-       unsigned idx = seq & JOURNAL_BUF_MASK;
+       unsigned idx = seq & JOURNAL_STATE_BUF_MASK;
        union journal_res_state s;
 
        s = journal_state_buf_put(j, idx);
@@ -365,9 +378,7 @@ static inline int journal_res_get_fast(struct journal *j,
        res->ref        = true;
        res->offset     = old.cur_entry_offset;
        res->seq        = journal_cur_seq(j);
-       res->seq -= (res->seq - old.idx) & JOURNAL_BUF_MASK;
-
-       EBUG_ON(res->seq != le64_to_cpu(j->buf[old.idx].data->seq));
+       res->seq -= (res->seq - old.idx) & JOURNAL_STATE_BUF_MASK;
        return 1;
 }
 
@@ -394,6 +405,7 @@ out:
                                    (flags & JOURNAL_RES_GET_NONBLOCK) != 0,
                                    NULL, _THIS_IP_);
                EBUG_ON(!res->ref);
+               BUG_ON(!res->seq);
        }
        return 0;
 }

diff --git a/fs/bcachefs/journal_types.h b/fs/bcachefs/journal_types.h
index ee9cb17c3ccff5cb6442d7223840531a888cdd31..a0b17c6ed83e56e13b7305dc4806f0eedec2e022 100644 (file)
--- a/fs/bcachefs/journal_types.h
+++ b/fs/bcachefs/journal_types.h
@@ -12,7 +12,11 @@
 /* btree write buffer steals 8 bits for its own purposes: */
 #define JOURNAL_SEQ_MAX                ((1ULL << 56) - 1)
 
-#define JOURNAL_BUF_BITS       2
+#define JOURNAL_STATE_BUF_BITS 2
+#define JOURNAL_STATE_BUF_NR   (1U << JOURNAL_STATE_BUF_BITS)
+#define JOURNAL_STATE_BUF_MASK (JOURNAL_STATE_BUF_NR - 1)
+
+#define JOURNAL_BUF_BITS       4
 #define JOURNAL_BUF_NR         (1U << JOURNAL_BUF_BITS)
 #define JOURNAL_BUF_MASK       (JOURNAL_BUF_NR - 1)
 
@@ -153,6 +157,7 @@ enum journal_flags {
        x(retry)                        \
        x(blocked)                      \
        x(max_in_flight)                \
+       x(max_open)                     \
        x(journal_full)                 \
        x(journal_pin_full)             \
        x(journal_stuck)                \
@@ -238,6 +243,7 @@ struct journal {
        /* Sequence number of most recent journal entry (last entry in @pin) */
        atomic64_t              seq;
 
+       u64                     seq_write_started;
        /* seq, last_seq from the most recent journal entry successfully written */
        u64                     seq_ondisk;
        u64                     flushed_seq_ondisk;