git.ipfire.org Git - thirdparty/kernel/stable-queue.git/blob

   1 From 137b1125848af6a44ce33ca52c9c684998b3d99b Mon Sep 17 00:00:00 2001
   2 From: Sasha Levin <sashal@kernel.org>
   3 Date: Mon, 12 Oct 2020 09:39:06 -0400
   4 Subject: ceph: check session state after bumping session->s_seq
   5
   6 From: Jeff Layton <jlayton@kernel.org>
   7
   8 [ Upstream commit 62575e270f661aba64778cbc5f354511cf9abb21 ]
   9
  10 Some messages sent by the MDS entail a session sequence number
  11 increment, and the MDS will drop certain types of requests on the floor
  12 when the sequence numbers don't match.
  13
  14 In particular, a REQUEST_CLOSE message can cross with one of the
  15 sequence morphing messages from the MDS which can cause the client to
  16 stall, waiting for a response that will never come.
  17
  18 Originally, this meant an up to 5s delay before the recurring workqueue
  19 job kicked in and resent the request, but a recent change made it so
  20 that the client would never resend, causing a 60s stall unmounting and
  21 sometimes a blockisting event.
  22
  23 Add a new helper for incrementing the session sequence and then testing
  24 to see whether a REQUEST_CLOSE needs to be resent, and move the handling
  25 of CEPH_MDS_SESSION_CLOSING into that function. Change all of the
  26 bare sequence counter increments to use the new helper.
  27
  28 Reorganize check_session_state with a switch statement.  It should no
  29 longer be called when the session is CLOSING, so throw a warning if it
  30 ever is (but still handle that case sanely).
  31
  32 [ idryomov: whitespace, pr_err() call fixup ]
  33
  34 URL: https://tracker.ceph.com/issues/47563
  35 Fixes: fa9967734227 ("ceph: fix potential mdsc use-after-free crash")
  36 Reported-by: Patrick Donnelly <pdonnell@redhat.com>
  37 Signed-off-by: Jeff Layton <jlayton@kernel.org>
  38 Reviewed-by: Ilya Dryomov <idryomov@gmail.com>
  39 Reviewed-by: Xiubo Li <xiubli@redhat.com>
  40 Signed-off-by: Ilya Dryomov <idryomov@gmail.com>
  41 Signed-off-by: Sasha Levin <sashal@kernel.org>
  42 ---
  43  fs/ceph/caps.c       |  2 +-
  44  fs/ceph/mds_client.c | 50 +++++++++++++++++++++++++++++++-------------
  45  fs/ceph/mds_client.h |  1 +
  46  fs/ceph/quota.c      |  2 +-
  47  fs/ceph/snap.c       |  2 +-
  48  5 files changed, 39 insertions(+), 18 deletions(-)
  49
  50 diff --git a/fs/ceph/caps.c b/fs/ceph/caps.c
  51 index 034b3f4fdd3a7..64a64a29f5c79 100644
  52 --- a/fs/ceph/caps.c
  53 +++ b/fs/ceph/caps.c
  54 @@ -4064,7 +4064,7 @@ void ceph_handle_caps(struct ceph_mds_session *session,
  55              vino.snap, inode);
  56
  57         mutex_lock(&session->s_mutex);
  58 -       session->s_seq++;
  59 +       inc_session_sequence(session);
  60         dout(" mds%d seq %lld cap seq %u\n", session->s_mds, session->s_seq,
  61              (unsigned)seq);
  62
  63 diff --git a/fs/ceph/mds_client.c b/fs/ceph/mds_client.c
  64 index 76d8d9495d1d4..b2214679baf4e 100644
  65 --- a/fs/ceph/mds_client.c
  66 +++ b/fs/ceph/mds_client.c
  67 @@ -4227,7 +4227,7 @@ static void handle_lease(struct ceph_mds_client *mdsc,
  68              dname.len, dname.name);
  69
  70         mutex_lock(&session->s_mutex);
  71 -       session->s_seq++;
  72 +       inc_session_sequence(session);
  73
  74         if (!inode) {
  75                 dout("handle_lease no inode %llx\n", vino.ino);
  76 @@ -4381,28 +4381,48 @@ static void maybe_recover_session(struct ceph_mds_client *mdsc)
  77
  78  bool check_session_state(struct ceph_mds_session *s)
  79  {
  80 -       if (s->s_state == CEPH_MDS_SESSION_CLOSING) {
  81 -               dout("resending session close request for mds%d\n",
  82 -                               s->s_mds);
  83 -               request_close_session(s);
  84 -               return false;
  85 -       }
  86 -       if (s->s_ttl && time_after(jiffies, s->s_ttl)) {
  87 -               if (s->s_state == CEPH_MDS_SESSION_OPEN) {
  88 +       switch (s->s_state) {
  89 +       case CEPH_MDS_SESSION_OPEN:
  90 +               if (s->s_ttl && time_after(jiffies, s->s_ttl)) {
  91                         s->s_state = CEPH_MDS_SESSION_HUNG;
  92                         pr_info("mds%d hung\n", s->s_mds);
  93                 }
  94 -       }
  95 -       if (s->s_state == CEPH_MDS_SESSION_NEW ||
  96 -           s->s_state == CEPH_MDS_SESSION_RESTARTING ||
  97 -           s->s_state == CEPH_MDS_SESSION_CLOSED ||
  98 -           s->s_state == CEPH_MDS_SESSION_REJECTED)
  99 -               /* this mds is failed or recovering, just wait */
 100 +               break;
 101 +       case CEPH_MDS_SESSION_CLOSING:
 102 +               /* Should never reach this when we're unmounting */
 103 +               WARN_ON_ONCE(true);
 104 +               fallthrough;
 105 +       case CEPH_MDS_SESSION_NEW:
 106 +       case CEPH_MDS_SESSION_RESTARTING:
 107 +       case CEPH_MDS_SESSION_CLOSED:
 108 +       case CEPH_MDS_SESSION_REJECTED:
 109                 return false;
 110 +       }
 111
 112         return true;
 113  }
 114
 115 +/*
 116 + * If the sequence is incremented while we're waiting on a REQUEST_CLOSE reply,
 117 + * then we need to retransmit that request.
 118 + */
 119 +void inc_session_sequence(struct ceph_mds_session *s)
 120 +{
 121 +       lockdep_assert_held(&s->s_mutex);
 122 +
 123 +       s->s_seq++;
 124 +
 125 +       if (s->s_state == CEPH_MDS_SESSION_CLOSING) {
 126 +               int ret;
 127 +
 128 +               dout("resending session close request for mds%d\n", s->s_mds);
 129 +               ret = request_close_session(s);
 130 +               if (ret < 0)
 131 +                       pr_err("unable to close session to mds%d: %d\n",
 132 +                              s->s_mds, ret);
 133 +       }
 134 +}
 135 +
 136  /*
 137   * delayed work -- periodically trim expired leases, renew caps with mds
 138   */
 139 diff --git a/fs/ceph/mds_client.h b/fs/ceph/mds_client.h
 140 index 658800605bfb4..11f20a4d36bc5 100644
 141 --- a/fs/ceph/mds_client.h
 142 +++ b/fs/ceph/mds_client.h
 143 @@ -480,6 +480,7 @@ struct ceph_mds_client {
 144  extern const char *ceph_mds_op_name(int op);
 145
 146  extern bool check_session_state(struct ceph_mds_session *s);
 147 +void inc_session_sequence(struct ceph_mds_session *s);
 148
 149  extern struct ceph_mds_session *
 150  __ceph_lookup_mds_session(struct ceph_mds_client *, int mds);
 151 diff --git a/fs/ceph/quota.c b/fs/ceph/quota.c
 152 index cc2c4d40b0222..2b213f864c564 100644
 153 --- a/fs/ceph/quota.c
 154 +++ b/fs/ceph/quota.c
 155 @@ -53,7 +53,7 @@ void ceph_handle_quota(struct ceph_mds_client *mdsc,
 156
 157         /* increment msg sequence number */
 158         mutex_lock(&session->s_mutex);
 159 -       session->s_seq++;
 160 +       inc_session_sequence(session);
 161         mutex_unlock(&session->s_mutex);
 162
 163         /* lookup inode */
 164 diff --git a/fs/ceph/snap.c b/fs/ceph/snap.c
 165 index 923be9399b21c..cc9a9bfc790a3 100644
 166 --- a/fs/ceph/snap.c
 167 +++ b/fs/ceph/snap.c
 168 @@ -873,7 +873,7 @@ void ceph_handle_snap(struct ceph_mds_client *mdsc,
 169              ceph_snap_op_name(op), split, trace_len);
 170
 171         mutex_lock(&session->s_mutex);
 172 -       session->s_seq++;
 173 +       inc_session_sequence(session);
 174         mutex_unlock(&session->s_mutex);
 175
 176         down_write(&mdsc->snap_rwsem);
 177 --
 178 2.27.0
 179