src/patches/suse-2.6.27.31/patches.suse/dlm-replace-idr-with-hash-table-for-connections.patch

   1 From: Christine Caulfield <ccaulfie@redhat.com>
   2 commit 5e9ccc372dc855900c4a75b21286038938e288c7
   3 Author: Christine Caulfield <ccaulfie@redhat.com>
   4 Date:   Wed Jan 28 12:57:40 2009 -0600
   5 Subject: dlm: replace idr with hash table for connections
   6
   7     Integer nodeids can be too large for the idr code; use a hash
   8     table instead.
   9
  10 Signed-off-by: Christine Caulfield <ccaulfie@redhat.com>
  11 Signed-off-by: David Teigland <teigland@redhat.com>
  12 Signed-off-by: Coly Li <coly.li@suse.de>
  13
  14 diff --git a/fs/dlm/lowcomms.c b/fs/dlm/lowcomms.c
  15 index 982314c..609108a 100644
  16 --- a/fs/dlm/lowcomms.c
  17 +++ b/fs/dlm/lowcomms.c
  18 @@ -2,7 +2,7 @@
  19  *******************************************************************************
  20  **
  21  **  Copyright (C) Sistina Software, Inc.  1997-2003  All rights reserved.
  22 -**  Copyright (C) 2004-2007 Red Hat, Inc.  All rights reserved.
  23 +**  Copyright (C) 2004-2009 Red Hat, Inc.  All rights reserved.
  24  **
  25  **  This copyrighted material is made available to anyone wishing to use,
  26  **  modify, copy, or redistribute it subject to the terms and conditions
  27 @@ -48,7 +48,6 @@
  28  #include <net/sock.h>
  29  #include <net/tcp.h>
  30  #include <linux/pagemap.h>
  31 -#include <linux/idr.h>
  32  #include <linux/file.h>
  33  #include <linux/mutex.h>
  34  #include <linux/sctp.h>
  35 @@ -61,6 +60,7 @@
  36  #include "config.h"
  37
  38  #define NEEDED_RMEM (4*1024*1024)
  39 +#define CONN_HASH_SIZE 32
  40
  41  struct cbuf {
  42         unsigned int base;
  43 @@ -115,6 +115,7 @@ struct connection {
  44         int retries;
  45  #define MAX_CONNECT_RETRIES 3
  46         int sctp_assoc;
  47 +       struct hlist_node list;
  48         struct connection *othercon;
  49         struct work_struct rwork; /* Receive workqueue */
  50         struct work_struct swork; /* Send workqueue */
  51 @@ -139,14 +140,37 @@ static int dlm_local_count;
  52  static struct workqueue_struct *recv_workqueue;
  53  static struct workqueue_struct *send_workqueue;
  54
  55 -static DEFINE_IDR(connections_idr);
  56 +static struct hlist_head connection_hash[CONN_HASH_SIZE];
  57  static DEFINE_MUTEX(connections_lock);
  58 -static int max_nodeid;
  59  static struct kmem_cache *con_cache;
  60
  61  static void process_recv_sockets(struct work_struct *work);
  62  static void process_send_sockets(struct work_struct *work);
  63
  64 +
  65 +/* This is deliberately very simple because most clusters have simple
  66 +   sequential nodeids, so we should be able to go straight to a connection
  67 +   struct in the array */
  68 +static inline int nodeid_hash(int nodeid)
  69 +{
  70 +       return nodeid & (CONN_HASH_SIZE-1);
  71 +}
  72 +
  73 +static struct connection *__find_con(int nodeid)
  74 +{
  75 +       int r;
  76 +       struct hlist_node *h;
  77 +       struct connection *con;
  78 +
  79 +       r = nodeid_hash(nodeid);
  80 +
  81 +       hlist_for_each_entry(con, h, &connection_hash[r], list) {
  82 +               if (con->nodeid == nodeid)
  83 +                       return con;
  84 +       }
  85 +       return NULL;
  86 +}
  87 +
  88  /*
  89   * If 'allocation' is zero then we don't attempt to create a new
  90   * connection structure for this node.
  91 @@ -155,31 +179,17 @@ static struct connection *__nodeid2con(int nodeid, gfp_t alloc)
  92  {
  93         struct connection *con = NULL;
  94         int r;
  95 -       int n;
  96
  97 -       con = idr_find(&connections_idr, nodeid);
  98 +       con = __find_con(nodeid);
  99         if (con || !alloc)
 100                 return con;
 101
 102 -       r = idr_pre_get(&connections_idr, alloc);
 103 -       if (!r)
 104 -               return NULL;
 105 -
 106         con = kmem_cache_zalloc(con_cache, alloc);
 107         if (!con)
 108                 return NULL;
 109
 110 -       r = idr_get_new_above(&connections_idr, con, nodeid, &n);
 111 -       if (r) {
 112 -               kmem_cache_free(con_cache, con);
 113 -               return NULL;
 114 -       }
 115 -
 116 -       if (n != nodeid) {
 117 -               idr_remove(&connections_idr, n);
 118 -               kmem_cache_free(con_cache, con);
 119 -               return NULL;
 120 -       }
 121 +       r = nodeid_hash(nodeid);
 122 +       hlist_add_head(&con->list, &connection_hash[r]);
 123
 124         con->nodeid = nodeid;
 125         mutex_init(&con->sock_mutex);
 126 @@ -190,19 +200,30 @@ static struct connection *__nodeid2con(int nodeid, gfp_t alloc)
 127
 128         /* Setup action pointers for child sockets */
 129         if (con->nodeid) {
 130 -               struct connection *zerocon = idr_find(&connections_idr, 0);
 131 +               struct connection *zerocon = __find_con(0);
 132
 133                 con->connect_action = zerocon->connect_action;
 134                 if (!con->rx_action)
 135                         con->rx_action = zerocon->rx_action;
 136         }
 137
 138 -       if (nodeid > max_nodeid)
 139 -               max_nodeid = nodeid;
 140 -
 141         return con;
 142  }
 143
 144 +/* Loop round all connections */
 145 +static void foreach_conn(void (*conn_func)(struct connection *c))
 146 +{
 147 +       int i;
 148 +       struct hlist_node *h, *n;
 149 +       struct connection *con;
 150 +
 151 +       for (i = 0; i < CONN_HASH_SIZE; i++) {
 152 +               hlist_for_each_entry_safe(con, h, n, &connection_hash[i], list){
 153 +                       conn_func(con);
 154 +               }
 155 +       }
 156 +}
 157 +
 158  static struct connection *nodeid2con(int nodeid, gfp_t allocation)
 159  {
 160         struct connection *con;
 161 @@ -218,14 +239,17 @@ static struct connection *nodeid2con(int nodeid, gfp_t allocation)
 162  static struct connection *assoc2con(int assoc_id)
 163  {
 164         int i;
 165 +       struct hlist_node *h;
 166         struct connection *con;
 167
 168         mutex_lock(&connections_lock);
 169 -       for (i=0; i<=max_nodeid; i++) {
 170 -               con = __nodeid2con(i, 0);
 171 -               if (con && con->sctp_assoc == assoc_id) {
 172 -                       mutex_unlock(&connections_lock);
 173 -                       return con;
 174 +
 175 +       for (i = 0 ; i < CONN_HASH_SIZE; i++) {
 176 +               hlist_for_each_entry(con, h, &connection_hash[i], list) {
 177 +                       if (con && con->sctp_assoc == assoc_id) {
 178 +                               mutex_unlock(&connections_lock);
 179 +                               return con;
 180 +                       }
 181                 }
 182         }
 183         mutex_unlock(&connections_lock);
 184 @@ -376,25 +400,23 @@ static void sctp_send_shutdown(sctp_assoc_t associd)
 185                 log_print("send EOF to node failed: %d", ret);
 186  }
 187
 188 +static void sctp_init_failed_foreach(struct connection *con)
 189 +{
 190 +       con->sctp_assoc = 0;
 191 +       if (test_and_clear_bit(CF_CONNECT_PENDING, &con->flags)) {
 192 +               if (!test_and_set_bit(CF_WRITE_PENDING, &con->flags))
 193 +                       queue_work(send_workqueue, &con->swork);
 194 +       }
 195 +}
 196 +
 197  /* INIT failed but we don't know which node...
 198     restart INIT on all pending nodes */
 199  static void sctp_init_failed(void)
 200  {
 201 -       int i;
 202 -       struct connection *con;
 203 -
 204         mutex_lock(&connections_lock);
 205 -       for (i=1; i<=max_nodeid; i++) {
 206 -               con = __nodeid2con(i, 0);
 207 -               if (!con)
 208 -                       continue;
 209 -               con->sctp_assoc = 0;
 210 -               if (test_and_clear_bit(CF_CONNECT_PENDING, &con->flags)) {
 211 -                       if (!test_and_set_bit(CF_WRITE_PENDING, &con->flags)) {
 212 -                               queue_work(send_workqueue, &con->swork);
 213 -                       }
 214 -               }
 215 -       }
 216 +
 217 +       foreach_conn(sctp_init_failed_foreach);
 218 +
 219         mutex_unlock(&connections_lock);
 220  }
 221
 222 @@ -1313,13 +1335,10 @@ out_connect:
 223
 224  static void clean_one_writequeue(struct connection *con)
 225  {
 226 -       struct list_head *list;
 227 -       struct list_head *temp;
 228 +       struct writequeue_entry *e, *safe;
 229
 230         spin_lock(&con->writequeue_lock);
 231 -       list_for_each_safe(list, temp, &con->writequeue) {
 232 -               struct writequeue_entry *e =
 233 -                       list_entry(list, struct writequeue_entry, list);
 234 +       list_for_each_entry_safe(e, safe, &con->writequeue, list) {
 235                 list_del(&e->list);
 236                 free_entry(e);
 237         }
 238 @@ -1369,14 +1388,7 @@ static void process_send_sockets(struct work_struct *work)
 239  /* Discard all entries on the write queues */
 240  static void clean_writequeues(void)
 241  {
 242 -       int nodeid;
 243 -
 244 -       for (nodeid = 1; nodeid <= max_nodeid; nodeid++) {
 245 -               struct connection *con = __nodeid2con(nodeid, 0);
 246 -
 247 -               if (con)
 248 -                       clean_one_writequeue(con);
 249 -       }
 250 +       foreach_conn(clean_one_writequeue);
 251  }
 252
 253  static void work_stop(void)
 254 @@ -1406,23 +1418,29 @@ static int work_start(void)
 255         return 0;
 256  }
 257
 258 -void dlm_lowcomms_stop(void)
 259 +static void stop_conn(struct connection *con)
 260  {
 261 -       int i;
 262 -       struct connection *con;
 263 +       con->flags |= 0x0F;
 264 +       if (con->sock)
 265 +               con->sock->sk->sk_user_data = NULL;
 266 +}
 267
 268 +static void free_conn(struct connection *con)
 269 +{
 270 +       close_connection(con, true);
 271 +       if (con->othercon)
 272 +               kmem_cache_free(con_cache, con->othercon);
 273 +       hlist_del(&con->list);
 274 +       kmem_cache_free(con_cache, con);
 275 +}
 276 +
 277 +void dlm_lowcomms_stop(void)
 278 +{
 279         /* Set all the flags to prevent any
 280            socket activity.
 281         */
 282         mutex_lock(&connections_lock);
 283 -       for (i = 0; i <= max_nodeid; i++) {
 284 -               con = __nodeid2con(i, 0);
 285 -               if (con) {
 286 -                       con->flags |= 0x0F;
 287 -                       if (con->sock)
 288 -                               con->sock->sk->sk_user_data = NULL;
 289 -               }
 290 -       }
 291 +       foreach_conn(stop_conn);
 292         mutex_unlock(&connections_lock);
 293
 294         work_stop();
 295 @@ -1430,25 +1448,20 @@ void dlm_lowcomms_stop(void)
 296         mutex_lock(&connections_lock);
 297         clean_writequeues();
 298
 299 -       for (i = 0; i <= max_nodeid; i++) {
 300 -               con = __nodeid2con(i, 0);
 301 -               if (con) {
 302 -                       close_connection(con, true);
 303 -                       if (con->othercon)
 304 -                               kmem_cache_free(con_cache, con->othercon);
 305 -                       kmem_cache_free(con_cache, con);
 306 -               }
 307 -       }
 308 -       max_nodeid = 0;
 309 +       foreach_conn(free_conn);
 310 +
 311         mutex_unlock(&connections_lock);
 312         kmem_cache_destroy(con_cache);
 313 -       idr_init(&connections_idr);
 314  }
 315
 316  int dlm_lowcomms_start(void)
 317  {
 318         int error = -EINVAL;
 319         struct connection *con;
 320 +       int i;
 321 +
 322 +       for (i = 0; i < CONN_HASH_SIZE; i++)
 323 +               INIT_HLIST_HEAD(&connection_hash[i]);
 324
 325         init_local();
 326         if (!dlm_local_count) {