+++ /dev/null
-From: "Heinz Mauelshagen <hjm@redhat.de>
-Subject: DMRAID45 module
-X-URL: http://people.redhat.com/~heinzm/sw/dm/dm-raid45/
-
- DM-RAID 45 module.
-
- This driver is used for "Fake RAID" devices.
-
-Acked-by: Jeff Mahoney <jeffm@suse.com>
-
----
-
- drivers/md/Kconfig | 8
- drivers/md/Makefile | 4
- drivers/md/dm-memcache.c | 301 ++
- drivers/md/dm-memcache.h | 68
- drivers/md/dm-message.c | 182 +
- drivers/md/dm-message.h | 91
- drivers/md/dm-raid45.c | 4516 +++++++++++++++++++++++++++++++++++++++++++++
- drivers/md/dm-raid45.h | 28
- drivers/md/dm-regions.c | 723 +++++++
- drivers/md/dm.c | 1
- include/linux/dm-regions.h | 115 +
- 11 files changed, 6036 insertions(+), 1 deletion(-)
-
---- a/drivers/md/dm.c
-+++ b/drivers/md/dm.c
-@@ -1680,6 +1680,7 @@ struct gendisk *dm_disk(struct mapped_de
- {
- return md->disk;
- }
-+EXPORT_SYMBOL_GPL(dm_disk);
-
- int dm_suspended(struct mapped_device *md)
- {
---- /dev/null
-+++ b/drivers/md/dm-memcache.c
-@@ -0,0 +1,301 @@
-+/*
-+ * Copyright (C) 2006-2008 Red Hat, Inc. All rights reserved.
-+ *
-+ * Module Author: Heinz Mauelshagen <heinzm@redhat.com>
-+ *
-+ * Device-mapper memory object handling:
-+ *
-+ * o allocate/free total_pages in a per client page pool.
-+ *
-+ * o allocate/free memory objects with chunks (1..n) of
-+ * pages_per_chunk pages hanging off.
-+ *
-+ * This file is released under the GPL.
-+ */
-+
-+#define DM_MEM_CACHE_VERSION "0.2"
-+
-+#include "dm.h"
-+#include "dm-memcache.h"
-+#include <linux/dm-io.h>
-+
-+struct dm_mem_cache_client {
-+ spinlock_t lock;
-+ mempool_t *objs_pool;
-+ struct page_list *free_list;
-+ unsigned objects;
-+ unsigned chunks;
-+ unsigned pages_per_chunk;
-+ unsigned free_pages;
-+ unsigned total_pages;
-+};
-+
-+/*
-+ * Free pages and page_list elements of client.
-+ */
-+static void free_cache_pages(struct page_list *list)
-+{
-+ while (list) {
-+ struct page_list *pl = list;
-+
-+ list = pl->next;
-+ BUG_ON(!pl->page);
-+ __free_page(pl->page);
-+ kfree(pl);
-+ }
-+}
-+
-+/*
-+ * Alloc number of pages and page_list elements as required by client.
-+ */
-+static struct page_list *alloc_cache_pages(unsigned pages)
-+{
-+ struct page_list *pl, *ret = NULL;
-+ struct page *page;
-+
-+ while (pages--) {
-+ page = alloc_page(GFP_NOIO);
-+ if (!page)
-+ goto err;
-+
-+ pl = kmalloc(sizeof(*pl), GFP_NOIO);
-+ if (!pl) {
-+ __free_page(page);
-+ goto err;
-+ }
-+
-+ pl->page = page;
-+ pl->next = ret;
-+ ret = pl;
-+ }
-+
-+ return ret;
-+
-+err:
-+ free_cache_pages(ret);
-+ return NULL;
-+}
-+
-+/*
-+ * Allocate page_list elements from the pool to chunks of the memory object.
-+ */
-+static void alloc_chunks(struct dm_mem_cache_client *cl,
-+ struct dm_mem_cache_object *obj)
-+{
-+ unsigned chunks = cl->chunks;
-+ unsigned long flags;
-+
-+ local_irq_save(flags);
-+ local_irq_disable();
-+ while (chunks--) {
-+ unsigned p = cl->pages_per_chunk;
-+
-+ obj[chunks].pl = NULL;
-+
-+ while (p--) {
-+ struct page_list *pl;
-+
-+ /* Take next element from free list */
-+ spin_lock(&cl->lock);
-+ pl = cl->free_list;
-+ BUG_ON(!pl);
-+ cl->free_list = pl->next;
-+ spin_unlock(&cl->lock);
-+
-+ pl->next = obj[chunks].pl;
-+ obj[chunks].pl = pl;
-+ }
-+ }
-+
-+ local_irq_restore(flags);
-+}
-+
-+/*
-+ * Free page_list elements putting them back onto free list
-+ */
-+static void free_chunks(struct dm_mem_cache_client *cl,
-+ struct dm_mem_cache_object *obj)
-+{
-+ unsigned chunks = cl->chunks;
-+ unsigned long flags;
-+ struct page_list *next, *pl;
-+
-+ local_irq_save(flags);
-+ local_irq_disable();
-+ while (chunks--) {
-+ for (pl = obj[chunks].pl; pl; pl = next) {
-+ next = pl->next;
-+
-+ spin_lock(&cl->lock);
-+ pl->next = cl->free_list;
-+ cl->free_list = pl;
-+ cl->free_pages++;
-+ spin_unlock(&cl->lock);
-+ }
-+ }
-+
-+ local_irq_restore(flags);
-+}
-+
-+/*
-+ * Create/destroy dm memory cache client resources.
-+ */
-+struct dm_mem_cache_client *
-+dm_mem_cache_client_create(unsigned objects, unsigned chunks,
-+ unsigned pages_per_chunk)
-+{
-+ unsigned total_pages = objects * chunks * pages_per_chunk;
-+ struct dm_mem_cache_client *client;
-+
-+ BUG_ON(!total_pages);
-+ client = kzalloc(sizeof(*client), GFP_KERNEL);
-+ if (!client)
-+ return ERR_PTR(-ENOMEM);
-+
-+ client->objs_pool = mempool_create_kmalloc_pool(objects,
-+ chunks * sizeof(struct dm_mem_cache_object));
-+ if (!client->objs_pool)
-+ goto err;
-+
-+ client->free_list = alloc_cache_pages(total_pages);
-+ if (!client->free_list)
-+ goto err1;
-+
-+ spin_lock_init(&client->lock);
-+ client->objects = objects;
-+ client->chunks = chunks;
-+ client->pages_per_chunk = pages_per_chunk;
-+ client->free_pages = client->total_pages = total_pages;
-+ return client;
-+
-+err1:
-+ mempool_destroy(client->objs_pool);
-+err:
-+ kfree(client);
-+ return ERR_PTR(-ENOMEM);
-+}
-+EXPORT_SYMBOL(dm_mem_cache_client_create);
-+
-+void dm_mem_cache_client_destroy(struct dm_mem_cache_client *cl)
-+{
-+ BUG_ON(cl->free_pages != cl->total_pages);
-+ free_cache_pages(cl->free_list);
-+ mempool_destroy(cl->objs_pool);
-+ kfree(cl);
-+}
-+EXPORT_SYMBOL(dm_mem_cache_client_destroy);
-+
-+/*
-+ * Grow a clients cache by an amount of pages.
-+ *
-+ * Don't call from interrupt context!
-+ */
-+int dm_mem_cache_grow(struct dm_mem_cache_client *cl, unsigned objects)
-+{
-+ unsigned pages = objects * cl->chunks * cl->pages_per_chunk;
-+ struct page_list *pl, *last;
-+
-+ BUG_ON(!pages);
-+ pl = alloc_cache_pages(pages);
-+ if (!pl)
-+ return -ENOMEM;
-+
-+ last = pl;
-+ while (last->next)
-+ last = last->next;
-+
-+ spin_lock_irq(&cl->lock);
-+ last->next = cl->free_list;
-+ cl->free_list = pl;
-+ cl->free_pages += pages;
-+ cl->total_pages += pages;
-+ cl->objects++;
-+ spin_unlock_irq(&cl->lock);
-+
-+ mempool_resize(cl->objs_pool, cl->objects, GFP_NOIO);
-+ return 0;
-+}
-+EXPORT_SYMBOL(dm_mem_cache_grow);
-+
-+/* Shrink a clients cache by an amount of pages */
-+int dm_mem_cache_shrink(struct dm_mem_cache_client *cl, unsigned objects)
-+{
-+ int r;
-+ unsigned pages = objects * cl->chunks * cl->pages_per_chunk, p = pages;
-+ unsigned long flags;
-+ struct page_list *last = NULL, *pl, *pos;
-+
-+ BUG_ON(!pages);
-+
-+ spin_lock_irqsave(&cl->lock, flags);
-+ pl = pos = cl->free_list;
-+ while (p-- && pos->next) {
-+ last = pos;
-+ pos = pos->next;
-+ }
-+
-+ if (++p)
-+ r = -ENOMEM;
-+ else {
-+ r = 0;
-+ cl->free_list = pos;
-+ cl->free_pages -= pages;
-+ cl->total_pages -= pages;
-+ cl->objects--;
-+ last->next = NULL;
-+ }
-+ spin_unlock_irqrestore(&cl->lock, flags);
-+
-+ if (!r) {
-+ free_cache_pages(pl);
-+ mempool_resize(cl->objs_pool, cl->objects, GFP_NOIO);
-+ }
-+
-+ return r;
-+}
-+EXPORT_SYMBOL(dm_mem_cache_shrink);
-+
-+/*
-+ * Allocate/free a memory object
-+ *
-+ * Can be called from interrupt context
-+ */
-+struct dm_mem_cache_object *dm_mem_cache_alloc(struct dm_mem_cache_client *cl)
-+{
-+ int r = 0;
-+ unsigned pages = cl->chunks * cl->pages_per_chunk;
-+ unsigned long flags;
-+ struct dm_mem_cache_object *obj;
-+
-+ obj = mempool_alloc(cl->objs_pool, GFP_NOIO);
-+ if (!obj)
-+ return ERR_PTR(-ENOMEM);
-+
-+ spin_lock_irqsave(&cl->lock, flags);
-+ if (pages > cl->free_pages)
-+ r = -ENOMEM;
-+ else
-+ cl->free_pages -= pages;
-+ spin_unlock_irqrestore(&cl->lock, flags);
-+
-+ if (r) {
-+ mempool_free(obj, cl->objs_pool);
-+ return ERR_PTR(r);
-+ }
-+
-+ alloc_chunks(cl, obj);
-+ return obj;
-+}
-+EXPORT_SYMBOL(dm_mem_cache_alloc);
-+
-+void dm_mem_cache_free(struct dm_mem_cache_client *cl,
-+ struct dm_mem_cache_object *obj)
-+{
-+ free_chunks(cl, obj);
-+ mempool_free(obj, cl->objs_pool);
-+}
-+EXPORT_SYMBOL(dm_mem_cache_free);
-+
-+MODULE_DESCRIPTION(DM_NAME " dm memory cache");
-+MODULE_AUTHOR("Heinz Mauelshagen <hjm@redhat.com>");
-+MODULE_LICENSE("GPL");
---- /dev/null
-+++ b/drivers/md/dm-memcache.h
-@@ -0,0 +1,68 @@
-+/*
-+ * Copyright (C) 2006-2008 Red Hat, Inc. All rights reserved.
-+ *
-+ * Module Author: Heinz Mauelshagen <Mauelshagen@RedHat.com>
-+ *
-+ * Device-mapper memory object handling:
-+ *
-+ * o allocate/free total_pages in a per client page pool.
-+ *
-+ * o allocate/free memory objects with chunks (1..n) of
-+ * pages_per_chunk pages hanging off.
-+ *
-+ * This file is released under the GPL.
-+ */
-+
-+#ifndef _DM_MEM_CACHE_H
-+#define _DM_MEM_CACHE_H
-+
-+#define DM_MEM_CACHE_H_VERSION "0.1"
-+
-+#include "dm.h"
-+#include <linux/dm-io.h>
-+
-+static inline struct page_list *pl_elem(struct page_list *pl, unsigned p)
-+{
-+ while (pl && p--)
-+ pl = pl->next;
-+
-+ return pl;
-+}
-+
-+struct dm_mem_cache_object {
-+ struct page_list *pl; /* Dynamically allocated array */
-+ void *private; /* Caller context reference */
-+};
-+
-+struct dm_mem_cache_client;
-+
-+/*
-+ * Create/destroy dm memory cache client resources.
-+ *
-+ * On creation, a number of @objects with @chunks of
-+ * @pages_per_chunk pages will be allocated.
-+ */
-+struct dm_mem_cache_client *
-+dm_mem_cache_client_create(unsigned objects, unsigned chunks,
-+ unsigned pages_per_chunk);
-+void dm_mem_cache_client_destroy(struct dm_mem_cache_client *client);
-+
-+/*
-+ * Grow/shrink a dm memory cache client resources
-+ * by @objetcs amount of objects.
-+ */
-+int dm_mem_cache_grow(struct dm_mem_cache_client *client, unsigned objects);
-+int dm_mem_cache_shrink(struct dm_mem_cache_client *client, unsigned objects);
-+
-+/*
-+ * Allocate/free a memory object
-+ *
-+ * On allocation one object with an amount of chunks and
-+ * an amount of pages per chunk will be returned on success.
-+ */
-+struct dm_mem_cache_object *
-+dm_mem_cache_alloc(struct dm_mem_cache_client *client);
-+void dm_mem_cache_free(struct dm_mem_cache_client *client,
-+ struct dm_mem_cache_object *object);
-+
-+#endif
---- /dev/null
-+++ b/drivers/md/dm-message.c
-@@ -0,0 +1,182 @@
-+/*
-+ * Copyright (C) 2007,2008 Red Hat Inc. All rights reserved.
-+ *
-+ * Module Author: Heinz Mauelshagen <heinzm@redhat.com>
-+ *
-+ * General device-mapper message interface argument parser.
-+ *
-+ * This file is released under the GPL.
-+ *
-+ * device-mapper message parser.
-+ *
-+ */
-+
-+#include "dm.h"
-+#include "dm-message.h"
-+#include <linux/kernel.h>
-+
-+#define DM_MSG_PREFIX "dm_message"
-+
-+/* Basename of a path. */
-+static inline char *
-+basename(char *s)
-+{
-+ char *p = strrchr(s, '/');
-+
-+ return p ? p + 1 : s;
-+}
-+
-+/* Get an argument depending on type. */
-+static void
-+message_arguments(struct dm_msg *msg, int argc, char **argv)
-+{
-+
-+ if (argc) {
-+ int i;
-+ struct dm_message_argument *args = msg->spec->args;
-+
-+ for (i = 0; i < args->num_args; i++) {
-+ int r;
-+ unsigned long **ptr = args->ptr;
-+ enum dm_message_argument_type type = args->types[i];
-+
-+ switch (type) {
-+ case dm_msg_base_t:
-+ ((char **) ptr)[i] = basename(argv[i]);
-+ break;
-+
-+ case dm_msg_str_t:
-+ ((char **) ptr)[i] = argv[i];
-+ break;
-+
-+ case dm_msg_int_t:
-+ r = sscanf(argv[i], "%d", ((int **) ptr)[i]);
-+ goto check;
-+
-+ case dm_msg_uint_t:
-+ r = sscanf(argv[i], "%u",
-+ ((unsigned **) ptr)[i]);
-+ goto check;
-+
-+ case dm_msg_uint64_t:
-+ r = sscanf(argv[i], "%llu",
-+ ((unsigned long long **) ptr)[i]);
-+
-+check:
-+ if (r != 1) {
-+ set_bit(dm_msg_ret_undef, &msg->ret);
-+ set_bit(dm_msg_ret_arg, &msg->ret);
-+ }
-+ }
-+ }
-+ }
-+}
-+
-+/* Parse message options. */
-+static void
-+message_options_parse(struct dm_msg *msg, int argc, char **argv)
-+{
-+ int hit = 0;
-+ unsigned long *action;
-+ size_t l1 = strlen(*argv), l_hit = 0;
-+ struct dm_message_option *o = msg->spec->options;
-+ char **option, **option_end = o->options + o->num_options;
-+
-+ for (option = o->options, action = o->actions;
-+ option < option_end; option++, action++) {
-+ size_t l2 = strlen(*option);
-+
-+ if (!strnicmp(*argv, *option, min(l1, l2))) {
-+ hit++;
-+ l_hit = l2;
-+ set_bit(*action, &msg->action);
-+ }
-+ }
-+
-+ /* Assume error. */
-+ msg->ret = 0;
-+ set_bit(dm_msg_ret_option, &msg->ret);
-+ if (!hit || l1 > l_hit)
-+ set_bit(dm_msg_ret_undef, &msg->ret); /* Undefined option. */
-+ else if (hit > 1)
-+ set_bit(dm_msg_ret_ambiguous, &msg->ret); /* Ambiguous option.*/
-+ else {
-+ clear_bit(dm_msg_ret_option, &msg->ret); /* Option OK. */
-+ message_arguments(msg, --argc, ++argv);
-+ }
-+}
-+
-+static inline void
-+print_ret(const char *caller, unsigned long ret)
-+{
-+ struct {
-+ unsigned long err;
-+ const char *err_str;
-+ } static err_msg[] = {
-+ { dm_msg_ret_ambiguous, "message ambiguous" },
-+ { dm_msg_ret_inval, "message invalid" },
-+ { dm_msg_ret_undef, "message undefined" },
-+ { dm_msg_ret_arg, "message argument" },
-+ { dm_msg_ret_argcount, "message argument count" },
-+ { dm_msg_ret_option, "option" },
-+ }, *e = ARRAY_END(err_msg);
-+
-+ while (e-- > err_msg) {
-+ if (test_bit(e->err, &ret))
-+ DMERR("%s %s", caller, e->err_str);
-+ }
-+}
-+
-+/* Parse a message action. */
-+int
-+dm_message_parse(const char *caller, struct dm_msg *msg, void *context,
-+ int argc, char **argv)
-+{
-+ int hit = 0;
-+ size_t l1 = strlen(*argv), l_hit = 0;
-+ struct dm_msg_spec *s, *s_hit = NULL,
-+ *s_end = msg->specs + msg->num_specs;
-+
-+ if (argc < 2)
-+ return -EINVAL;
-+
-+ for (s = msg->specs; s < s_end; s++) {
-+ size_t l2 = strlen(s->cmd);
-+
-+ if (!strnicmp(*argv, s->cmd, min(l1, l2))) {
-+ hit++;
-+ l_hit = l2;
-+ s_hit = s;
-+ }
-+ }
-+
-+ msg->ret = 0;
-+ if (!hit || l1 > l_hit) /* No hit or message string too long. */
-+ set_bit(dm_msg_ret_undef, &msg->ret);
-+ else if (hit > 1) /* Ambiguous message. */
-+ set_bit(dm_msg_ret_ambiguous, &msg->ret);
-+ else if (argc - 2 != s_hit->args->num_args) {
-+ set_bit(dm_msg_ret_undef, &msg->ret);
-+ set_bit(dm_msg_ret_argcount, &msg->ret);
-+ }
-+
-+ if (msg->ret)
-+ goto bad;
-+
-+ msg->action = 0;
-+ msg->spec = s_hit;
-+ set_bit(s_hit->action, &msg->action);
-+ message_options_parse(msg, --argc, ++argv);
-+
-+ if (!msg->ret)
-+ return msg->spec->f(msg, context);
-+
-+bad:
-+ print_ret(caller, msg->ret);
-+ return -EINVAL;
-+}
-+EXPORT_SYMBOL(dm_message_parse);
-+
-+MODULE_DESCRIPTION(DM_NAME " device-mapper target message parser");
-+MODULE_AUTHOR("Heinz Mauelshagen <hjm@redhat.com>");
-+MODULE_LICENSE("GPL");
---- /dev/null
-+++ b/drivers/md/dm-message.h
-@@ -0,0 +1,91 @@
-+/*
-+ * Copyright (C) 2007,2008 Red Hat, Inc. All rights reserved.
-+ *
-+ * Module Author: Heinz Mauelshagen <Mauelshagen@RedHat.de>
-+ *
-+ * General device-mapper message interface argument parser.
-+ *
-+ * This file is released under the GPL.
-+ *
-+ */
-+
-+#ifndef DM_MESSAGE_H
-+#define DM_MESSAGE_H
-+
-+/* Factor out to dm.h. */
-+/* Reference to array end. */
-+#define ARRAY_END(a) ((a) + ARRAY_SIZE(a))
-+
-+/* Message return bits. */
-+enum dm_message_return {
-+ dm_msg_ret_ambiguous, /* Action ambiguous. */
-+ dm_msg_ret_inval, /* Action invalid. */
-+ dm_msg_ret_undef, /* Action undefined. */
-+
-+ dm_msg_ret_option, /* Option error. */
-+ dm_msg_ret_arg, /* Argument error. */
-+ dm_msg_ret_argcount, /* Argument count error. */
-+};
-+
-+/* Message argument type conversions. */
-+enum dm_message_argument_type {
-+ dm_msg_base_t, /* Basename string. */
-+ dm_msg_str_t, /* String. */
-+ dm_msg_int_t, /* Signed int. */
-+ dm_msg_uint_t, /* Unsigned int. */
-+ dm_msg_uint64_t, /* Unsigned int 64. */
-+};
-+
-+/* A message option. */
-+struct dm_message_option {
-+ unsigned num_options;
-+ char **options;
-+ unsigned long *actions;
-+};
-+
-+/* Message arguments and types. */
-+struct dm_message_argument {
-+ unsigned num_args;
-+ unsigned long **ptr;
-+ enum dm_message_argument_type types[];
-+};
-+
-+/* Client message. */
-+struct dm_msg {
-+ unsigned long action; /* Identified action. */
-+ unsigned long ret; /* Return bits. */
-+ unsigned num_specs; /* # of sepcifications listed. */
-+ struct dm_msg_spec *specs; /* Specification list. */
-+ struct dm_msg_spec *spec; /* Specification selected. */
-+};
-+
-+/* Secification of the message. */
-+struct dm_msg_spec {
-+ const char *cmd; /* Name of the command (i.e. 'bandwidth'). */
-+ unsigned long action;
-+ struct dm_message_option *options;
-+ struct dm_message_argument *args;
-+ unsigned long parm; /* Parameter to pass through to callback. */
-+ /* Function to process for action. */
-+ int (*f) (struct dm_msg *msg, void *context);
-+};
-+
-+/* Parameter access macros. */
-+#define DM_MSG_PARM(msg) ((msg)->spec->parm)
-+
-+#define DM_MSG_STR_ARGS(msg, idx) ((char *) *(msg)->spec->args->ptr[idx])
-+#define DM_MSG_INT_ARGS(msg, idx) ((int) *(msg)->spec->args->ptr[idx])
-+#define DM_MSG_UINT_ARGS(msg, idx) ((unsigned) DM_MSG_INT_ARG(msg, idx))
-+#define DM_MSG_UINT64_ARGS(msg, idx) ((uint64_t) *(msg)->spec->args->ptr[idx])
-+
-+#define DM_MSG_STR_ARG(msg) DM_MSG_STR_ARGS(msg, 0)
-+#define DM_MSG_INT_ARG(msg) DM_MSG_INT_ARGS(msg, 0)
-+#define DM_MSG_UINT_ARG(msg) DM_MSG_UINT_ARGS(msg, 0)
-+#define DM_MSG_UINT64_ARG(msg) DM_MSG_UINT64_ARGS(msg, 0)
-+
-+
-+/* Parse a message and its options and optionally call a function back. */
-+int dm_message_parse(const char *caller, struct dm_msg *msg, void *context,
-+ int argc, char **argv);
-+
-+#endif
---- /dev/null
-+++ b/drivers/md/dm-raid45.c
-@@ -0,0 +1,4516 @@
-+/*
-+ * Copyright (C) 2005-2008 Red Hat, Inc. All rights reserved.
-+ *
-+ * Module Author: Heinz Mauelshagen <Mauelshagen@RedHat.com>
-+ *
-+ * This file is released under the GPL.
-+ *
-+ *
-+ * Linux 2.6 Device Mapper RAID4 and RAID5 target.
-+ *
-+ * Supports:
-+ * o RAID4 with dedicated and selectable parity device
-+ * o RAID5 with rotating parity (left+right, symmetric+asymmetric)
-+ * o run time optimization of xor algorithm used to calculate parity
-+ *
-+ *
-+ * Thanks to MD for:
-+ * o the raid address calculation algorithm
-+ * o the base of the biovec <-> page list copier.
-+ *
-+ *
-+ * Uses region hash to keep track of how many writes are in flight to
-+ * regions in order to use dirty log to keep state of regions to recover:
-+ *
-+ * o clean regions (those which are synchronized
-+ * and don't have write io in flight)
-+ * o dirty regions (those with write io in flight)
-+ *
-+ *
-+ * On startup, any dirty regions are migrated to the 'nosync' state
-+ * and are subject to recovery by the daemon.
-+ *
-+ * See raid_ctr() for table definition.
-+ *
-+ *
-+ * FIXME:
-+ * o add virtual interface for locking
-+ * o remove instrumentation (REMOVEME:)
-+ *
-+ */
-+
-+static const char *version = "v0.2431";
-+
-+#include "dm.h"
-+#include "dm-bio-list.h"
-+#include "dm-memcache.h"
-+#include "dm-message.h"
-+#include "dm-raid45.h"
-+
-+#include <linux/kernel.h>
-+#include <linux/vmalloc.h>
-+
-+#include <linux/dm-io.h>
-+#include <linux/dm-dirty-log.h>
-+#include <linux/dm-regions.h>
-+
-+/* # of parallel recovered regions */
-+/* FIXME: cope with multiple recovery stripes in raid_set struct. */
-+#define MAX_RECOVER 1 /* needs to be 1! */
-+
-+/*
-+ * Configurable parameters
-+ */
-+#define INLINE
-+
-+/* Default # of stripes if not set in constructor. */
-+#define STRIPES 64
-+
-+/* Minimum/maximum # of selectable stripes. */
-+#define STRIPES_MIN 8
-+#define STRIPES_MAX 16384
-+
-+/* Default chunk size in sectors if not set in constructor. */
-+#define CHUNK_SIZE 64
-+
-+/* Default io size in sectors if not set in constructor. */
-+#define IO_SIZE_MIN SECTORS_PER_PAGE
-+#define IO_SIZE IO_SIZE_MIN
-+
-+/* Maximum setable chunk size in sectors. */
-+#define CHUNK_SIZE_MAX 16384
-+
-+/* Recover io size default in sectors. */
-+#define RECOVER_IO_SIZE_MIN 64
-+#define RECOVER_IO_SIZE 256
-+
-+/* Default percentage recover io bandwidth. */
-+#define BANDWIDTH 10
-+#define BANDWIDTH_MIN 1
-+#define BANDWIDTH_MAX 100
-+/*
-+ * END Configurable parameters
-+ */
-+
-+#define TARGET "dm-raid45"
-+#define DAEMON "kraid45d"
-+#define DM_MSG_PREFIX TARGET
-+
-+#define SECTORS_PER_PAGE (PAGE_SIZE >> SECTOR_SHIFT)
-+
-+/* Amount/size for __xor(). */
-+#define SECTORS_PER_XOR SECTORS_PER_PAGE
-+#define XOR_SIZE PAGE_SIZE
-+
-+/* Derive raid_set from stripe_cache pointer. */
-+#define RS(x) container_of(x, struct raid_set, sc)
-+
-+/* Check value in range. */
-+#define range_ok(i, min, max) (i >= min && i <= max)
-+
-+/* Page reference. */
-+#define PAGE(stripe, p) ((stripe)->obj[p].pl->page)
-+
-+/* Bio list reference. */
-+#define BL(stripe, p, rw) (stripe->ss[p].bl + rw)
-+
-+/* Page list reference. */
-+#define PL(stripe, p) (stripe->obj[p].pl)
-+
-+/* Check argument is power of 2. */
-+#define POWER_OF_2(a) (!(a & (a - 1)))
-+
-+/* Factor out to dm-bio-list.h */
-+static inline void bio_list_push(struct bio_list *bl, struct bio *bio)
-+{
-+ bio->bi_next = bl->head;
-+ bl->head = bio;
-+
-+ if (!bl->tail)
-+ bl->tail = bio;
-+}
-+
-+/* Factor out to dm.h */
-+#define TI_ERR_RET(str, ret) \
-+ do { ti->error = DM_MSG_PREFIX ": " str; return ret; } while (0);
-+#define TI_ERR(str) TI_ERR_RET(str, -EINVAL)
-+
-+/*-----------------------------------------------------------------
-+ * Stripe cache
-+ *
-+ * Cache for all reads and writes to raid sets (operational or degraded)
-+ *
-+ * We need to run all data to and from a RAID set through this cache,
-+ * because parity chunks need to get calculated from data chunks
-+ * or, in the degraded/resynchronization case, missing chunks need
-+ * to be reconstructed using the other chunks of the stripe.
-+ *---------------------------------------------------------------*/
-+/* Protect kmem cache # counter. */
-+static atomic_t _stripe_sc_nr = ATOMIC_INIT(-1); /* kmem cache # counter. */
-+
-+/* A stripe set (holds bios hanging off). */
-+struct stripe_set {
-+ struct stripe *stripe; /* Backpointer to stripe for endio(). */
-+ struct bio_list bl[3]; /* Reads, writes, and writes merged. */
-+#define WRITE_MERGED 2
-+};
-+
-+#if READ != 0 || WRITE != 1
-+#error dm-raid45: READ/WRITE != 0/1 used as index!!!
-+#endif
-+
-+/*
-+ * Stripe linked list indexes. Keep order, because the stripe
-+ * and the stripe cache rely on the first 3!
-+ */
-+enum list_types {
-+ LIST_IO = 0, /* Stripes with io pending. */
-+ LIST_ENDIO, /* Stripes to endio. */
-+ LIST_LRU, /* Least recently used stripes. */
-+ LIST_HASH, /* Hashed stripes. */
-+ LIST_RECOVER = LIST_HASH, /* For recovery type stripes only. */
-+ NR_LISTS, /* To size array in struct stripe. */
-+};
-+
-+enum lock_types {
-+ LOCK_ENDIO = 0, /* Protect endio list. */
-+ LOCK_LRU, /* Protect lru list. */
-+ NR_LOCKS, /* To size array in struct stripe_cache. */
-+};
-+
-+/* A stripe: the io object to handle all reads and writes to a RAID set. */
-+struct stripe {
-+ struct stripe_cache *sc; /* Backpointer to stripe cache. */
-+
-+ sector_t key; /* Hash key. */
-+ sector_t region; /* Region stripe is mapped to. */
-+
-+ /* Reference count. */
-+ atomic_t cnt;
-+
-+ struct {
-+ unsigned long flags; /* flags (see below). */
-+
-+ /*
-+ * Pending ios in flight:
-+ *
-+ * used as a 'lock' to control move of stripe to endio list
-+ */
-+ atomic_t pending; /* Pending ios in flight. */
-+
-+ /* Sectors to read and write for multi page stripe sets. */
-+ unsigned size;
-+ } io;
-+
-+ /* Lock on stripe (for clustering). */
-+ void *lock;
-+
-+ /*
-+ * 4 linked lists:
-+ * o io list to flush io
-+ * o endio list
-+ * o LRU list to put stripes w/o reference count on
-+ * o stripe cache hash
-+ */
-+ struct list_head lists[NR_LISTS];
-+
-+ struct {
-+ unsigned short parity; /* Parity chunk index. */
-+ short recover; /* Recovery chunk index. */
-+ } idx;
-+
-+ /* This sets memory cache object (dm-mem-cache). */
-+ struct dm_mem_cache_object *obj;
-+
-+ /* Array of stripe sets (dynamically allocated). */
-+ struct stripe_set ss[0];
-+};
-+
-+/* States stripes can be in (flags field). */
-+enum stripe_states {
-+ STRIPE_ACTIVE, /* Active io on stripe. */
-+ STRIPE_ERROR, /* io error on stripe. */
-+ STRIPE_MERGED, /* Writes got merged. */
-+ STRIPE_READ, /* Read. */
-+ STRIPE_RBW, /* Read-before-write. */
-+ STRIPE_RECONSTRUCT, /* reconstruct of a missing chunk required. */
-+ STRIPE_RECOVER, /* Stripe used for RAID set recovery. */
-+};
-+
-+/* ... and macros to access them. */
-+#define BITOPS(name, what, var, flag) \
-+static inline int TestClear ## name ## what(struct var *v) \
-+{ return test_and_clear_bit(flag, &v->io.flags); } \
-+static inline int TestSet ## name ## what(struct var *v) \
-+{ return test_and_set_bit(flag, &v->io.flags); } \
-+static inline void Clear ## name ## what(struct var *v) \
-+{ clear_bit(flag, &v->io.flags); } \
-+static inline void Set ## name ## what(struct var *v) \
-+{ set_bit(flag, &v->io.flags); } \
-+static inline int name ## what(struct var *v) \
-+{ return test_bit(flag, &v->io.flags); }
-+
-+
-+BITOPS(Stripe, Active, stripe, STRIPE_ACTIVE)
-+BITOPS(Stripe, Merged, stripe, STRIPE_MERGED)
-+BITOPS(Stripe, Error, stripe, STRIPE_ERROR)
-+BITOPS(Stripe, Read, stripe, STRIPE_READ)
-+BITOPS(Stripe, RBW, stripe, STRIPE_RBW)
-+BITOPS(Stripe, Reconstruct, stripe, STRIPE_RECONSTRUCT)
-+BITOPS(Stripe, Recover, stripe, STRIPE_RECOVER)
-+
-+/* A stripe hash. */
-+struct stripe_hash {
-+ struct list_head *hash;
-+ unsigned buckets;
-+ unsigned mask;
-+ unsigned prime;
-+ unsigned shift;
-+};
-+
-+/* A stripe cache. */
-+struct stripe_cache {
-+ /* Stripe hash. */
-+ struct stripe_hash hash;
-+
-+ /* Stripes with io to flush, stripes to endio and LRU lists. */
-+ struct list_head lists[3];
-+
-+ /* Locks to protect endio and lru lists. */
-+ spinlock_t locks[NR_LOCKS];
-+
-+ /* Slab cache to allocate stripes from. */
-+ struct {
-+ struct kmem_cache *cache; /* Cache itself. */
-+ char name[32]; /* Unique name. */
-+ } kc;
-+
-+ struct dm_io_client *dm_io_client; /* dm-io client resource context. */
-+
-+ /* dm-mem-cache client resource context. */
-+ struct dm_mem_cache_client *mem_cache_client;
-+
-+ int stripes_parm; /* # stripes parameter from constructor. */
-+ atomic_t stripes; /* actual # of stripes in cache. */
-+ atomic_t stripes_to_shrink; /* # of stripes to shrink cache by. */
-+ atomic_t stripes_last; /* last # of stripes in cache. */
-+ atomic_t active_stripes; /* actual # of active stripes in cache. */
-+
-+ /* REMOVEME: */
-+ atomic_t max_active_stripes; /* actual # of active stripes in cache. */
-+};
-+
-+/* Flag specs for raid_dev */ ;
-+enum raid_dev_flags { DEVICE_FAILED, IO_QUEUED };
-+
-+/* The raid device in a set. */
-+struct raid_dev {
-+ struct dm_dev *dev;
-+ unsigned long flags; /* raid_dev_flags. */
-+ sector_t start; /* offset to map to. */
-+};
-+
-+/* Flags spec for raid_set. */
-+enum raid_set_flags {
-+ RS_CHECK_OVERWRITE, /* Check for chunk overwrites. */
-+ RS_DEAD, /* RAID set inoperational. */
-+ RS_DEVEL_STATS, /* REMOVEME: display status information. */
-+ RS_IO_ERROR, /* io error on set. */
-+ RS_RECOVER, /* Do recovery. */
-+ RS_RECOVERY_BANDWIDTH, /* Allow recovery bandwidth (delayed bios). */
-+ RS_REGION_GET, /* get a region to recover. */
-+ RS_SC_BUSY, /* stripe cache busy -> send an event. */
-+ RS_SUSPENDED, /* RAID set suspendedn. */
-+};
-+
-+/* REMOVEME: devel stats counters. */
-+enum stats_types {
-+ S_BIOS_READ,
-+ S_BIOS_ADDED_READ,
-+ S_BIOS_ENDIO_READ,
-+ S_BIOS_WRITE,
-+ S_BIOS_ADDED_WRITE,
-+ S_BIOS_ENDIO_WRITE,
-+ S_CAN_MERGE,
-+ S_CANT_MERGE,
-+ S_CONGESTED,
-+ S_DM_IO_READ,
-+ S_DM_IO_WRITE,
-+ S_ACTIVE_READS,
-+ S_BANDWIDTH,
-+ S_BARRIER,
-+ S_BIO_COPY_PL_NEXT,
-+ S_DEGRADED,
-+ S_DELAYED_BIOS,
-+ S_EVICT,
-+ S_FLUSHS,
-+ S_HITS_1ST,
-+ S_IOS_POST,
-+ S_INSCACHE,
-+ S_MAX_LOOKUP,
-+ S_MERGE_PAGE_LOCKED,
-+ S_NO_BANDWIDTH,
-+ S_NOT_CONGESTED,
-+ S_NO_RW,
-+ S_NOSYNC,
-+ S_PROHIBITPAGEIO,
-+ S_RECONSTRUCT_EI,
-+ S_RECONSTRUCT_DEV,
-+ S_REDO,
-+ S_REQUEUE,
-+ S_STRIPE_ERROR,
-+ S_SUM_DELAYED_BIOS,
-+ S_XORS,
-+ S_NR_STATS, /* # of stats counters. */
-+};
-+
-+/* Status type -> string mappings. */
-+struct stats_map {
-+ const enum stats_types type;
-+ const char *str;
-+};
-+
-+static struct stats_map stats_map[] = {
-+ { S_BIOS_READ, "r=" },
-+ { S_BIOS_ADDED_READ, "/" },
-+ { S_BIOS_ENDIO_READ, "/" },
-+ { S_BIOS_WRITE, " w=" },
-+ { S_BIOS_ADDED_WRITE, "/" },
-+ { S_BIOS_ENDIO_WRITE, "/" },
-+ { S_DM_IO_READ, " rc=" },
-+ { S_DM_IO_WRITE, " wc=" },
-+ { S_ACTIVE_READS, " active_reads=" },
-+ { S_BANDWIDTH, " bandwidth=" },
-+ { S_NO_BANDWIDTH, " no_bandwidth=" },
-+ { S_BARRIER, " barrier=" },
-+ { S_BIO_COPY_PL_NEXT, " bio_copy_pl_next=" },
-+ { S_CAN_MERGE, " can_merge=" },
-+ { S_MERGE_PAGE_LOCKED, "/page_locked=" },
-+ { S_CANT_MERGE, "/cant_merge=" },
-+ { S_CONGESTED, " congested=" },
-+ { S_NOT_CONGESTED, "/not_congested=" },
-+ { S_DEGRADED, " degraded=" },
-+ { S_DELAYED_BIOS, " delayed_bios=" },
-+ { S_SUM_DELAYED_BIOS, "/sum_delayed_bios=" },
-+ { S_EVICT, " evict=" },
-+ { S_FLUSHS, " flushs=" },
-+ { S_HITS_1ST, " hits_1st=" },
-+ { S_IOS_POST, " ios_post=" },
-+ { S_INSCACHE, " inscache=" },
-+ { S_MAX_LOOKUP, " max_lookup=" },
-+ { S_NO_RW, " no_rw=" },
-+ { S_NOSYNC, " nosync=" },
-+ { S_PROHIBITPAGEIO, " ProhibitPageIO=" },
-+ { S_RECONSTRUCT_EI, " reconstruct_ei=" },
-+ { S_RECONSTRUCT_DEV, " reconstruct_dev=" },
-+ { S_REDO, " redo=" },
-+ { S_REQUEUE, " requeue=" },
-+ { S_STRIPE_ERROR, " stripe_error=" },
-+ { S_XORS, " xors=" },
-+};
-+
-+/*
-+ * A RAID set.
-+ */
-+typedef void (*xor_function_t)(unsigned count, unsigned long **data);
-+struct raid_set {
-+ struct dm_target *ti; /* Target pointer. */
-+
-+ struct {
-+ unsigned long flags; /* State flags. */
-+ spinlock_t in_lock; /* Protects central input list below. */
-+ struct bio_list in; /* Pending ios (central input list). */
-+ struct bio_list work; /* ios work set. */
-+ wait_queue_head_t suspendq; /* suspend synchronization. */
-+ atomic_t in_process; /* counter of queued bios (suspendq). */
-+ atomic_t in_process_max;/* counter of queued bios max. */
-+
-+ /* io work. */
-+ struct workqueue_struct *wq;
-+ struct delayed_work dws;
-+ } io;
-+
-+ /* External locking. */
-+ struct dm_raid45_locking_type *locking;
-+
-+ struct stripe_cache sc; /* Stripe cache for this set. */
-+
-+ /* Xor optimization. */
-+ struct {
-+ struct xor_func *f;
-+ unsigned chunks;
-+ unsigned speed;
-+ } xor;
-+
-+ /* Recovery parameters. */
-+ struct recover {
-+ struct dm_dirty_log *dl; /* Dirty log. */
-+ struct dm_rh_client *rh; /* Region hash. */
-+
-+ /* dm-mem-cache client resource context for recovery stripes. */
-+ struct dm_mem_cache_client *mem_cache_client;
-+
-+ struct list_head stripes; /* List of recovery stripes. */
-+
-+ region_t nr_regions;
-+ region_t nr_regions_to_recover;
-+ region_t nr_regions_recovered;
-+ unsigned long start_jiffies;
-+ unsigned long end_jiffies;
-+
-+ unsigned bandwidth; /* Recovery bandwidth [%]. */
-+ unsigned bandwidth_work; /* Recovery bandwidth [factor]. */
-+ unsigned bandwidth_parm; /* " constructor parm. */
-+ unsigned io_size; /* io size <= chunk size. */
-+ unsigned io_size_parm; /* io size ctr parameter. */
-+
-+ /* recovery io throttling. */
-+ atomic_t io_count[2]; /* counter recover/regular io. */
-+ unsigned long last_jiffies;
-+
-+ struct dm_region *reg; /* Actual region to recover. */
-+ sector_t pos; /* Position within region to recover. */
-+ sector_t end; /* End of region to recover. */
-+ } recover;
-+
-+ /* RAID set parameters. */
-+ struct {
-+ struct raid_type *raid_type; /* RAID type (eg, RAID4). */
-+ unsigned raid_parms; /* # variable raid parameters. */
-+
-+ unsigned chunk_size; /* Sectors per chunk. */
-+ unsigned chunk_size_parm;
-+ unsigned chunk_mask; /* Mask for amount. */
-+ unsigned chunk_shift; /* rsector chunk size shift. */
-+
-+ unsigned io_size; /* Sectors per io. */
-+ unsigned io_size_parm;
-+ unsigned io_mask; /* Mask for amount. */
-+ unsigned io_shift_mask; /* Mask for raid_address(). */
-+ unsigned io_shift; /* rsector io size shift. */
-+ unsigned pages_per_io; /* Pages per io. */
-+
-+ sector_t sectors_per_dev; /* Sectors per device. */
-+
-+ atomic_t failed_devs; /* Amount of devices failed. */
-+
-+ /* Index of device to initialize. */
-+ int dev_to_init;
-+ int dev_to_init_parm;
-+
-+ /* Raid devices dynamically allocated. */
-+ unsigned raid_devs; /* # of RAID devices below. */
-+ unsigned data_devs; /* # of RAID data devices. */
-+
-+ int ei; /* index of failed RAID device. */
-+
-+ /* index of dedicated parity device (i.e. RAID4). */
-+ int pi;
-+ int pi_parm; /* constructor parm for status output. */
-+ } set;
-+
-+ /* REMOVEME: devel stats counters. */
-+ atomic_t stats[S_NR_STATS];
-+
-+ /* Dynamically allocated temporary pointers for xor(). */
-+ unsigned long **data;
-+
-+ /* Dynamically allocated RAID devices. Alignment? */
-+ struct raid_dev dev[0];
-+};
-+
-+
-+BITOPS(RS, Bandwidth, raid_set, RS_RECOVERY_BANDWIDTH)
-+BITOPS(RS, CheckOverwrite, raid_set, RS_CHECK_OVERWRITE)
-+BITOPS(RS, Dead, raid_set, RS_DEAD)
-+BITOPS(RS, DevelStats, raid_set, RS_DEVEL_STATS)
-+BITOPS(RS, IoError, raid_set, RS_IO_ERROR)
-+BITOPS(RS, Recover, raid_set, RS_RECOVER)
-+BITOPS(RS, RegionGet, raid_set, RS_REGION_GET)
-+BITOPS(RS, ScBusy, raid_set, RS_SC_BUSY)
-+BITOPS(RS, Suspended, raid_set, RS_SUSPENDED)
-+#undef BITOPS
-+
-+#define PageIO(page) PageChecked(page)
-+#define AllowPageIO(page) SetPageChecked(page)
-+#define ProhibitPageIO(page) ClearPageChecked(page)
-+
-+/*-----------------------------------------------------------------
-+ * Raid-4/5 set structures.
-+ *---------------------------------------------------------------*/
-+/* RAID level definitions. */
-+enum raid_level {
-+ raid4,
-+ raid5,
-+};
-+
-+/* Symmetric/Asymmetric, Left/Right parity rotating algorithms. */
-+enum raid_algorithm {
-+ none,
-+ left_asym,
-+ right_asym,
-+ left_sym,
-+ right_sym,
-+};
-+
-+struct raid_type {
-+ const char *name; /* RAID algorithm. */
-+ const char *descr; /* Descriptor text for logging. */
-+ const unsigned parity_devs; /* # of parity devices. */
-+ const unsigned minimal_devs; /* minimal # of devices in set. */
-+ const enum raid_level level; /* RAID level. */
-+ const enum raid_algorithm algorithm; /* RAID algorithm. */
-+};
-+
-+/* Supported raid types and properties. */
-+static struct raid_type raid_types[] = {
-+ {"raid4", "RAID4 (dedicated parity disk)", 1, 3, raid4, none},
-+ {"raid5_la", "RAID5 (left asymmetric)", 1, 3, raid5, left_asym},
-+ {"raid5_ra", "RAID5 (right asymmetric)", 1, 3, raid5, right_asym},
-+ {"raid5_ls", "RAID5 (left symmetric)", 1, 3, raid5, left_sym},
-+ {"raid5_rs", "RAID5 (right symmetric)", 1, 3, raid5, right_sym},
-+};
-+
-+/* Address as calculated by raid_address(). */
-+struct address {
-+ sector_t key; /* Hash key (start address of stripe). */
-+ unsigned di, pi; /* Data and parity disks index. */
-+};
-+
-+/* REMOVEME: reset statistics counters. */
-+static void stats_reset(struct raid_set *rs)
-+{
-+ unsigned s = S_NR_STATS;
-+
-+ while (s--)
-+ atomic_set(rs->stats + s, 0);
-+}
-+
-+/*----------------------------------------------------------------
-+ * RAID set management routines.
-+ *--------------------------------------------------------------*/
-+/*
-+ * Begin small helper functions.
-+ */
-+/* Queue (optionally delayed) io work. */
-+static void wake_do_raid_delayed(struct raid_set *rs, unsigned long delay)
-+{
-+ struct delayed_work *dws = &rs->io.dws;
-+
-+ cancel_delayed_work(dws);
-+ queue_delayed_work(rs->io.wq, dws, delay);
-+}
-+
-+/* Queue io work immediately (called from region hash too). */
-+static INLINE void wake_do_raid(void *context)
-+{
-+ wake_do_raid_delayed(context, 0);
-+}
-+
-+/* Wait until all io has been processed. */
-+static INLINE void wait_ios(struct raid_set *rs)
-+{
-+ wait_event(rs->io.suspendq, !atomic_read(&rs->io.in_process));
-+}
-+
-+/* Declare io queued to device. */
-+static INLINE void io_dev_queued(struct raid_dev *dev)
-+{
-+ set_bit(IO_QUEUED, &dev->flags);
-+}
-+
-+/* Io on device and reset ? */
-+static inline int io_dev_clear(struct raid_dev *dev)
-+{
-+ return test_and_clear_bit(IO_QUEUED, &dev->flags);
-+}
-+
-+/* Get an io reference. */
-+static INLINE void io_get(struct raid_set *rs)
-+{
-+ int p = atomic_inc_return(&rs->io.in_process);
-+
-+ if (p > atomic_read(&rs->io.in_process_max))
-+ atomic_set(&rs->io.in_process_max, p); /* REMOVEME: max. */
-+}
-+
-+/* Put the io reference and conditionally wake io waiters. */
-+static INLINE void io_put(struct raid_set *rs)
-+{
-+ /* Intel: rebuild data corrupter? */
-+ if (!atomic_read(&rs->io.in_process)) {
-+ DMERR("%s would go negative!!!", __func__);
-+ return;
-+ }
-+
-+ if (atomic_dec_and_test(&rs->io.in_process))
-+ wake_up(&rs->io.suspendq);
-+}
-+
-+/* Calculate device sector offset. */
-+static INLINE sector_t _sector(struct raid_set *rs, struct bio *bio)
-+{
-+ sector_t sector = bio->bi_sector;
-+
-+ sector_div(sector, rs->set.data_devs);
-+ return sector;
-+}
-+
-+/* Test device operational. */
-+static INLINE int dev_operational(struct raid_set *rs, unsigned p)
-+{
-+ return !test_bit(DEVICE_FAILED, &rs->dev[p].flags);
-+}
-+
-+/* Return # of active stripes in stripe cache. */
-+static INLINE int sc_active(struct stripe_cache *sc)
-+{
-+ return atomic_read(&sc->active_stripes);
-+}
-+
-+/* Test io pending on stripe. */
-+static INLINE int stripe_io(struct stripe *stripe)
-+{
-+ return atomic_read(&stripe->io.pending);
-+}
-+
-+static INLINE void stripe_io_inc(struct stripe *stripe)
-+{
-+ atomic_inc(&stripe->io.pending);
-+}
-+
-+static INLINE void stripe_io_dec(struct stripe *stripe)
-+{
-+ atomic_dec(&stripe->io.pending);
-+}
-+
-+/* Wrapper needed by for_each_io_dev(). */
-+static void _stripe_io_inc(struct stripe *stripe, unsigned p)
-+{
-+ stripe_io_inc(stripe);
-+}
-+
-+/* Error a stripe. */
-+static INLINE void stripe_error(struct stripe *stripe, struct page *page)
-+{
-+ SetStripeError(stripe);
-+ SetPageError(page);
-+ atomic_inc(RS(stripe->sc)->stats + S_STRIPE_ERROR);
-+}
-+
-+/* Page IOed ok. */
-+enum dirty_type { CLEAN, DIRTY };
-+static INLINE void page_set(struct page *page, enum dirty_type type)
-+{
-+ switch (type) {
-+ case DIRTY:
-+ SetPageDirty(page);
-+ AllowPageIO(page);
-+ break;
-+
-+ case CLEAN:
-+ ClearPageDirty(page);
-+ break;
-+
-+ default:
-+ BUG();
-+ }
-+
-+ SetPageUptodate(page);
-+ ClearPageError(page);
-+}
-+
-+/* Return region state for a sector. */
-+static INLINE int
-+region_state(struct raid_set *rs, sector_t sector, unsigned long state)
-+{
-+ struct dm_rh_client *rh = rs->recover.rh;
-+
-+ return RSRecover(rs) ?
-+ (dm_rh_get_state(rh, dm_rh_sector_to_region(rh, sector), 1) &
-+ state) : 0;
-+}
-+
-+/* Check maximum devices which may fail in a raid set. */
-+static inline int raid_set_degraded(struct raid_set *rs)
-+{
-+ return RSIoError(rs);
-+}
-+
-+/* Check # of devices which may fail in a raid set. */
-+static INLINE int raid_set_operational(struct raid_set *rs)
-+{
-+ /* Too many failed devices -> BAD. */
-+ return atomic_read(&rs->set.failed_devs) <=
-+ rs->set.raid_type->parity_devs;
-+}
-+
-+/*
-+ * Return true in case a page_list should be read/written
-+ *
-+ * Conditions to read/write:
-+ * o 1st page in list not uptodate
-+ * o 1st page in list dirty
-+ * o if we optimized io away, we flag it using the pages checked bit.
-+ */
-+static INLINE unsigned page_io(struct page *page)
-+{
-+ /* Optimization: page was flagged to need io during first run. */
-+ if (PagePrivate(page)) {
-+ ClearPagePrivate(page);
-+ return 1;
-+ }
-+
-+ /* Avoid io if prohibited or a locked page. */
-+ if (!PageIO(page) || PageLocked(page))
-+ return 0;
-+
-+ if (!PageUptodate(page) || PageDirty(page)) {
-+ /* Flag page needs io for second run optimization. */
-+ SetPagePrivate(page);
-+ return 1;
-+ }
-+
-+ return 0;
-+}
-+
-+/* Call a function on each page list needing io. */
-+static INLINE unsigned
-+for_each_io_dev(struct raid_set *rs, struct stripe *stripe,
-+ void (*f_io)(struct stripe *stripe, unsigned p))
-+{
-+ unsigned p = rs->set.raid_devs, r = 0;
-+
-+ while (p--) {
-+ if (page_io(PAGE(stripe, p))) {
-+ f_io(stripe, p);
-+ r++;
-+ }
-+ }
-+
-+ return r;
-+}
-+
-+/* Reconstruct a particular device ?. */
-+static INLINE int dev_to_init(struct raid_set *rs)
-+{
-+ return rs->set.dev_to_init > -1;
-+}
-+
-+/*
-+ * Index of device to calculate parity on.
-+ * Either the parity device index *or* the selected device to init
-+ * after a spare replacement.
-+ */
-+static INLINE unsigned dev_for_parity(struct stripe *stripe)
-+{
-+ struct raid_set *rs = RS(stripe->sc);
-+
-+ return dev_to_init(rs) ? rs->set.dev_to_init : stripe->idx.parity;
-+}
-+
-+/* Return the index of the device to be recovered. */
-+static int idx_get(struct raid_set *rs)
-+{
-+ /* Avoid to read in the pages to be reconstructed anyway. */
-+ if (dev_to_init(rs))
-+ return rs->set.dev_to_init;
-+ else if (rs->set.raid_type->level == raid4)
-+ return rs->set.pi;
-+
-+ return -1;
-+}
-+
-+/* RAID set congested function. */
-+static int raid_set_congested(void *congested_data, int bdi_bits)
-+{
-+ struct raid_set *rs = congested_data;
-+ int r = 0; /* Assume uncongested. */
-+ unsigned p = rs->set.raid_devs;
-+
-+ /* If any of our component devices are overloaded. */
-+ while (p--) {
-+ struct request_queue *q = bdev_get_queue(rs->dev[p].dev->bdev);
-+
-+ r |= bdi_congested(&q->backing_dev_info, bdi_bits);
-+ }
-+
-+ /* REMOVEME: statistics. */
-+ atomic_inc(rs->stats + (r ? S_CONGESTED : S_NOT_CONGESTED));
-+ return r;
-+}
-+
-+/* Display RAID set dead message once. */
-+static void raid_set_dead(struct raid_set *rs)
-+{
-+ if (!TestSetRSDead(rs)) {
-+ unsigned p;
-+ char buf[BDEVNAME_SIZE];
-+
-+ DMERR("FATAL: too many devices failed -> RAID set dead");
-+
-+ for (p = 0; p < rs->set.raid_devs; p++) {
-+ if (!dev_operational(rs, p))
-+ DMERR("device /dev/%s failed",
-+ bdevname(rs->dev[p].dev->bdev, buf));
-+ }
-+ }
-+}
-+
-+/* RAID set degrade check. */
-+static INLINE int
-+raid_set_check_and_degrade(struct raid_set *rs,
-+ struct stripe *stripe, unsigned p)
-+{
-+ if (test_and_set_bit(DEVICE_FAILED, &rs->dev[p].flags))
-+ return -EPERM;
-+
-+ /* Through an event in case of member device errors. */
-+ dm_table_event(rs->ti->table);
-+ atomic_inc(&rs->set.failed_devs);
-+
-+ /* Only log the first member error. */
-+ if (!TestSetRSIoError(rs)) {
-+ char buf[BDEVNAME_SIZE];
-+
-+ /* Store index for recovery. */
-+ mb();
-+ rs->set.ei = p;
-+ mb();
-+
-+ DMERR("CRITICAL: %sio error on device /dev/%s "
-+ "in region=%llu; DEGRADING RAID set",
-+ stripe ? "" : "FAKED ",
-+ bdevname(rs->dev[p].dev->bdev, buf),
-+ (unsigned long long) (stripe ? stripe->key : 0));
-+ DMERR("further device error messages suppressed");
-+ }
-+
-+ return 0;
-+}
-+
-+static void
-+raid_set_check_degrade(struct raid_set *rs, struct stripe *stripe)
-+{
-+ unsigned p = rs->set.raid_devs;
-+
-+ while (p--) {
-+ struct page *page = PAGE(stripe, p);
-+
-+ if (PageError(page)) {
-+ ClearPageError(page);
-+ raid_set_check_and_degrade(rs, stripe, p);
-+ }
-+ }
-+}
-+
-+/* RAID set upgrade check. */
-+static int raid_set_check_and_upgrade(struct raid_set *rs, unsigned p)
-+{
-+ if (!test_and_clear_bit(DEVICE_FAILED, &rs->dev[p].flags))
-+ return -EPERM;
-+
-+ if (atomic_dec_and_test(&rs->set.failed_devs)) {
-+ ClearRSIoError(rs);
-+ rs->set.ei = -1;
-+ }
-+
-+ return 0;
-+}
-+
-+/* Lookup a RAID device by name or by major:minor number. */
-+union dev_lookup {
-+ const char *dev_name;
-+ struct raid_dev *dev;
-+};
-+enum lookup_type { byname, bymajmin, bynumber };
-+static int raid_dev_lookup(struct raid_set *rs, enum lookup_type by,
-+ union dev_lookup *dl)
-+{
-+ unsigned p;
-+
-+ /*
-+ * Must be an incremental loop, because the device array
-+ * can have empty slots still on calls from raid_ctr()
-+ */
-+ for (p = 0; p < rs->set.raid_devs; p++) {
-+ char buf[BDEVNAME_SIZE];
-+ struct raid_dev *dev = rs->dev + p;
-+
-+ if (!dev->dev)
-+ break;
-+
-+ /* Format dev string appropriately if necessary. */
-+ if (by == byname)
-+ bdevname(dev->dev->bdev, buf);
-+ else if (by == bymajmin)
-+ format_dev_t(buf, dev->dev->bdev->bd_dev);
-+
-+ /* Do the actual check. */
-+ if (by == bynumber) {
-+ if (dl->dev->dev->bdev->bd_dev ==
-+ dev->dev->bdev->bd_dev)
-+ return p;
-+ } else if (!strcmp(dl->dev_name, buf))
-+ return p;
-+ }
-+
-+ return -ENODEV;
-+}
-+
-+/* End io wrapper. */
-+static INLINE void
-+_bio_endio(struct raid_set *rs, struct bio *bio, int error)
-+{
-+ /* REMOVEME: statistics. */
-+ atomic_inc(rs->stats + (bio_data_dir(bio) == WRITE ?
-+ S_BIOS_ENDIO_WRITE : S_BIOS_ENDIO_READ));
-+ bio_endio(bio, error);
-+ io_put(rs); /* Wake any suspend waiters. */
-+}
-+
-+/*
-+ * End small helper functions.
-+ */
-+
-+
-+/*
-+ * Stripe hash functions
-+ */
-+/* Initialize/destroy stripe hash. */
-+static int hash_init(struct stripe_hash *hash, unsigned stripes)
-+{
-+ unsigned buckets = 2, max_buckets = stripes / 4;
-+ unsigned hash_primes[] = {
-+ /* Table of primes for hash_fn/table size optimization. */
-+ 3, 7, 13, 27, 53, 97, 193, 389, 769,
-+ 1543, 3079, 6151, 12289, 24593,
-+ };
-+
-+ /* Calculate number of buckets (2^^n <= stripes / 4). */
-+ while (buckets < max_buckets)
-+ buckets <<= 1;
-+
-+ /* Allocate stripe hash. */
-+ hash->hash = vmalloc(buckets * sizeof(*hash->hash));
-+ if (!hash->hash)
-+ return -ENOMEM;
-+
-+ hash->buckets = buckets;
-+ hash->mask = buckets - 1;
-+ hash->shift = ffs(buckets);
-+ if (hash->shift > ARRAY_SIZE(hash_primes) + 1)
-+ hash->shift = ARRAY_SIZE(hash_primes) + 1;
-+
-+ BUG_ON(hash->shift - 2 > ARRAY_SIZE(hash_primes) + 1);
-+ hash->prime = hash_primes[hash->shift - 2];
-+
-+ /* Initialize buckets. */
-+ while (buckets--)
-+ INIT_LIST_HEAD(hash->hash + buckets);
-+
-+ return 0;
-+}
-+
-+static INLINE void hash_exit(struct stripe_hash *hash)
-+{
-+ if (hash->hash) {
-+ vfree(hash->hash);
-+ hash->hash = NULL;
-+ }
-+}
-+
-+/* List add (head/tail/locked/unlocked) inlines. */
-+enum list_lock_type { LIST_LOCKED, LIST_UNLOCKED };
-+#define LIST_DEL(name, list) \
-+static void stripe_ ## name ## _del(struct stripe *stripe, \
-+ enum list_lock_type lock) { \
-+ struct list_head *lh = stripe->lists + (list); \
-+ spinlock_t *l = NULL; \
-+\
-+ if (lock == LIST_LOCKED) { \
-+ l = stripe->sc->locks + LOCK_LRU; \
-+ spin_lock_irq(l); \
-+ } \
-+\
-+\
-+ if (!list_empty(lh)) \
-+ list_del_init(lh); \
-+\
-+ if (lock == LIST_LOCKED) \
-+ spin_unlock_irq(l); \
-+}
-+
-+LIST_DEL(hash, LIST_HASH)
-+LIST_DEL(lru, LIST_LRU)
-+#undef LIST_DEL
-+
-+enum list_pos_type { POS_HEAD, POS_TAIL };
-+#define LIST_ADD(name, list) \
-+static void stripe_ ## name ## _add(struct stripe *stripe, \
-+ enum list_pos_type pos, \
-+ enum list_lock_type lock) { \
-+ struct list_head *lh = stripe->lists + (list); \
-+ struct stripe_cache *sc = stripe->sc; \
-+ spinlock_t *l = NULL; \
-+\
-+ if (lock == LIST_LOCKED) { \
-+ l = sc->locks + LOCK_LRU; \
-+ spin_lock_irq(l); \
-+ } \
-+\
-+ if (list_empty(lh)) { \
-+ if (pos == POS_HEAD) \
-+ list_add(lh, sc->lists + (list)); \
-+ else \
-+ list_add_tail(lh, sc->lists + (list)); \
-+ } \
-+\
-+ if (lock == LIST_LOCKED) \
-+ spin_unlock_irq(l); \
-+}
-+
-+LIST_ADD(endio, LIST_ENDIO)
-+LIST_ADD(io, LIST_IO)
-+LIST_ADD(lru, LIST_LRU)
-+#undef LIST_ADD
-+
-+#define POP(list) \
-+ do { \
-+ if (list_empty(sc->lists + list)) \
-+ stripe = NULL; \
-+ else { \
-+ stripe = list_first_entry(&sc->lists[list], \
-+ struct stripe, \
-+ lists[list]); \
-+ list_del_init(&stripe->lists[list]); \
-+ } \
-+ } while (0);
-+
-+/* Pop an available stripe off the lru list. */
-+static struct stripe *stripe_lru_pop(struct stripe_cache *sc)
-+{
-+ struct stripe *stripe;
-+ spinlock_t *lock = sc->locks + LOCK_LRU;
-+
-+ spin_lock_irq(lock);
-+ POP(LIST_LRU);
-+ spin_unlock_irq(lock);
-+
-+ if (stripe)
-+ /* Remove from hash before reuse. */
-+ stripe_hash_del(stripe, LIST_UNLOCKED);
-+
-+ return stripe;
-+}
-+
-+static inline unsigned hash_fn(struct stripe_hash *hash, sector_t key)
-+{
-+ return (unsigned) (((key * hash->prime) >> hash->shift) & hash->mask);
-+}
-+
-+static inline struct list_head *
-+hash_bucket(struct stripe_hash *hash, sector_t key)
-+{
-+ return hash->hash + hash_fn(hash, key);
-+}
-+
-+/* Insert an entry into a hash. */
-+static inline void hash_insert(struct stripe_hash *hash, struct stripe *stripe)
-+{
-+ list_add(stripe->lists + LIST_HASH, hash_bucket(hash, stripe->key));
-+}
-+
-+/* Insert an entry into the stripe hash. */
-+static inline void
-+sc_insert(struct stripe_cache *sc, struct stripe *stripe)
-+{
-+ hash_insert(&sc->hash, stripe);
-+}
-+
-+/* Lookup an entry in the stripe hash. */
-+static inline struct stripe *
-+stripe_lookup(struct stripe_cache *sc, sector_t key)
-+{
-+ unsigned c = 0;
-+ struct stripe *stripe;
-+ struct list_head *bucket = hash_bucket(&sc->hash, key);
-+
-+ list_for_each_entry(stripe, bucket, lists[LIST_HASH]) {
-+ /* REMOVEME: statisics. */
-+ if (++c > atomic_read(RS(sc)->stats + S_MAX_LOOKUP))
-+ atomic_set(RS(sc)->stats + S_MAX_LOOKUP, c);
-+
-+ if (stripe->key == key)
-+ return stripe;
-+ }
-+
-+ return NULL;
-+}
-+
-+/* Resize the stripe cache hash on size changes. */
-+static int hash_resize(struct stripe_cache *sc)
-+{
-+ /* Resize threshold reached? */
-+ if (atomic_read(&sc->stripes) > 2 * atomic_read(&sc->stripes_last)
-+ || atomic_read(&sc->stripes) < atomic_read(&sc->stripes_last) / 4) {
-+ int r;
-+ struct stripe_hash hash, hash_tmp;
-+ spinlock_t *lock;
-+
-+ r = hash_init(&hash, atomic_read(&sc->stripes));
-+ if (r)
-+ return r;
-+
-+ lock = sc->locks + LOCK_LRU;
-+ spin_lock_irq(lock);
-+ if (sc->hash.hash) {
-+ unsigned b = sc->hash.buckets;
-+ struct list_head *pos, *tmp;
-+
-+ /* Walk old buckets and insert into new. */
-+ while (b--) {
-+ list_for_each_safe(pos, tmp, sc->hash.hash + b)
-+ hash_insert(&hash,
-+ list_entry(pos, struct stripe,
-+ lists[LIST_HASH]));
-+ }
-+
-+ }
-+
-+ memcpy(&hash_tmp, &sc->hash, sizeof(hash_tmp));
-+ memcpy(&sc->hash, &hash, sizeof(sc->hash));
-+ atomic_set(&sc->stripes_last, atomic_read(&sc->stripes));
-+ spin_unlock_irq(lock);
-+
-+ hash_exit(&hash_tmp);
-+ }
-+
-+ return 0;
-+}
-+
-+/*
-+ * Stripe cache locking functions
-+ */
-+/* Dummy lock function for local RAID4+5. */
-+static void *no_lock(sector_t key, enum dm_lock_type type)
-+{
-+ return &no_lock;
-+}
-+
-+/* Dummy unlock function for local RAID4+5. */
-+static void no_unlock(void *lock_handle)
-+{
-+}
-+
-+/* No locking (for local RAID 4+5). */
-+static struct dm_raid45_locking_type locking_none = {
-+ .lock = no_lock,
-+ .unlock = no_unlock,
-+};
-+
-+/* Clustered RAID 4+5. */
-+/* FIXME: code this. */
-+static struct dm_raid45_locking_type locking_cluster = {
-+ .lock = no_lock,
-+ .unlock = no_unlock,
-+};
-+
-+/* Lock a stripe (for clustering). */
-+static int
-+stripe_lock(struct raid_set *rs, struct stripe *stripe, int rw, sector_t key)
-+{
-+ stripe->lock = rs->locking->lock(key, rw == READ ? DM_RAID45_SHARED :
-+ DM_RAID45_EX);
-+ return stripe->lock ? 0 : -EPERM;
-+}
-+
-+/* Unlock a stripe (for clustering). */
-+static void stripe_unlock(struct raid_set *rs, struct stripe *stripe)
-+{
-+ rs->locking->unlock(stripe->lock);
-+ stripe->lock = NULL;
-+}
-+
-+/*
-+ * Stripe cache functions.
-+ */
-+/*
-+ * Invalidate all page lists pages of a stripe.
-+ *
-+ * I only keep state for the whole list in the first page.
-+ */
-+static INLINE void
-+stripe_pages_invalidate(struct stripe *stripe)
-+{
-+ unsigned p = RS(stripe->sc)->set.raid_devs;
-+
-+ while (p--) {
-+ struct page *page = PAGE(stripe, p);
-+
-+ ProhibitPageIO(page);
-+ ClearPageChecked(page);
-+ ClearPageDirty(page);
-+ ClearPageError(page);
-+ clear_page_locked(page);
-+ ClearPagePrivate(page);
-+ ClearPageUptodate(page);
-+ }
-+}
-+
-+/* Prepare stripe for (re)use. */
-+static INLINE void stripe_invalidate(struct stripe *stripe)
-+{
-+ stripe->io.flags = 0;
-+ stripe_pages_invalidate(stripe);
-+}
-+
-+/* Allow io on all chunks of a stripe. */
-+static INLINE void stripe_allow_io(struct stripe *stripe)
-+{
-+ unsigned p = RS(stripe->sc)->set.raid_devs;
-+
-+ while (p--)
-+ AllowPageIO(PAGE(stripe, p));
-+}
-+
-+/* Initialize a stripe. */
-+static void
-+stripe_init(struct stripe_cache *sc, struct stripe *stripe)
-+{
-+ unsigned p = RS(sc)->set.raid_devs;
-+ unsigned i;
-+
-+ /* Work all io chunks. */
-+ while (p--) {
-+ struct stripe_set *ss = stripe->ss + p;
-+
-+ stripe->obj[p].private = ss;
-+ ss->stripe = stripe;
-+
-+ i = ARRAY_SIZE(ss->bl);
-+ while (i--)
-+ bio_list_init(ss->bl + i);
-+ }
-+
-+ stripe->sc = sc;
-+
-+ i = ARRAY_SIZE(stripe->lists);
-+ while (i--)
-+ INIT_LIST_HEAD(stripe->lists + i);
-+
-+ atomic_set(&stripe->cnt, 0);
-+ atomic_set(&stripe->io.pending, 0);
-+
-+ stripe_invalidate(stripe);
-+}
-+
-+/* Number of pages per chunk. */
-+static inline unsigned chunk_pages(unsigned io_size)
-+{
-+ return dm_div_up(io_size, SECTORS_PER_PAGE);
-+}
-+
-+/* Number of pages per stripe. */
-+static inline unsigned stripe_pages(struct raid_set *rs, unsigned io_size)
-+{
-+ return chunk_pages(io_size) * rs->set.raid_devs;
-+}
-+
-+/* Initialize part of page_list (recovery). */
-+static INLINE void stripe_zero_pl_part(struct stripe *stripe, unsigned p,
-+ unsigned start, unsigned count)
-+{
-+ unsigned pages = chunk_pages(count);
-+ /* Get offset into the page_list. */
-+ struct page_list *pl = pl_elem(PL(stripe, p), start / SECTORS_PER_PAGE);
-+
-+ BUG_ON(!pl);
-+ while (pl && pages--) {
-+ BUG_ON(!pl->page);
-+ memset(page_address(pl->page), 0, PAGE_SIZE);
-+ pl = pl->next;
-+ }
-+}
-+
-+/* Initialize parity chunk of stripe. */
-+static INLINE void stripe_zero_chunk(struct stripe *stripe, unsigned p)
-+{
-+ stripe_zero_pl_part(stripe, p, 0, stripe->io.size);
-+}
-+
-+/* Return dynamic stripe structure size. */
-+static INLINE size_t stripe_size(struct raid_set *rs)
-+{
-+ return sizeof(struct stripe) +
-+ rs->set.raid_devs * sizeof(struct stripe_set);
-+}
-+
-+/* Allocate a stripe and its memory object. */
-+/* XXX adjust to cope with stripe cache and recovery stripe caches. */
-+enum grow { SC_GROW, SC_KEEP };
-+static struct stripe *stripe_alloc(struct stripe_cache *sc,
-+ struct dm_mem_cache_client *mc,
-+ enum grow grow)
-+{
-+ int r;
-+ struct stripe *stripe;
-+
-+ stripe = kmem_cache_zalloc(sc->kc.cache, GFP_KERNEL);
-+ if (stripe) {
-+ /* Grow the dm-mem-cache by one object. */
-+ if (grow == SC_GROW) {
-+ r = dm_mem_cache_grow(mc, 1);
-+ if (r)
-+ goto err_free;
-+ }
-+
-+ stripe->obj = dm_mem_cache_alloc(mc);
-+ if (!stripe->obj)
-+ goto err_shrink;
-+
-+ stripe_init(sc, stripe);
-+ }
-+
-+ return stripe;
-+
-+err_shrink:
-+ if (grow == SC_GROW)
-+ dm_mem_cache_shrink(mc, 1);
-+err_free:
-+ kmem_cache_free(sc->kc.cache, stripe);
-+ return NULL;
-+}
-+
-+/*
-+ * Free a stripes memory object, shrink the
-+ * memory cache and free the stripe itself
-+ */
-+static void stripe_free(struct stripe *stripe, struct dm_mem_cache_client *mc)
-+{
-+ dm_mem_cache_free(mc, stripe->obj);
-+ dm_mem_cache_shrink(mc, 1);
-+ kmem_cache_free(stripe->sc->kc.cache, stripe);
-+}
-+
-+/* Free the recovery stripe. */
-+static void stripe_recover_free(struct raid_set *rs)
-+{
-+ struct recover *rec = &rs->recover;
-+ struct list_head *stripes = &rec->stripes;
-+
-+ while (!list_empty(stripes)) {
-+ struct stripe *stripe = list_first_entry(stripes, struct stripe,
-+ lists[LIST_RECOVER]);
-+ list_del(stripe->lists + LIST_RECOVER);
-+ stripe_free(stripe, rec->mem_cache_client);
-+ }
-+}
-+
-+/* Push a stripe safely onto the endio list to be handled by do_endios(). */
-+static INLINE void stripe_endio_push(struct stripe *stripe)
-+{
-+ int wake;
-+ unsigned long flags;
-+ struct stripe_cache *sc = stripe->sc;
-+ spinlock_t *lock = sc->locks + LOCK_ENDIO;
-+
-+ spin_lock_irqsave(lock, flags);
-+ wake = list_empty(sc->lists + LIST_ENDIO);
-+ stripe_endio_add(stripe, POS_HEAD, LIST_UNLOCKED);
-+ spin_unlock_irqrestore(lock, flags);
-+
-+ if (wake)
-+ wake_do_raid(RS(sc));
-+}
-+
-+/* Protected check for stripe cache endio list empty. */
-+static INLINE int stripe_endio_empty(struct stripe_cache *sc)
-+{
-+ int r;
-+ spinlock_t *lock = sc->locks + LOCK_ENDIO;
-+
-+ spin_lock_irq(lock);
-+ r = list_empty(sc->lists + LIST_ENDIO);
-+ spin_unlock_irq(lock);
-+
-+ return r;
-+}
-+
-+/* Pop a stripe off safely off the endio list. */
-+static struct stripe *stripe_endio_pop(struct stripe_cache *sc)
-+{
-+ struct stripe *stripe;
-+ spinlock_t *lock = sc->locks + LOCK_ENDIO;
-+
-+ /* This runs in parallel with endio(). */
-+ spin_lock_irq(lock);
-+ POP(LIST_ENDIO)
-+ spin_unlock_irq(lock);
-+ return stripe;
-+}
-+
-+#undef POP
-+
-+/* Evict stripe from cache. */
-+static void stripe_evict(struct stripe *stripe)
-+{
-+ struct raid_set *rs = RS(stripe->sc);
-+ stripe_hash_del(stripe, LIST_UNLOCKED); /* Take off hash. */
-+
-+ if (list_empty(stripe->lists + LIST_LRU)) {
-+ stripe_lru_add(stripe, POS_TAIL, LIST_LOCKED);
-+ atomic_inc(rs->stats + S_EVICT); /* REMOVEME: statistics. */
-+ }
-+}
-+
-+/* Grow stripe cache. */
-+static int
-+sc_grow(struct stripe_cache *sc, unsigned stripes, enum grow grow)
-+{
-+ int r = 0;
-+ struct raid_set *rs = RS(sc);
-+
-+ /* Try to allocate this many (additional) stripes. */
-+ while (stripes--) {
-+ struct stripe *stripe =
-+ stripe_alloc(sc, sc->mem_cache_client, grow);
-+
-+ if (likely(stripe)) {
-+ stripe->io.size = rs->set.io_size;
-+ stripe_lru_add(stripe, POS_TAIL, LIST_LOCKED);
-+ atomic_inc(&sc->stripes);
-+ } else {
-+ r = -ENOMEM;
-+ break;
-+ }
-+ }
-+
-+ ClearRSScBusy(rs);
-+ return r ? r : hash_resize(sc);
-+}
-+
-+/* Shrink stripe cache. */
-+static int sc_shrink(struct stripe_cache *sc, unsigned stripes)
-+{
-+ int r = 0;
-+
-+ /* Try to get unused stripe from LRU list. */
-+ while (stripes--) {
-+ struct stripe *stripe;
-+
-+ stripe = stripe_lru_pop(sc);
-+ if (stripe) {
-+ /* An lru stripe may never have ios pending! */
-+ BUG_ON(stripe_io(stripe));
-+ stripe_free(stripe, sc->mem_cache_client);
-+ atomic_dec(&sc->stripes);
-+ } else {
-+ r = -ENOENT;
-+ break;
-+ }
-+ }
-+
-+ /* Check if stats are still sane. */
-+ if (atomic_read(&sc->max_active_stripes) >
-+ atomic_read(&sc->stripes))
-+ atomic_set(&sc->max_active_stripes, 0);
-+
-+ if (r)
-+ return r;
-+
-+ ClearRSScBusy(RS(sc));
-+ return hash_resize(sc);
-+}
-+
-+/* Create stripe cache. */
-+static int sc_init(struct raid_set *rs, unsigned stripes)
-+{
-+ unsigned i, nr;
-+ struct stripe_cache *sc = &rs->sc;
-+ struct stripe *stripe;
-+ struct recover *rec = &rs->recover;
-+
-+ /* Initialize lists and locks. */
-+ i = ARRAY_SIZE(sc->lists);
-+ while (i--)
-+ INIT_LIST_HEAD(sc->lists + i);
-+
-+ i = NR_LOCKS;
-+ while (i--)
-+ spin_lock_init(sc->locks + i);
-+
-+ /* Initialize atomic variables. */
-+ atomic_set(&sc->stripes, 0);
-+ atomic_set(&sc->stripes_last, 0);
-+ atomic_set(&sc->stripes_to_shrink, 0);
-+ atomic_set(&sc->active_stripes, 0);
-+ atomic_set(&sc->max_active_stripes, 0); /* REMOVEME: statistics. */
-+
-+ /*
-+ * We need a runtime unique # to suffix the kmem cache name
-+ * because we'll have one for each active RAID set.
-+ */
-+ nr = atomic_inc_return(&_stripe_sc_nr);
-+ sprintf(sc->kc.name, "%s_%d", TARGET, nr);
-+ sc->kc.cache = kmem_cache_create(sc->kc.name, stripe_size(rs),
-+ 0, 0, NULL);
-+ if (!sc->kc.cache)
-+ return -ENOMEM;
-+
-+ /* Create memory cache client context for RAID stripe cache. */
-+ sc->mem_cache_client =
-+ dm_mem_cache_client_create(stripes, rs->set.raid_devs,
-+ chunk_pages(rs->set.io_size));
-+ if (IS_ERR(sc->mem_cache_client))
-+ return PTR_ERR(sc->mem_cache_client);
-+
-+ /* Create memory cache client context for RAID recovery stripe(s). */
-+ rec->mem_cache_client =
-+ dm_mem_cache_client_create(MAX_RECOVER, rs->set.raid_devs,
-+ chunk_pages(rec->io_size));
-+ if (IS_ERR(rec->mem_cache_client))
-+ return PTR_ERR(rec->mem_cache_client);
-+
-+ /* Allocate stripe for set recovery. */
-+ /* XXX: cope with MAX_RECOVERY. */
-+ INIT_LIST_HEAD(&rec->stripes);
-+ for (i = 0; i < MAX_RECOVER; i++) {
-+ stripe = stripe_alloc(sc, rec->mem_cache_client, SC_KEEP);
-+ if (!stripe)
-+ return -ENOMEM;
-+
-+ SetStripeRecover(stripe);
-+ stripe->io.size = rec->io_size;
-+ list_add(stripe->lists + LIST_RECOVER, &rec->stripes);
-+ }
-+
-+ /*
-+ * Allocate the stripe objetcs from the
-+ * cache and add them to the LRU list.
-+ */
-+ return sc_grow(sc, stripes, SC_KEEP);
-+}
-+
-+/* Destroy the stripe cache. */
-+static void sc_exit(struct stripe_cache *sc)
-+{
-+ if (sc->kc.cache) {
-+ BUG_ON(sc_shrink(sc, atomic_read(&sc->stripes)));
-+ kmem_cache_destroy(sc->kc.cache);
-+ }
-+
-+ if (sc->mem_cache_client)
-+ dm_mem_cache_client_destroy(sc->mem_cache_client);
-+
-+ ClearRSRecover(RS(sc));
-+ stripe_recover_free(RS(sc));
-+ if (RS(sc)->recover.mem_cache_client)
-+ dm_mem_cache_client_destroy(RS(sc)->recover.mem_cache_client);
-+
-+ hash_exit(&sc->hash);
-+}
-+
-+/*
-+ * Calculate RAID address
-+ *
-+ * Delivers tuple with the index of the data disk holding the chunk
-+ * in the set, the parity disks index and the start of the stripe
-+ * within the address space of the set (used as the stripe cache hash key).
-+ */
-+/* thx MD. */
-+static struct address *
-+raid_address(struct raid_set *rs, sector_t sector, struct address *addr)
-+{
-+ unsigned data_devs = rs->set.data_devs, di, pi,
-+ raid_devs = rs->set.raid_devs;
-+ sector_t stripe, tmp;
-+
-+ /*
-+ * chunk_number = sector / chunk_size
-+ * stripe = chunk_number / data_devs
-+ * di = stripe % data_devs;
-+ */
-+ stripe = sector >> rs->set.chunk_shift;
-+ di = sector_div(stripe, data_devs);
-+
-+ switch (rs->set.raid_type->level) {
-+ case raid5:
-+ tmp = stripe;
-+ pi = sector_div(tmp, raid_devs);
-+
-+ switch (rs->set.raid_type->algorithm) {
-+ case left_asym: /* Left asymmetric. */
-+ pi = data_devs - pi;
-+ case right_asym: /* Right asymmetric. */
-+ if (di >= pi)
-+ di++;
-+ break;
-+
-+ case left_sym: /* Left symmetric. */
-+ pi = data_devs - pi;
-+ case right_sym: /* Right symmetric. */
-+ di = (pi + di + 1) % raid_devs;
-+ break;
-+
-+ default:
-+ DMERR("Unknown RAID algorithm %d",
-+ rs->set.raid_type->algorithm);
-+ goto out;
-+ }
-+
-+ break;
-+
-+ case raid4:
-+ pi = rs->set.pi;
-+ if (di >= pi)
-+ di++;
-+ break;
-+
-+ default:
-+ DMERR("Unknown RAID level %d", rs->set.raid_type->level);
-+ goto out;
-+ }
-+
-+ /*
-+ * Hash key = start offset on any single device of the RAID set;
-+ * adjusted in case io size differs from chunk size.
-+ */
-+ addr->key = (stripe << rs->set.chunk_shift) +
-+ (sector & rs->set.io_shift_mask);
-+ addr->di = di;
-+ addr->pi = pi;
-+
-+out:
-+ return addr;
-+}
-+
-+/*
-+ * Copy data across between stripe pages and bio vectors.
-+ *
-+ * Pay attention to data alignment in stripe and bio pages.
-+ */
-+static void
-+bio_copy_page_list(int rw, struct stripe *stripe,
-+ struct page_list *pl, struct bio *bio)
-+{
-+ unsigned i, page_offset;
-+ void *page_addr;
-+ struct raid_set *rs = RS(stripe->sc);
-+ struct bio_vec *bv;
-+
-+ /* Get start page in page list for this sector. */
-+ i = (bio->bi_sector & rs->set.io_mask) / SECTORS_PER_PAGE;
-+ pl = pl_elem(pl, i);
-+
-+ page_addr = page_address(pl->page);
-+ page_offset = to_bytes(bio->bi_sector & (SECTORS_PER_PAGE - 1));
-+
-+ /* Walk all segments and copy data across between bio_vecs and pages. */
-+ bio_for_each_segment(bv, bio, i) {
-+ int len = bv->bv_len, size;
-+ unsigned bio_offset = 0;
-+ void *bio_addr = __bio_kmap_atomic(bio, i, KM_USER0);
-+redo:
-+ size = (page_offset + len > PAGE_SIZE) ?
-+ PAGE_SIZE - page_offset : len;
-+
-+ if (rw == READ)
-+ memcpy(bio_addr + bio_offset,
-+ page_addr + page_offset, size);
-+ else
-+ memcpy(page_addr + page_offset,
-+ bio_addr + bio_offset, size);
-+
-+ page_offset += size;
-+ if (page_offset == PAGE_SIZE) {
-+ /*
-+ * We reached the end of the chunk page ->
-+ * need refer to the next one to copy more data.
-+ */
-+ len -= size;
-+ if (len) {
-+ /* Get next page. */
-+ pl = pl->next;
-+ BUG_ON(!pl);
-+ page_addr = page_address(pl->page);
-+ page_offset = 0;
-+ bio_offset += size;
-+ /* REMOVEME: statistics. */
-+ atomic_inc(rs->stats + S_BIO_COPY_PL_NEXT);
-+ goto redo;
-+ }
-+ }
-+
-+ __bio_kunmap_atomic(bio_addr, KM_USER0);
-+ }
-+}
-+
-+/*
-+ * Xor optimization macros.
-+ */
-+/* Xor data pointer declaration and initialization macros. */
-+#define DECLARE_2 unsigned long *d0 = data[0], *d1 = data[1]
-+#define DECLARE_3 DECLARE_2, *d2 = data[2]
-+#define DECLARE_4 DECLARE_3, *d3 = data[3]
-+#define DECLARE_5 DECLARE_4, *d4 = data[4]
-+#define DECLARE_6 DECLARE_5, *d5 = data[5]
-+#define DECLARE_7 DECLARE_6, *d6 = data[6]
-+#define DECLARE_8 DECLARE_7, *d7 = data[7]
-+
-+/* Xor unrole macros. */
-+#define D2(n) d0[n] = d0[n] ^ d1[n]
-+#define D3(n) D2(n) ^ d2[n]
-+#define D4(n) D3(n) ^ d3[n]
-+#define D5(n) D4(n) ^ d4[n]
-+#define D6(n) D5(n) ^ d5[n]
-+#define D7(n) D6(n) ^ d6[n]
-+#define D8(n) D7(n) ^ d7[n]
-+
-+#define X_2(macro, offset) macro(offset); macro(offset + 1);
-+#define X_4(macro, offset) X_2(macro, offset); X_2(macro, offset + 2);
-+#define X_8(macro, offset) X_4(macro, offset); X_4(macro, offset + 4);
-+#define X_16(macro, offset) X_8(macro, offset); X_8(macro, offset + 8);
-+#define X_32(macro, offset) X_16(macro, offset); X_16(macro, offset + 16);
-+#define X_64(macro, offset) X_32(macro, offset); X_32(macro, offset + 32);
-+
-+/* Define a _xor_#chunks_#xors_per_run() function. */
-+#define _XOR(chunks, xors_per_run) \
-+static void _xor ## chunks ## _ ## xors_per_run(unsigned long **data) \
-+{ \
-+ unsigned end = XOR_SIZE / sizeof(data[0]), i; \
-+ DECLARE_ ## chunks; \
-+\
-+ for (i = 0; i < end; i += xors_per_run) { \
-+ X_ ## xors_per_run(D ## chunks, i); \
-+ } \
-+}
-+
-+/* Define xor functions for 2 - 8 chunks. */
-+#define MAKE_XOR_PER_RUN(xors_per_run) \
-+ _XOR(2, xors_per_run); _XOR(3, xors_per_run); \
-+ _XOR(4, xors_per_run); _XOR(5, xors_per_run); \
-+ _XOR(6, xors_per_run); _XOR(7, xors_per_run); \
-+ _XOR(8, xors_per_run);
-+
-+MAKE_XOR_PER_RUN(8) /* Define _xor_*_8() functions. */
-+MAKE_XOR_PER_RUN(16) /* Define _xor_*_16() functions. */
-+MAKE_XOR_PER_RUN(32) /* Define _xor_*_32() functions. */
-+MAKE_XOR_PER_RUN(64) /* Define _xor_*_64() functions. */
-+
-+#define MAKE_XOR(xors_per_run) \
-+struct { \
-+ void (*f)(unsigned long **); \
-+} static xor_funcs ## xors_per_run[] = { \
-+ { NULL }, \
-+ { NULL }, \
-+ { _xor2_ ## xors_per_run }, \
-+ { _xor3_ ## xors_per_run }, \
-+ { _xor4_ ## xors_per_run }, \
-+ { _xor5_ ## xors_per_run }, \
-+ { _xor6_ ## xors_per_run }, \
-+ { _xor7_ ## xors_per_run }, \
-+ { _xor8_ ## xors_per_run }, \
-+}; \
-+\
-+static void xor_ ## xors_per_run(unsigned n, unsigned long **data) \
-+{ \
-+ /* Call respective function for amount of chunks. */ \
-+ xor_funcs ## xors_per_run[n].f(data); \
-+}
-+
-+/* Define xor_8() - xor_64 functions. */
-+MAKE_XOR(8)
-+MAKE_XOR(16)
-+MAKE_XOR(32)
-+MAKE_XOR(64)
-+
-+/* Maximum number of chunks, which can be xor'ed in one go. */
-+#define XOR_CHUNKS_MAX (ARRAY_SIZE(xor_funcs8) - 1)
-+
-+struct xor_func {
-+ xor_function_t f;
-+ const char *name;
-+} static xor_funcs[] = {
-+ {xor_8, "xor_8"},
-+ {xor_16, "xor_16"},
-+ {xor_32, "xor_32"},
-+ {xor_64, "xor_64"},
-+};
-+
-+/*
-+ * Calculate crc.
-+ *
-+ * This indexes into the page list of the stripe.
-+ *
-+ * All chunks will be xored into the parity chunk
-+ * in maximum groups of xor.chunks.
-+ *
-+ * FIXME: try mapping the pages on discontiguous memory.
-+ */
-+static void xor(struct stripe *stripe, unsigned pi, unsigned sector)
-+{
-+ struct raid_set *rs = RS(stripe->sc);
-+ unsigned max_chunks = rs->xor.chunks, n, p;
-+ unsigned o = sector / SECTORS_PER_PAGE; /* Offset into the page_list. */
-+ unsigned long **d = rs->data;
-+ xor_function_t xor_f = rs->xor.f->f;
-+
-+ /* Address of parity page to xor into. */
-+ d[0] = page_address(pl_elem(PL(stripe, pi), o)->page);
-+
-+ /* Preset pointers to data pages. */
-+ for (n = 1, p = rs->set.raid_devs; p--; ) {
-+ if (p != pi && PageIO(PAGE(stripe, p)))
-+ d[n++] = page_address(pl_elem(PL(stripe, p), o)->page);
-+
-+ /* If max chunks -> xor .*/
-+ if (n == max_chunks) {
-+ xor_f(n, d);
-+ n = 1;
-+ }
-+ }
-+
-+ /* If chunks -> xor. */
-+ if (n > 1)
-+ xor_f(n, d);
-+
-+ /* Set parity page uptodate and clean. */
-+ page_set(PAGE(stripe, pi), CLEAN);
-+}
-+
-+/* Common xor loop through all stripe page lists. */
-+static void common_xor(struct stripe *stripe, sector_t count,
-+ unsigned off, unsigned p)
-+{
-+ unsigned sector;
-+
-+ for (sector = off; sector < count; sector += SECTORS_PER_XOR)
-+ xor(stripe, p, sector);
-+
-+ atomic_inc(RS(stripe->sc)->stats + S_XORS); /* REMOVEME: statistics. */
-+}
-+
-+/*
-+ * Calculate parity sectors on intact stripes.
-+ *
-+ * Need to calculate raid address for recover stripe, because its
-+ * chunk sizes differs and is typically larger than io chunk size.
-+ */
-+static void parity_xor(struct stripe *stripe)
-+{
-+ struct raid_set *rs = RS(stripe->sc);
-+ unsigned chunk_size = rs->set.chunk_size,
-+ io_size = stripe->io.size,
-+ xor_size = chunk_size > io_size ? io_size : chunk_size;
-+ sector_t off;
-+
-+ /* This can be the recover stripe with a larger io size. */
-+ for (off = 0; off < io_size; off += xor_size) {
-+ unsigned pi;
-+
-+ /*
-+ * Recover stripe likely is bigger than regular io
-+ * ones and has no precalculated parity disk index ->
-+ * need to calculate RAID address.
-+ */
-+ if (unlikely(StripeRecover(stripe))) {
-+ struct address addr;
-+
-+ raid_address(rs,
-+ (stripe->key + off) * rs->set.data_devs,
-+ &addr);
-+ pi = addr.pi;
-+ stripe_zero_pl_part(stripe, pi, off,
-+ rs->set.chunk_size);
-+ } else
-+ pi = stripe->idx.parity;
-+
-+ common_xor(stripe, xor_size, off, pi);
-+ page_set(PAGE(stripe, pi), DIRTY);
-+ }
-+}
-+
-+/* Reconstruct missing chunk. */
-+static void reconstruct_xor(struct stripe *stripe)
-+{
-+ struct raid_set *rs = RS(stripe->sc);
-+ int p = stripe->idx.recover;
-+
-+ BUG_ON(p < 0);
-+
-+ /* REMOVEME: statistics. */
-+ atomic_inc(rs->stats + (raid_set_degraded(rs) ?
-+ S_RECONSTRUCT_EI : S_RECONSTRUCT_DEV));
-+
-+ /* Zero chunk to be reconstructed. */
-+ stripe_zero_chunk(stripe, p);
-+ common_xor(stripe, stripe->io.size, 0, p);
-+}
-+
-+/*
-+ * Try getting a stripe either from the hash or from the lru list
-+ */
-+static inline void _stripe_get(struct stripe *stripe)
-+{
-+ atomic_inc(&stripe->cnt);
-+}
-+
-+static struct stripe *stripe_get(struct raid_set *rs, struct address *addr)
-+{
-+ struct stripe_cache *sc = &rs->sc;
-+ struct stripe *stripe;
-+
-+ stripe = stripe_lookup(sc, addr->key);
-+ if (stripe) {
-+ _stripe_get(stripe);
-+ /* Remove from the lru list if on. */
-+ stripe_lru_del(stripe, LIST_LOCKED);
-+ atomic_inc(rs->stats + S_HITS_1ST); /* REMOVEME: statistics. */
-+ } else {
-+ /* Second try to get an LRU stripe. */
-+ stripe = stripe_lru_pop(sc);
-+ if (stripe) {
-+ _stripe_get(stripe);
-+ /* Invalidate before reinserting with changed key. */
-+ stripe_invalidate(stripe);
-+ stripe->key = addr->key;
-+ stripe->region = dm_rh_sector_to_region(rs->recover.rh,
-+ addr->key);
-+ stripe->idx.parity = addr->pi;
-+ sc_insert(sc, stripe);
-+ /* REMOVEME: statistics. */
-+ atomic_inc(rs->stats + S_INSCACHE);
-+ }
-+ }
-+
-+ return stripe;
-+}
-+
-+/*
-+ * Decrement reference count on a stripe.
-+ *
-+ * Move it to list of LRU stripes if zero.
-+ */
-+static void stripe_put(struct stripe *stripe)
-+{
-+ if (atomic_dec_and_test(&stripe->cnt)) {
-+ if (TestClearStripeActive(stripe))
-+ atomic_dec(&stripe->sc->active_stripes);
-+
-+ /* Put stripe onto the LRU list. */
-+ stripe_lru_add(stripe, POS_TAIL, LIST_LOCKED);
-+ }
-+
-+ BUG_ON(atomic_read(&stripe->cnt) < 0);
-+}
-+
-+/*
-+ * Process end io
-+ *
-+ * I need to do it here because I can't in interrupt
-+ *
-+ * Read and write functions are split in order to avoid
-+ * conditionals in the main loop for performamce reasons.
-+ */
-+
-+/* Helper read bios on a page list. */
-+static void _bio_copy_page_list(struct stripe *stripe, struct page_list *pl,
-+ struct bio *bio)
-+{
-+ bio_copy_page_list(READ, stripe, pl, bio);
-+}
-+
-+/* Helper write bios on a page list. */
-+static void _rh_dec(struct stripe *stripe, struct page_list *pl,
-+ struct bio *bio)
-+{
-+ dm_rh_dec(RS(stripe->sc)->recover.rh, stripe->region);
-+}
-+
-+/* End io all bios on a page list. */
-+static inline int
-+page_list_endio(int rw, struct stripe *stripe, unsigned p, unsigned *count)
-+{
-+ int r = 0;
-+ struct bio_list *bl = BL(stripe, p, rw);
-+
-+ if (!bio_list_empty(bl)) {
-+ struct page_list *pl = PL(stripe, p);
-+ struct page *page = pl->page;
-+
-+ if (PageLocked(page))
-+ r = -EBUSY;
-+ /*
-+ * FIXME: PageUptodate() not cleared
-+ * properly for missing chunks ?
-+ */
-+ else if (PageUptodate(page)) {
-+ struct bio *bio;
-+ struct raid_set *rs = RS(stripe->sc);
-+ void (*h_f)(struct stripe *, struct page_list *,
-+ struct bio *) =
-+ (rw == READ) ? _bio_copy_page_list : _rh_dec;
-+
-+ while ((bio = bio_list_pop(bl))) {
-+ h_f(stripe, pl, bio);
-+ _bio_endio(rs, bio, 0);
-+ stripe_put(stripe);
-+ if (count)
-+ (*count)++;
-+ }
-+ } else
-+ r = -EAGAIN;
-+ }
-+
-+ return r;
-+}
-+
-+/*
-+ * End io all reads/writes on a stripe copying
-+ * read date accross from stripe to bios.
-+ */
-+static int stripe_endio(int rw, struct stripe *stripe, unsigned *count)
-+{
-+ int r = 0;
-+ unsigned p = RS(stripe->sc)->set.raid_devs;
-+
-+ while (p--) {
-+ int rr = page_list_endio(rw, stripe, p, count);
-+
-+ if (rr && r != -EIO)
-+ r = rr;
-+ }
-+
-+ return r;
-+}
-+
-+/* Fail all ios on a bio list and return # of bios. */
-+static unsigned
-+bio_list_fail(struct raid_set *rs, struct stripe *stripe, struct bio_list *bl)
-+{
-+ unsigned r;
-+ struct bio *bio;
-+
-+ raid_set_dead(rs);
-+
-+ /* Update region counters. */
-+ if (stripe) {
-+ struct dm_rh_client *rh = rs->recover.rh;
-+
-+ bio_list_for_each(bio, bl) {
-+ if (bio_data_dir(bio) == WRITE)
-+ dm_rh_dec(rh, stripe->region);
-+ }
-+ }
-+
-+ /* Error end io all bios. */
-+ for (r = 0; (bio = bio_list_pop(bl)); r++)
-+ _bio_endio(rs, bio, -EIO);
-+
-+ return r;
-+}
-+
-+/* Fail all ios of a bio list of a stripe and drop io pending count. */
-+static void
-+stripe_bio_list_fail(struct raid_set *rs, struct stripe *stripe,
-+ struct bio_list *bl)
-+{
-+ unsigned put = bio_list_fail(rs, stripe, bl);
-+
-+ while (put--)
-+ stripe_put(stripe);
-+}
-+
-+/* Fail all ios hanging off all bio lists of a stripe. */
-+static void stripe_fail_io(struct stripe *stripe)
-+{
-+ struct raid_set *rs = RS(stripe->sc);
-+ unsigned p = rs->set.raid_devs;
-+
-+ stripe_evict(stripe);
-+
-+ while (p--) {
-+ struct stripe_set *ss = stripe->ss + p;
-+ int i = ARRAY_SIZE(ss->bl);
-+
-+ while (i--)
-+ stripe_bio_list_fail(rs, stripe, ss->bl + i);
-+ }
-+}
-+
-+/*
-+ * Handle all stripes by handing them to the daemon, because we can't
-+ * map their pages to copy the data in interrupt context.
-+ *
-+ * We don't want to handle them here either, while interrupts are disabled.
-+ */
-+
-+/* Read/write endio function for dm-io (interrupt context). */
-+static void endio(unsigned long error, void *context)
-+{
-+ struct dm_mem_cache_object *obj = context;
-+ struct stripe_set *ss = obj->private;
-+ struct stripe *stripe = ss->stripe;
-+ struct page *page = obj->pl->page;
-+
-+ if (unlikely(error))
-+ stripe_error(stripe, page);
-+ else
-+ page_set(page, CLEAN);
-+
-+ clear_page_locked(page);
-+ stripe_io_dec(stripe);
-+
-+ /* Add stripe to endio list and wake daemon. */
-+ stripe_endio_push(stripe);
-+}
-+
-+/*
-+ * Recovery io throttling
-+ */
-+/* Conditionally reset io counters. */
-+enum count_type { IO_WORK = 0, IO_RECOVER };
-+static int recover_io_reset(struct raid_set *rs)
-+{
-+ unsigned long j = jiffies;
-+
-+ /* Pay attention to jiffies overflows. */
-+ if (j > rs->recover.last_jiffies + HZ
-+ || j < rs->recover.last_jiffies) {
-+ rs->recover.last_jiffies = j;
-+ atomic_set(rs->recover.io_count + IO_WORK, 0);
-+ atomic_set(rs->recover.io_count + IO_RECOVER, 0);
-+ return 1;
-+ }
-+
-+ return 0;
-+}
-+
-+/* Count ios. */
-+static INLINE void
-+recover_io_count(struct raid_set *rs, struct stripe *stripe)
-+{
-+ if (RSRecover(rs)) {
-+ recover_io_reset(rs);
-+ atomic_inc(rs->recover.io_count +
-+ (StripeRecover(stripe) ? IO_RECOVER : IO_WORK));
-+ }
-+}
-+
-+/* Read/Write a page_list asynchronously. */
-+static void page_list_rw(struct stripe *stripe, unsigned p)
-+{
-+ struct stripe_cache *sc = stripe->sc;
-+ struct raid_set *rs = RS(sc);
-+ struct dm_mem_cache_object *obj = stripe->obj + p;
-+ struct page_list *pl = obj->pl;
-+ struct page *page = pl->page;
-+ struct raid_dev *dev = rs->dev + p;
-+ struct dm_io_region io = {
-+ .bdev = dev->dev->bdev,
-+ .sector = stripe->key,
-+ .count = stripe->io.size,
-+ };
-+ struct dm_io_request control = {
-+ .bi_rw = PageDirty(page) ? WRITE : READ,
-+ .mem.type = DM_IO_PAGE_LIST,
-+ .mem.ptr.pl = pl,
-+ .mem.offset = 0,
-+ .notify.fn = endio,
-+ .notify.context = obj,
-+ .client = sc->dm_io_client,
-+ };
-+
-+ BUG_ON(PageLocked(page));
-+
-+ /*
-+ * Don't rw past end of device, which can happen, because
-+ * typically sectors_per_dev isn't divisable by io_size.
-+ */
-+ if (unlikely(io.sector + io.count > rs->set.sectors_per_dev))
-+ io.count = rs->set.sectors_per_dev - io.sector;
-+
-+ io.sector += dev->start; /* Add <offset>. */
-+ recover_io_count(rs, stripe); /* Recovery io accounting. */
-+
-+ /* REMOVEME: statistics. */
-+ atomic_inc(rs->stats +
-+ (PageDirty(page) ? S_DM_IO_WRITE : S_DM_IO_READ));
-+
-+ ClearPageError(page);
-+ set_page_locked(page);
-+ io_dev_queued(dev);
-+ BUG_ON(dm_io(&control, 1, &io, NULL));
-+}
-+
-+/*
-+ * Write dirty / read not uptodate page lists of a stripe.
-+ */
-+static unsigned stripe_page_lists_rw(struct raid_set *rs, struct stripe *stripe)
-+{
-+ unsigned r;
-+
-+ /*
-+ * Increment the pending count on the stripe
-+ * first, so that we don't race in endio().
-+ *
-+ * An inc (IO) is needed for any page:
-+ *
-+ * o not uptodate
-+ * o dirtied by writes merged
-+ * o dirtied by parity calculations
-+ */
-+ r = for_each_io_dev(rs, stripe, _stripe_io_inc);
-+ if (r) {
-+ /* io needed: chunks are not uptodate/dirty. */
-+ int max; /* REMOVEME: */
-+ struct stripe_cache *sc = &rs->sc;
-+
-+ if (!TestSetStripeActive(stripe))
-+ atomic_inc(&sc->active_stripes);
-+
-+ /* Take off the lru list in case it got added there. */
-+ stripe_lru_del(stripe, LIST_LOCKED);
-+
-+ /* Submit actual io. */
-+ for_each_io_dev(rs, stripe, page_list_rw);
-+
-+ /* REMOVEME: statistics */
-+ max = sc_active(sc);
-+ if (atomic_read(&sc->max_active_stripes) < max)
-+ atomic_set(&sc->max_active_stripes, max);
-+
-+ atomic_inc(rs->stats + S_FLUSHS);
-+ /* END REMOVEME: statistics */
-+ }
-+
-+ return r;
-+}
-+
-+/* Work in all pending writes. */
-+static INLINE void _writes_merge(struct stripe *stripe, unsigned p)
-+{
-+ struct bio_list *write = BL(stripe, p, WRITE);
-+
-+ if (!bio_list_empty(write)) {
-+ struct page_list *pl = stripe->obj[p].pl;
-+ struct bio *bio;
-+ struct bio_list *write_merged = BL(stripe, p, WRITE_MERGED);
-+
-+ /*
-+ * We can play with the lists without holding a lock,
-+ * because it is just us accessing them anyway.
-+ */
-+ bio_list_for_each(bio, write)
-+ bio_copy_page_list(WRITE, stripe, pl, bio);
-+
-+ bio_list_merge(write_merged, write);
-+ bio_list_init(write);
-+ page_set(pl->page, DIRTY);
-+ }
-+}
-+
-+/* Merge in all writes hence dirtying respective pages. */
-+static INLINE void writes_merge(struct stripe *stripe)
-+{
-+ unsigned p = RS(stripe->sc)->set.raid_devs;
-+
-+ while (p--)
-+ _writes_merge(stripe, p);
-+}
-+
-+/* Check, if a chunk gets completely overwritten. */
-+static INLINE int stripe_check_overwrite(struct stripe *stripe, unsigned p)
-+{
-+ unsigned sectors = 0;
-+ struct bio *bio;
-+ struct bio_list *bl = BL(stripe, p, WRITE);
-+
-+ bio_list_for_each(bio, bl)
-+ sectors += bio_sectors(bio);
-+
-+ return sectors == RS(stripe->sc)->set.io_size;
-+}
-+
-+/*
-+ * Prepare stripe to avoid io on broken/reconstructed
-+ * drive in order to reconstruct date on endio.
-+ */
-+enum prepare_type { IO_ALLOW, IO_PROHIBIT };
-+static void stripe_prepare(struct stripe *stripe, unsigned p,
-+ enum prepare_type type)
-+{
-+ struct page *page = PAGE(stripe, p);
-+
-+ switch (type) {
-+ case IO_PROHIBIT:
-+ /*
-+ * In case we prohibit, we gotta make sure, that
-+ * io on all other chunks than the one which failed
-+ * or is being reconstructed is allowed and that it
-+ * doesn't have state uptodate.
-+ */
-+ stripe_allow_io(stripe);
-+ ClearPageUptodate(page);
-+ ProhibitPageIO(page);
-+
-+ /* REMOVEME: statistics. */
-+ atomic_inc(RS(stripe->sc)->stats + S_PROHIBITPAGEIO);
-+ stripe->idx.recover = p;
-+ SetStripeReconstruct(stripe);
-+ break;
-+
-+ case IO_ALLOW:
-+ AllowPageIO(page);
-+ stripe->idx.recover = -1;
-+ ClearStripeReconstruct(stripe);
-+ break;
-+
-+ default:
-+ BUG();
-+ }
-+}
-+
-+/*
-+ * Degraded/reconstruction mode.
-+ *
-+ * Check stripe state to figure which chunks don't need IO.
-+ */
-+static INLINE void stripe_check_reconstruct(struct stripe *stripe,
-+ int prohibited)
-+{
-+ struct raid_set *rs = RS(stripe->sc);
-+
-+ /*
-+ * Degraded mode (device(s) failed) ->
-+ * avoid io on the failed device.
-+ */
-+ if (unlikely(raid_set_degraded(rs))) {
-+ /* REMOVEME: statistics. */
-+ atomic_inc(rs->stats + S_DEGRADED);
-+ stripe_prepare(stripe, rs->set.ei, IO_PROHIBIT);
-+ return;
-+ } else {
-+ /*
-+ * Reconstruction mode (ie. a particular device or
-+ * some (rotating) parity chunk is being resynchronized) ->
-+ * o make sure all needed pages are read in
-+ * o writes are allowed to go through
-+ */
-+ int r = region_state(rs, stripe->key, DM_RH_NOSYNC);
-+
-+ if (r) {
-+ /* REMOVEME: statistics. */
-+ atomic_inc(rs->stats + S_NOSYNC);
-+ stripe_prepare(stripe, dev_for_parity(stripe),
-+ IO_PROHIBIT);
-+ return;
-+ }
-+ }
-+
-+ /*
-+ * All disks good. Avoid reading parity chunk and reconstruct it
-+ * unless we have prohibited io to chunk(s).
-+ */
-+ if (!prohibited) {
-+ if (StripeMerged(stripe))
-+ stripe_prepare(stripe, stripe->idx.parity, IO_ALLOW);
-+ else {
-+ stripe_prepare(stripe, stripe->idx.parity, IO_PROHIBIT);
-+
-+ /*
-+ * Overrule stripe_prepare to reconstruct the
-+ * parity chunk, because it'll be created new anyway.
-+ */
-+ ClearStripeReconstruct(stripe);
-+ }
-+ }
-+}
-+
-+/* Check, if stripe is ready to merge writes. */
-+static INLINE int stripe_check_merge(struct stripe *stripe)
-+{
-+ struct raid_set *rs = RS(stripe->sc);
-+ int prohibited = 0;
-+ unsigned chunks = 0, p = rs->set.raid_devs;
-+
-+ /* Walk all chunks. */
-+ while (p--) {
-+ struct page *page = PAGE(stripe, p);
-+
-+ /* Can't merge active chunks. */
-+ if (PageLocked(page)) {
-+ /* REMOVEME: statistics. */
-+ atomic_inc(rs->stats + S_MERGE_PAGE_LOCKED);
-+ break;
-+ }
-+
-+ /* Can merge uptodate chunks and have to count parity chunk. */
-+ if (PageUptodate(page) || p == stripe->idx.parity) {
-+ chunks++;
-+ continue;
-+ }
-+
-+ /* Read before write ordering. */
-+ if (RSCheckOverwrite(rs) &&
-+ bio_list_empty(BL(stripe, p, READ))) {
-+ int r = stripe_check_overwrite(stripe, p);
-+
-+ if (r) {
-+ chunks++;
-+ /* REMOVEME: statistics. */
-+ atomic_inc(RS(stripe->sc)->stats +
-+ S_PROHIBITPAGEIO);
-+ ProhibitPageIO(page);
-+ prohibited = 1;
-+ }
-+ }
-+ }
-+
-+ if (chunks == rs->set.raid_devs) {
-+ /* All pages are uptodate or get written over or mixture. */
-+ /* REMOVEME: statistics. */
-+ atomic_inc(rs->stats + S_CAN_MERGE);
-+ return 0;
-+ } else
-+ /* REMOVEME: statistics.*/
-+ atomic_inc(rs->stats + S_CANT_MERGE);
-+
-+ return prohibited ? 1 : -EPERM;
-+}
-+
-+/* Check, if stripe is ready to merge writes. */
-+static INLINE int stripe_check_read(struct stripe *stripe)
-+{
-+ int r = 0;
-+ unsigned p = RS(stripe->sc)->set.raid_devs;
-+
-+ /* Walk all chunks. */
-+ while (p--) {
-+ struct page *page = PAGE(stripe, p);
-+
-+ if (!PageLocked(page) &&
-+ bio_list_empty(BL(stripe, p, READ))) {
-+ ProhibitPageIO(page);
-+ r = 1;
-+ }
-+ }
-+
-+ return r;
-+}
-+
-+/*
-+ * Read/write a stripe.
-+ *
-+ * All stripe read/write activity goes through this function.
-+ *
-+ * States to cover:
-+ * o stripe to read and/or write
-+ * o stripe with error to reconstruct
-+ */
-+static int stripe_rw(struct stripe *stripe)
-+{
-+ struct raid_set *rs = RS(stripe->sc);
-+ int prohibited = 0, r;
-+
-+ /*
-+ * Check the state of the RAID set and if degraded (or
-+ * resynchronizing for reads), read in all other chunks but
-+ * the one on the dead/resynchronizing device in order to be
-+ * able to reconstruct the missing one.
-+ *
-+ * Merge all writes hanging off uptodate pages of the stripe.
-+ */
-+
-+ /* Initially allow io on all chunks and prohibit below, if necessary. */
-+ stripe_allow_io(stripe);
-+
-+ if (StripeRBW(stripe)) {
-+ r = stripe_check_merge(stripe);
-+ if (!r) {
-+ /*
-+ * If I could rely on valid parity (which would only
-+ * be sure in case of a full synchronization),
-+ * I could xor a fraction of chunks out of
-+ * parity and back in.
-+ *
-+ * For the time being, I got to redo parity...
-+ */
-+ /* parity_xor(stripe); */ /* Xor chunks out. */
-+ stripe_zero_chunk(stripe, stripe->idx.parity);
-+ writes_merge(stripe); /* Merge writes in. */
-+ parity_xor(stripe); /* Update parity. */
-+ ClearStripeRBW(stripe); /* Disable RBW. */
-+ SetStripeMerged(stripe); /* Writes merged. */
-+ }
-+
-+ if (r > 0)
-+ prohibited = 1;
-+ } else if (!raid_set_degraded(rs))
-+ /* Only allow for read avoidance if not degraded. */
-+ prohibited = stripe_check_read(stripe);
-+
-+ /*
-+ * Check, if io needs to be allowed/prohibeted on certain chunks
-+ * because of a degraded set or reconstruction on a region.
-+ */
-+ stripe_check_reconstruct(stripe, prohibited);
-+
-+ /* Now submit any reads/writes. */
-+ r = stripe_page_lists_rw(rs, stripe);
-+ if (!r) {
-+ /*
-+ * No io submitted because of chunk io prohibited or
-+ * locked pages -> push to end io list for processing.
-+ */
-+ atomic_inc(rs->stats + S_NO_RW); /* REMOVEME: statistics. */
-+ stripe_endio_push(stripe);
-+ wake_do_raid(rs); /* Wake myself. */
-+ }
-+
-+ return 0;
-+}
-+
-+/* Flush stripe either via flush list or imeediately. */
-+enum flush_type { FLUSH_DELAY, FLUSH_NOW };
-+static int stripe_flush(struct stripe *stripe, enum flush_type type)
-+{
-+ int r = 0;
-+
-+ stripe_lru_del(stripe, LIST_LOCKED);
-+
-+ /* Immediately flush. */
-+ if (type == FLUSH_NOW) {
-+ if (likely(raid_set_operational(RS(stripe->sc))))
-+ r = stripe_rw(stripe); /* Read/write stripe. */
-+ else
-+ /* Optimization: Fail early on failed sets. */
-+ stripe_fail_io(stripe);
-+ /* Delay flush by putting it on io list for later processing. */
-+ } else if (type == FLUSH_DELAY)
-+ stripe_io_add(stripe, POS_TAIL, LIST_UNLOCKED);
-+ else
-+ BUG();
-+
-+ return r;
-+}
-+
-+/*
-+ * Queue reads and writes to a stripe by hanging
-+ * their bios off the stripsets read/write lists.
-+ *
-+ * Endio reads on uptodate chunks.
-+ */
-+static INLINE int stripe_queue_bio(struct raid_set *rs, struct bio *bio,
-+ struct bio_list *reject)
-+{
-+ int r = 0;
-+ struct address addr;
-+ struct stripe *stripe =
-+ stripe_get(rs, raid_address(rs, bio->bi_sector, &addr));
-+
-+ if (stripe) {
-+ int rr, rw = bio_data_dir(bio);
-+
-+ rr = stripe_lock(rs, stripe, rw, addr.key); /* Lock stripe */
-+ if (rr) {
-+ stripe_put(stripe);
-+ goto out;
-+ }
-+
-+ /* Distinguish read and write cases. */
-+ bio_list_add(BL(stripe, addr.di, rw), bio);
-+
-+ /* REMOVEME: statistics */
-+ atomic_inc(rs->stats + (rw == WRITE ?
-+ S_BIOS_ADDED_WRITE : S_BIOS_ADDED_READ));
-+
-+ if (rw == READ)
-+ SetStripeRead(stripe);
-+ else {
-+ SetStripeRBW(stripe);
-+
-+ /* Inrement pending write count on region. */
-+ dm_rh_inc(rs->recover.rh, stripe->region);
-+ r = 1; /* Region hash needs a flush. */
-+ }
-+
-+ /*
-+ * Optimize stripe flushing:
-+ *
-+ * o directly start io for read stripes.
-+ *
-+ * o put stripe onto stripe caches io_list for RBW,
-+ * so that do_flush() can belabour it after we put
-+ * more bios to the stripe for overwrite optimization.
-+ */
-+ stripe_flush(stripe,
-+ StripeRead(stripe) ? FLUSH_NOW : FLUSH_DELAY);
-+
-+ /* Got no stripe from cache -> reject bio. */
-+ } else {
-+out:
-+ bio_list_add(reject, bio);
-+ /* REMOVEME: statistics. */
-+ atomic_inc(rs->stats + S_IOS_POST);
-+ }
-+
-+ return r;
-+}
-+
-+/*
-+ * Recovery functions
-+ */
-+/* Read a stripe off a raid set for recovery. */
-+static int recover_read(struct raid_set *rs, struct stripe *stripe, int idx)
-+{
-+ /* Invalidate all pages so that they get read in. */
-+ stripe_pages_invalidate(stripe);
-+
-+ /* Allow io on all recovery chunks. */
-+ stripe_allow_io(stripe);
-+
-+ if (idx > -1)
-+ ProhibitPageIO(PAGE(stripe, idx));
-+
-+ stripe->key = rs->recover.pos;
-+ return stripe_page_lists_rw(rs, stripe);
-+}
-+
-+/* Write a stripe to a raid set for recovery. */
-+static int recover_write(struct raid_set *rs, struct stripe *stripe, int idx)
-+{
-+ /*
-+ * If this is a reconstruct of a particular device, then
-+ * reconstruct the respective page(s), else create parity page(s).
-+ */
-+ if (idx > -1) {
-+ struct page *page = PAGE(stripe, idx);
-+
-+ AllowPageIO(page);
-+ stripe_zero_chunk(stripe, idx);
-+ common_xor(stripe, stripe->io.size, 0, idx);
-+ page_set(page, DIRTY);
-+ } else
-+ parity_xor(stripe);
-+
-+ return stripe_page_lists_rw(rs, stripe);
-+}
-+
-+/* Recover bandwidth available ?. */
-+static int recover_bandwidth(struct raid_set *rs)
-+{
-+ int r, work;
-+
-+ /* On reset -> allow recovery. */
-+ r = recover_io_reset(rs);
-+ if (r || RSBandwidth(rs))
-+ goto out;
-+
-+ work = atomic_read(rs->recover.io_count + IO_WORK);
-+ if (work) {
-+ /* Pay attention to larger recover stripe size. */
-+ int recover =
-+ atomic_read(rs->recover.io_count + IO_RECOVER) *
-+ rs->recover.io_size /
-+ rs->set.io_size;
-+
-+ /*
-+ * Don't use more than given bandwidth of
-+ * the work io for recovery.
-+ */
-+ if (recover > work / rs->recover.bandwidth_work) {
-+ /* REMOVEME: statistics. */
-+ atomic_inc(rs->stats + S_NO_BANDWIDTH);
-+ return 0;
-+ }
-+ }
-+
-+out:
-+ atomic_inc(rs->stats + S_BANDWIDTH); /* REMOVEME: statistics. */
-+ return 1;
-+}
-+
-+/* Try to get a region to recover. */
-+static int recover_get_region(struct raid_set *rs)
-+{
-+ struct recover *rec = &rs->recover;
-+ struct dm_rh_client *rh = rec->rh;
-+
-+ /* Start quiescing some regions. */
-+ if (!RSRegionGet(rs)) {
-+ int r = recover_bandwidth(rs); /* Enough bandwidth ?. */
-+
-+ if (r) {
-+ r = dm_rh_recovery_prepare(rh);
-+ if (r < 0) {
-+ DMINFO("No %sregions to recover",
-+ rec->nr_regions_to_recover ?
-+ "more " : "");
-+ return -ENOENT;
-+ }
-+ } else
-+ return -EAGAIN;
-+
-+ SetRSRegionGet(rs);
-+ }
-+
-+ if (!rec->reg) {
-+ rec->reg = dm_rh_recovery_start(rh);
-+ if (rec->reg) {
-+ /*
-+ * A reference for the the region I'll
-+ * keep till I've completely synced it.
-+ */
-+ io_get(rs);
-+ rec->pos = dm_rh_region_to_sector(rh,
-+ dm_rh_get_region_key(rec->reg));
-+ rec->end = rec->pos + dm_rh_get_region_size(rh);
-+ return 1;
-+ } else
-+ return -EAGAIN;
-+ }
-+
-+ return 0;
-+}
-+
-+/* Read/write a recovery stripe. */
-+static INLINE int recover_stripe_rw(struct raid_set *rs, struct stripe *stripe)
-+{
-+ /* Read/write flip-flop. */
-+ if (TestClearStripeRBW(stripe)) {
-+ SetStripeRead(stripe);
-+ return recover_read(rs, stripe, idx_get(rs));
-+ } else if (TestClearStripeRead(stripe))
-+ return recover_write(rs, stripe, idx_get(rs));
-+
-+ return 0;
-+}
-+
-+/* Reset recovery variables. */
-+static void recovery_region_reset(struct raid_set *rs)
-+{
-+ rs->recover.reg = NULL;
-+ ClearRSRegionGet(rs);
-+}
-+
-+/* Update region hash state. */
-+static void recover_rh_update(struct raid_set *rs, int error)
-+{
-+ struct recover *rec = &rs->recover;
-+ struct dm_rh_client *rh = rec->rh;
-+ struct dm_region *reg = rec->reg;
-+
-+ if (reg) {
-+ dm_rh_recovery_end(rh, reg, error);
-+ if (!error)
-+ rec->nr_regions_recovered++;
-+
-+ recovery_region_reset(rs);
-+ }
-+
-+ dm_rh_update_states(rh, 1);
-+ dm_rh_flush(rh);
-+ io_put(rs); /* Release the io reference for the region. */
-+}
-+
-+/* Called by main io daemon to recover regions. */
-+/* FIXME: cope with MAX_RECOVER > 1. */
-+static INLINE void _do_recovery(struct raid_set *rs, struct stripe *stripe)
-+{
-+ int r;
-+ struct recover *rec = &rs->recover;
-+
-+ /* If recovery is active -> return. */
-+ if (StripeActive(stripe))
-+ return;
-+
-+ /* io error is fatal for recovery -> stop it. */
-+ if (unlikely(StripeError(stripe)))
-+ goto err;
-+
-+ /* Get a region to recover. */
-+ r = recover_get_region(rs);
-+ switch (r) {
-+ case 1: /* Got a new region. */
-+ /* Flag read before write. */
-+ ClearStripeRead(stripe);
-+ SetStripeRBW(stripe);
-+ break;
-+
-+ case 0:
-+ /* Got a region in the works. */
-+ r = recover_bandwidth(rs);
-+ if (r) /* Got enough bandwidth. */
-+ break;
-+
-+ case -EAGAIN:
-+ /* No bandwidth/quiesced region yet, try later. */
-+ wake_do_raid_delayed(rs, HZ / 10);
-+ return;
-+
-+ case -ENOENT: /* No more regions. */
-+ dm_table_event(rs->ti->table);
-+ goto free;
-+ }
-+
-+ /* Read/write a recover stripe. */
-+ r = recover_stripe_rw(rs, stripe);
-+ if (r) {
-+ /* IO initiated, get another reference for the IO. */
-+ io_get(rs);
-+ return;
-+ }
-+
-+ /* Update recovery position within region. */
-+ rec->pos += stripe->io.size;
-+
-+ /* If we're at end of region, update region hash. */
-+ if (rec->pos >= rec->end ||
-+ rec->pos >= rs->set.sectors_per_dev)
-+ recover_rh_update(rs, 0);
-+ else
-+ SetStripeRBW(stripe);
-+
-+ /* Schedule myself for another round... */
-+ wake_do_raid(rs);
-+ return;
-+
-+err:
-+ raid_set_check_degrade(rs, stripe);
-+
-+ {
-+ char buf[BDEVNAME_SIZE];
-+
-+ DMERR("stopping recovery due to "
-+ "ERROR on /dev/%s, stripe at offset %llu",
-+ bdevname(rs->dev[rs->set.ei].dev->bdev, buf),
-+ (unsigned long long) stripe->key);
-+
-+ }
-+
-+ /* Make sure, that all quiesced regions get released. */
-+ do {
-+ if (rec->reg)
-+ dm_rh_recovery_end(rec->rh, rec->reg, -EIO);
-+
-+ rec->reg = dm_rh_recovery_start(rec->rh);
-+ } while (rec->reg);
-+
-+ recover_rh_update(rs, -EIO);
-+free:
-+ rs->set.dev_to_init = -1;
-+
-+ /* Check for jiffies overrun. */
-+ rs->recover.end_jiffies = jiffies;
-+ if (rs->recover.end_jiffies < rs->recover.start_jiffies)
-+ rs->recover.end_jiffies = ~0;
-+
-+ ClearRSRecover(rs);
-+}
-+
-+static INLINE void do_recovery(struct raid_set *rs)
-+{
-+ struct stripe *stripe;
-+
-+ list_for_each_entry(stripe, &rs->recover.stripes, lists[LIST_RECOVER])
-+ _do_recovery(rs, stripe);
-+
-+ if (!RSRecover(rs))
-+ stripe_recover_free(rs);
-+}
-+
-+/*
-+ * END recovery functions
-+ */
-+
-+/* End io process all stripes handed in by endio() callback. */
-+static void do_endios(struct raid_set *rs)
-+{
-+ struct stripe_cache *sc = &rs->sc;
-+ struct stripe *stripe;
-+
-+ while ((stripe = stripe_endio_pop(sc))) {
-+ unsigned count;
-+
-+ /* Recovery stripe special case. */
-+ if (unlikely(StripeRecover(stripe))) {
-+ if (stripe_io(stripe))
-+ continue;
-+
-+ io_put(rs); /* Release region io reference. */
-+ ClearStripeActive(stripe);
-+
-+ /* REMOVEME: statistics*/
-+ atomic_dec(&sc->active_stripes);
-+ continue;
-+ }
-+
-+ /* Early end io all reads on any uptodate chunks. */
-+ stripe_endio(READ, stripe, (count = 0, &count));
-+ if (stripe_io(stripe)) {
-+ if (count) /* REMOVEME: statistics. */
-+ atomic_inc(rs->stats + S_ACTIVE_READS);
-+
-+ continue;
-+ }
-+
-+ /* Set stripe inactive after all io got processed. */
-+ if (TestClearStripeActive(stripe))
-+ atomic_dec(&sc->active_stripes);
-+
-+ /* Unlock stripe (for clustering). */
-+ stripe_unlock(rs, stripe);
-+
-+ /*
-+ * If an io error on a stripe occured and the RAID set
-+ * is still operational, requeue the stripe for io.
-+ */
-+ if (TestClearStripeError(stripe)) {
-+ raid_set_check_degrade(rs, stripe);
-+ ClearStripeReconstruct(stripe);
-+
-+ if (!StripeMerged(stripe) &&
-+ raid_set_operational(rs)) {
-+ stripe_pages_invalidate(stripe);
-+ stripe_flush(stripe, FLUSH_DELAY);
-+ /* REMOVEME: statistics. */
-+ atomic_inc(rs->stats + S_REQUEUE);
-+ continue;
-+ }
-+ }
-+
-+ /* Check if the RAID set is inoperational to error ios. */
-+ if (!raid_set_operational(rs)) {
-+ ClearStripeReconstruct(stripe);
-+ stripe_fail_io(stripe);
-+ BUG_ON(atomic_read(&stripe->cnt));
-+ continue;
-+ }
-+
-+ /* Got to reconstruct a missing chunk. */
-+ if (TestClearStripeReconstruct(stripe))
-+ reconstruct_xor(stripe);
-+
-+ /*
-+ * Now that we've got a complete stripe, we can
-+ * process the rest of the end ios on reads.
-+ */
-+ BUG_ON(stripe_endio(READ, stripe, NULL));
-+ ClearStripeRead(stripe);
-+
-+ /*
-+ * Read-before-write stripes need to be flushed again in
-+ * order to work the write data into the pages *after*
-+ * they were read in.
-+ */
-+ if (TestClearStripeMerged(stripe))
-+ /* End io all bios which got merged already. */
-+ BUG_ON(stripe_endio(WRITE_MERGED, stripe, NULL));
-+
-+ /* Got to put on flush list because of new writes. */
-+ if (StripeRBW(stripe))
-+ stripe_flush(stripe, FLUSH_DELAY);
-+ }
-+}
-+
-+/*
-+ * Stripe cache shrinking.
-+ */
-+static INLINE void do_sc_shrink(struct raid_set *rs)
-+{
-+ unsigned shrink = atomic_read(&rs->sc.stripes_to_shrink);
-+
-+ if (shrink) {
-+ unsigned cur = atomic_read(&rs->sc.stripes);
-+
-+ sc_shrink(&rs->sc, shrink);
-+ shrink -= cur - atomic_read(&rs->sc.stripes);
-+ atomic_set(&rs->sc.stripes_to_shrink, shrink);
-+
-+ /*
-+ * Wake myself up in case we failed to shrink the
-+ * requested amount in order to try again later.
-+ */
-+ if (shrink)
-+ wake_do_raid(rs);
-+ }
-+}
-+
-+
-+/*
-+ * Process all ios
-+ *
-+ * We do different things with the io depending on the
-+ * state of the region that it's in:
-+ *
-+ * o reads: hang off stripe cache or postpone if full
-+ *
-+ * o writes:
-+ *
-+ * CLEAN/DIRTY/NOSYNC: increment pending and hang io off stripe's stripe set.
-+ * In case stripe cache is full or busy, postpone the io.
-+ *
-+ * RECOVERING: delay the io until recovery of the region completes.
-+ *
-+ */
-+static INLINE void do_ios(struct raid_set *rs, struct bio_list *ios)
-+{
-+ int r;
-+ unsigned flush = 0;
-+ struct dm_rh_client *rh = rs->recover.rh;
-+ struct bio *bio;
-+ struct bio_list delay, reject;
-+
-+ bio_list_init(&delay);
-+ bio_list_init(&reject);
-+
-+ /*
-+ * Classify each io:
-+ * o delay to recovering regions
-+ * o queue to all other regions
-+ */
-+ while ((bio = bio_list_pop(ios))) {
-+ /*
-+ * In case we get a barrier bio, push it back onto
-+ * the input queue unless all work queues are empty
-+ * and the stripe cache is inactive.
-+ */
-+ if (unlikely(bio_barrier(bio))) {
-+ /* REMOVEME: statistics. */
-+ atomic_inc(rs->stats + S_BARRIER);
-+ if (!list_empty(rs->sc.lists + LIST_IO) ||
-+ !bio_list_empty(&delay) ||
-+ !bio_list_empty(&reject) ||
-+ sc_active(&rs->sc)) {
-+ bio_list_push(ios, bio);
-+ break;
-+ }
-+ }
-+
-+ r = region_state(rs, _sector(rs, bio), DM_RH_RECOVERING);
-+ if (unlikely(r)) {
-+ /* Got to wait for recovering regions. */
-+ bio_list_add(&delay, bio);
-+ SetRSBandwidth(rs);
-+ } else {
-+ /*
-+ * Process ios to non-recovering regions by queueing
-+ * them to stripes (does rh_inc()) for writes).
-+ */
-+ flush += stripe_queue_bio(rs, bio, &reject);
-+ }
-+ }
-+
-+ if (flush) {
-+ r = dm_rh_flush(rh); /* Writes got queued -> flush dirty log. */
-+ if (r)
-+ DMERR("dirty log flush");
-+ }
-+
-+ /* Delay ios to regions which are recovering. */
-+ while ((bio = bio_list_pop(&delay))) {
-+ /* REMOVEME: statistics.*/
-+ atomic_inc(rs->stats + S_DELAYED_BIOS);
-+ atomic_inc(rs->stats + S_SUM_DELAYED_BIOS);
-+ dm_rh_delay_by_region(rh, bio,
-+ dm_rh_sector_to_region(rh, _sector(rs, bio)));
-+
-+ }
-+
-+ /* Merge any rejected bios back to the head of the input list. */
-+ bio_list_merge_head(ios, &reject);
-+}
-+
-+/* Flush any stripes on the io list. */
-+static INLINE void do_flush(struct raid_set *rs)
-+{
-+ struct list_head *list = rs->sc.lists + LIST_IO, *pos, *tmp;
-+
-+ list_for_each_safe(pos, tmp, list) {
-+ int r = stripe_flush(list_entry(pos, struct stripe,
-+ lists[LIST_IO]), FLUSH_NOW);
-+
-+ /* Remove from the list only if the stripe got processed. */
-+ if (!r)
-+ list_del_init(pos);
-+ }
-+}
-+
-+/* Send an event in case we're getting too busy. */
-+static INLINE void do_busy_event(struct raid_set *rs)
-+{
-+ if ((sc_active(&rs->sc) > atomic_read(&rs->sc.stripes) * 4 / 5)) {
-+ if (!TestSetRSScBusy(rs))
-+ dm_table_event(rs->ti->table);
-+ } else
-+ ClearRSScBusy(rs);
-+}
-+
-+/* Unplug: let the io role on the sets devices. */
-+static INLINE void do_unplug(struct raid_set *rs)
-+{
-+ struct raid_dev *dev = rs->dev + rs->set.raid_devs;
-+
-+ while (dev-- > rs->dev) {
-+ /* Only call any device unplug function, if io got queued. */
-+ if (io_dev_clear(dev))
-+ blk_unplug(bdev_get_queue(dev->dev->bdev));
-+ }
-+}
-+
-+/*-----------------------------------------------------------------
-+ * RAID daemon
-+ *---------------------------------------------------------------*/
-+/*
-+ * o belabour all end ios
-+ * o optionally shrink the stripe cache
-+ * o update the region hash states
-+ * o optionally do recovery
-+ * o grab the input queue
-+ * o work an all requeued or new ios and perform stripe cache flushs
-+ * unless the RAID set is inoperational (when we error ios)
-+ * o check, if the stripe cache gets too busy and throw an event if so
-+ * o unplug any component raid devices with queued bios
-+ */
-+static void do_raid(struct work_struct *ws)
-+{
-+ struct raid_set *rs = container_of(ws, struct raid_set, io.dws.work);
-+ struct bio_list *ios = &rs->io.work, *ios_in = &rs->io.in;
-+ spinlock_t *lock = &rs->io.in_lock;
-+
-+ /*
-+ * We always need to end io, so that ios
-+ * can get errored in case the set failed
-+ * and the region counters get decremented
-+ * before we update the region hash states.
-+ */
-+redo:
-+ do_endios(rs);
-+
-+ /*
-+ * Now that we've end io'd, which may have put stripes on
-+ * the LRU list, we shrink the stripe cache if requested.
-+ */
-+ do_sc_shrink(rs);
-+
-+ /* Update region hash states before we go any further. */
-+ dm_rh_update_states(rs->recover.rh, 1);
-+
-+ /* Try to recover regions. */
-+ if (RSRecover(rs))
-+ do_recovery(rs);
-+
-+ /* More endios -> process. */
-+ if (!stripe_endio_empty(&rs->sc)) {
-+ atomic_inc(rs->stats + S_REDO);
-+ goto redo;
-+ }
-+
-+ /* Quickly grab all new ios queued and add them to the work list. */
-+ spin_lock_irq(lock);
-+ bio_list_merge(ios, ios_in);
-+ bio_list_init(ios_in);
-+ spin_unlock_irq(lock);
-+
-+ /* Let's assume we're operational most of the time ;-). */
-+ if (likely(raid_set_operational(rs))) {
-+ /* If we got ios, work them into the cache. */
-+ if (!bio_list_empty(ios)) {
-+ do_ios(rs, ios);
-+ do_unplug(rs); /* Unplug the sets device queues. */
-+ }
-+
-+ do_flush(rs); /* Flush any stripes on io list. */
-+ do_unplug(rs); /* Unplug the sets device queues. */
-+ do_busy_event(rs); /* Check if we got too busy. */
-+
-+ /* More endios -> process. */
-+ if (!stripe_endio_empty(&rs->sc)) {
-+ atomic_inc(rs->stats + S_REDO);
-+ goto redo;
-+ }
-+ } else
-+ /* No way to reconstruct data with too many devices failed. */
-+ bio_list_fail(rs, NULL, ios);
-+}
-+
-+/*
-+ * Callback for region hash to dispatch
-+ * delayed bios queued to recovered regions
-+ * (Gets called via rh_update_states()).
-+ */
-+static void dispatch_delayed_bios(void *context, struct bio_list *bl, int dummy)
-+{
-+ struct raid_set *rs = context;
-+ struct bio *bio;
-+
-+ /* REMOVEME: decrement pending delayed bios counter. */
-+ bio_list_for_each(bio, bl)
-+ atomic_dec(rs->stats + S_DELAYED_BIOS);
-+
-+ /* Merge region hash private list to work list. */
-+ bio_list_merge_head(&rs->io.work, bl);
-+ bio_list_init(bl);
-+ ClearRSBandwidth(rs);
-+}
-+
-+/*************************************************************
-+ * Constructor helpers
-+ *************************************************************/
-+/* Calculate MB/sec. */
-+static INLINE unsigned mbpers(struct raid_set *rs, unsigned speed)
-+{
-+ return to_bytes(speed * rs->set.data_devs *
-+ rs->recover.io_size * HZ >> 10) >> 10;
-+}
-+
-+/*
-+ * Discover fastest xor algorithm and # of chunks combination.
-+ */
-+/* Calculate speed for algorithm and # of chunks. */
-+static INLINE unsigned xor_speed(struct stripe *stripe)
-+{
-+ unsigned r = 0;
-+ unsigned long j;
-+
-+ /* Wait for next tick. */
-+ for (j = jiffies; j == jiffies;)
-+ ;
-+
-+ /* Do xors for a full tick. */
-+ for (j = jiffies; j == jiffies;) {
-+ mb();
-+ common_xor(stripe, stripe->io.size, 0, 0);
-+ mb();
-+ r++;
-+ mb();
-+ }
-+
-+ return r;
-+}
-+
-+/* Optimize xor algorithm for this RAID set. */
-+static unsigned xor_optimize(struct raid_set *rs)
-+{
-+ unsigned chunks_max = 2, speed_max = 0;
-+ struct xor_func *f = ARRAY_END(xor_funcs), *f_max = NULL;
-+ struct stripe *stripe;
-+
-+ BUG_ON(list_empty(&rs->recover.stripes));
-+ stripe = list_first_entry(&rs->recover.stripes, struct stripe,
-+ lists[LIST_RECOVER]);
-+
-+ /*
-+ * Got to allow io on all chunks, so that
-+ * xor() will actually work on them.
-+ */
-+ stripe_allow_io(stripe);
-+
-+ /* Try all xor functions. */
-+ while (f-- > xor_funcs) {
-+ unsigned speed;
-+
-+ /* Set actual xor function for common_xor(). */
-+ rs->xor.f = f;
-+ rs->xor.chunks = XOR_CHUNKS_MAX + 1;
-+
-+ while (rs->xor.chunks-- > 2) {
-+ speed = xor_speed(stripe);
-+ if (speed > speed_max) {
-+ speed_max = speed;
-+ chunks_max = rs->xor.chunks;
-+ f_max = f;
-+ }
-+ }
-+ }
-+
-+ /* Memorize optimum parameters. */
-+ rs->xor.f = f_max;
-+ rs->xor.chunks = chunks_max;
-+ return speed_max;
-+}
-+
-+/*
-+ * Allocate a RAID context (a RAID set)
-+ */
-+static int
-+context_alloc(struct raid_set **raid_set, struct raid_type *raid_type,
-+ unsigned stripes, unsigned chunk_size, unsigned io_size,
-+ unsigned recover_io_size, unsigned raid_devs,
-+ sector_t sectors_per_dev,
-+ struct dm_target *ti, unsigned dl_parms, char **argv)
-+{
-+ int r;
-+ unsigned p;
-+ size_t len;
-+ sector_t region_size, ti_len;
-+ struct raid_set *rs = NULL;
-+ struct dm_dirty_log *dl;
-+ struct recover *rec;
-+
-+ /*
-+ * Create the dirty log
-+ *
-+ * We need to change length for the dirty log constructor,
-+ * because we want an amount of regions for all stripes derived
-+ * from the single device size, so that we can keep region
-+ * size = 2^^n independant of the number of devices
-+ */
-+ ti_len = ti->len;
-+ ti->len = sectors_per_dev;
-+ dl = dm_dirty_log_create(argv[0], ti, dl_parms, argv + 2);
-+ ti->len = ti_len;
-+ if (!dl)
-+ goto bad_dirty_log;
-+
-+ /* Chunk size *must* be smaller than region size. */
-+ region_size = dl->type->get_region_size(dl);
-+ if (chunk_size > region_size)
-+ goto bad_chunk_size;
-+
-+ /* Recover io size *must* be smaller than region size as well. */
-+ if (recover_io_size > region_size)
-+ goto bad_recover_io_size;
-+
-+ /* Size and allocate the RAID set structure. */
-+ len = sizeof(*rs->data) + sizeof(*rs->dev);
-+ if (array_too_big(sizeof(*rs), len, raid_devs))
-+ goto bad_array;
-+
-+ len = sizeof(*rs) + raid_devs * len;
-+ rs = kzalloc(len, GFP_KERNEL);
-+ if (!rs)
-+ goto bad_alloc;
-+
-+ rec = &rs->recover;
-+ atomic_set(&rs->io.in_process, 0);
-+ atomic_set(&rs->io.in_process_max, 0);
-+ rec->io_size = recover_io_size;
-+
-+ /* Pointer to data array. */
-+ rs->data = (unsigned long **)
-+ ((void *) rs->dev + raid_devs * sizeof(*rs->dev));
-+ rec->dl = dl;
-+ rs->set.raid_devs = p = raid_devs;
-+ rs->set.data_devs = raid_devs - raid_type->parity_devs;
-+ rs->set.raid_type = raid_type;
-+
-+ /*
-+ * Set chunk and io size and respective shifts
-+ * (used to avoid divisions)
-+ */
-+ rs->set.chunk_size = chunk_size;
-+ rs->set.chunk_mask = chunk_size - 1;
-+ rs->set.chunk_shift = ffs(chunk_size) - 1;
-+
-+ rs->set.io_size = io_size;
-+ rs->set.io_mask = io_size - 1;
-+ rs->set.io_shift = ffs(io_size) - 1;
-+ rs->set.io_shift_mask = rs->set.chunk_mask & ~rs->set.io_mask;
-+
-+ rs->set.pages_per_io = chunk_pages(io_size);
-+ rs->set.sectors_per_dev = sectors_per_dev;
-+
-+ rs->set.ei = -1; /* Indicate no failed device. */
-+ atomic_set(&rs->set.failed_devs, 0);
-+
-+ rs->ti = ti;
-+
-+ atomic_set(rec->io_count + IO_WORK, 0);
-+ atomic_set(rec->io_count + IO_RECOVER, 0);
-+
-+ /* Initialize io lock and queues. */
-+ spin_lock_init(&rs->io.in_lock);
-+ bio_list_init(&rs->io.in);
-+ bio_list_init(&rs->io.work);
-+
-+ init_waitqueue_head(&rs->io.suspendq); /* Suspend waiters (dm-io). */
-+
-+ rec->nr_regions = dm_sector_div_up(sectors_per_dev, region_size);
-+ rec->rh = dm_rh_client_create(MAX_RECOVER, dispatch_delayed_bios, rs,
-+ wake_do_raid, rs, dl, region_size,
-+ rs->recover.nr_regions);
-+ if (IS_ERR(rec->rh))
-+ goto bad_rh;
-+
-+ /* Initialize stripe cache. */
-+ r = sc_init(rs, stripes);
-+ if (r)
-+ goto bad_sc;
-+
-+ /* Create dm-io client context. */
-+ rs->sc.dm_io_client = dm_io_client_create(rs->set.raid_devs *
-+ rs->set.pages_per_io);
-+ if (IS_ERR(rs->sc.dm_io_client))
-+ goto bad_dm_io_client;
-+
-+ /* REMOVEME: statistics. */
-+ stats_reset(rs);
-+ ClearRSDevelStats(rs); /* Disnable development status. */
-+
-+ *raid_set = rs;
-+ return 0;
-+
-+bad_dirty_log:
-+ TI_ERR_RET("Error creating dirty log", -ENOMEM);
-+
-+
-+bad_chunk_size:
-+ dm_dirty_log_destroy(dl);
-+ TI_ERR("Chunk size larger than region size");
-+
-+bad_recover_io_size:
-+ dm_dirty_log_destroy(dl);
-+ TI_ERR("Recover stripe io size larger than region size");
-+
-+bad_array:
-+ dm_dirty_log_destroy(dl);
-+ TI_ERR("Arry too big");
-+
-+bad_alloc:
-+ dm_dirty_log_destroy(dl);
-+ TI_ERR_RET("Cannot allocate raid context", -ENOMEM);
-+
-+bad_rh:
-+ dm_dirty_log_destroy(dl);
-+ ti->error = DM_MSG_PREFIX "Error creating dirty region hash";
-+ goto free_rs;
-+
-+bad_sc:
-+ ti->error = DM_MSG_PREFIX "Error creating stripe cache";
-+ goto free;
-+
-+bad_dm_io_client:
-+ ti->error = DM_MSG_PREFIX "Error allocating dm-io resources";
-+free:
-+ dm_rh_client_destroy(rec->rh);
-+ sc_exit(&rs->sc);
-+ dm_rh_client_destroy(rec->rh); /* Destroys dirty log as well. */
-+free_rs:
-+ kfree(rs);
-+ return -ENOMEM;
-+}
-+
-+/* Free a RAID context (a RAID set). */
-+static void
-+context_free(struct raid_set *rs, struct dm_target *ti, unsigned r)
-+{
-+ while (r--)
-+ dm_put_device(ti, rs->dev[r].dev);
-+
-+ dm_io_client_destroy(rs->sc.dm_io_client);
-+ sc_exit(&rs->sc);
-+ dm_rh_client_destroy(rs->recover.rh);
-+ dm_dirty_log_destroy(rs->recover.dl);
-+ kfree(rs);
-+}
-+
-+/* Create work queue and initialize work. */
-+static int rs_workqueue_init(struct raid_set *rs)
-+{
-+ struct dm_target *ti = rs->ti;
-+
-+ rs->io.wq = create_singlethread_workqueue(DAEMON);
-+ if (!rs->io.wq)
-+ TI_ERR_RET("failed to create " DAEMON, -ENOMEM);
-+
-+ INIT_DELAYED_WORK(&rs->io.dws, do_raid);
-+ return 0;
-+}
-+
-+/* Return pointer to raid_type structure for raid name. */
-+static struct raid_type *get_raid_type(char *name)
-+{
-+ struct raid_type *r = ARRAY_END(raid_types);
-+
-+ while (r-- > raid_types) {
-+ if (!strnicmp(STR_LEN(r->name, name)))
-+ return r;
-+ }
-+
-+ return NULL;
-+}
-+
-+/* FIXME: factor out to dm core. */
-+static int multiple(sector_t a, sector_t b, sector_t *n)
-+{
-+ sector_t r = a;
-+
-+ sector_div(r, b);
-+ *n = r;
-+ return a == r * b;
-+}
-+
-+/* Log RAID set information to kernel log. */
-+static void raid_set_log(struct raid_set *rs, unsigned speed)
-+{
-+ unsigned p;
-+ char buf[BDEVNAME_SIZE];
-+
-+ for (p = 0; p < rs->set.raid_devs; p++)
-+ DMINFO("/dev/%s is raid disk %u",
-+ bdevname(rs->dev[p].dev->bdev, buf), p);
-+
-+ DMINFO("%d/%d/%d sectors chunk/io/recovery size, %u stripes",
-+ rs->set.chunk_size, rs->set.io_size, rs->recover.io_size,
-+ atomic_read(&rs->sc.stripes));
-+ DMINFO("algorithm \"%s\", %u chunks with %uMB/s", rs->xor.f->name,
-+ rs->xor.chunks, mbpers(rs, speed));
-+ DMINFO("%s set with net %u/%u devices", rs->set.raid_type->descr,
-+ rs->set.data_devs, rs->set.raid_devs);
-+}
-+
-+/* Get all devices and offsets. */
-+static int
-+dev_parms(struct dm_target *ti, struct raid_set *rs,
-+ char **argv, int *p)
-+{
-+ for (*p = 0; *p < rs->set.raid_devs; (*p)++, argv += 2) {
-+ int r;
-+ unsigned long long tmp;
-+ struct raid_dev *dev = rs->dev + *p;
-+ union dev_lookup dl = {.dev = dev };
-+
-+ /* Get offset and device. */
-+ r = sscanf(argv[1], "%llu", &tmp);
-+ if (r != 1)
-+ TI_ERR("Invalid RAID device offset parameter");
-+
-+ dev->start = tmp;
-+ r = dm_get_device(ti, argv[0], dev->start,
-+ rs->set.sectors_per_dev,
-+ dm_table_get_mode(ti->table), &dev->dev);
-+ if (r)
-+ TI_ERR_RET("RAID device lookup failure", r);
-+
-+ r = raid_dev_lookup(rs, bynumber, &dl);
-+ if (r != -ENODEV && r < *p) {
-+ (*p)++; /* Ensure dm_put_device() on actual device. */
-+ TI_ERR_RET("Duplicate RAID device", -ENXIO);
-+ }
-+ }
-+
-+ return 0;
-+}
-+
-+/* Set recovery bandwidth. */
-+static INLINE void
-+recover_set_bandwidth(struct raid_set *rs, unsigned bandwidth)
-+{
-+ rs->recover.bandwidth = bandwidth;
-+ rs->recover.bandwidth_work = 100 / bandwidth;
-+}
-+
-+/* Handle variable number of RAID parameters. */
-+static int
-+raid_variable_parms(struct dm_target *ti, char **argv,
-+ unsigned i, int *raid_parms,
-+ int *chunk_size, int *chunk_size_parm,
-+ int *stripes, int *stripes_parm,
-+ int *io_size, int *io_size_parm,
-+ int *recover_io_size, int *recover_io_size_parm,
-+ int *bandwidth, int *bandwidth_parm)
-+{
-+ /* Fetch # of variable raid parameters. */
-+ if (sscanf(argv[i++], "%d", raid_parms) != 1 ||
-+ !range_ok(*raid_parms, 0, 5))
-+ TI_ERR("Bad variable raid parameters number");
-+
-+ if (*raid_parms) {
-+ /*
-+ * If we've got variable RAID parameters,
-+ * chunk size is the first one
-+ */
-+ if (sscanf(argv[i++], "%d", chunk_size) != 1 ||
-+ (*chunk_size != -1 &&
-+ (!POWER_OF_2(*chunk_size) ||
-+ !range_ok(*chunk_size, IO_SIZE_MIN, CHUNK_SIZE_MAX))))
-+ TI_ERR("Invalid chunk size; must be 2^^n and <= 16384");
-+
-+ *chunk_size_parm = *chunk_size;
-+ if (*chunk_size == -1)
-+ *chunk_size = CHUNK_SIZE;
-+
-+ /*
-+ * In case we've got 2 or more variable raid
-+ * parameters, the number of stripes is the second one
-+ */
-+ if (*raid_parms > 1) {
-+ if (sscanf(argv[i++], "%d", stripes) != 1 ||
-+ (*stripes != -1 &&
-+ !range_ok(*stripes, STRIPES_MIN,
-+ STRIPES_MAX)))
-+ TI_ERR("Invalid number of stripes: must "
-+ "be >= 8 and <= 8192");
-+ }
-+
-+ *stripes_parm = *stripes;
-+ if (*stripes == -1)
-+ *stripes = STRIPES;
-+
-+ /*
-+ * In case we've got 3 or more variable raid
-+ * parameters, the io size is the third one.
-+ */
-+ if (*raid_parms > 2) {
-+ if (sscanf(argv[i++], "%d", io_size) != 1 ||
-+ (*io_size != -1 &&
-+ (!POWER_OF_2(*io_size) ||
-+ !range_ok(*io_size, IO_SIZE_MIN,
-+ min(BIO_MAX_SECTORS / 2,
-+ *chunk_size)))))
-+ TI_ERR("Invalid io size; must "
-+ "be 2^^n and less equal "
-+ "min(BIO_MAX_SECTORS/2, chunk size)");
-+ } else
-+ *io_size = *chunk_size;
-+
-+ *io_size_parm = *io_size;
-+ if (*io_size == -1)
-+ *io_size = *chunk_size;
-+
-+ /*
-+ * In case we've got 4 variable raid parameters,
-+ * the recovery stripe io_size is the fourth one
-+ */
-+ if (*raid_parms > 3) {
-+ if (sscanf(argv[i++], "%d", recover_io_size) != 1 ||
-+ (*recover_io_size != -1 &&
-+ (!POWER_OF_2(*recover_io_size) ||
-+ !range_ok(*recover_io_size, RECOVER_IO_SIZE_MIN,
-+ BIO_MAX_SECTORS / 2))))
-+ TI_ERR("Invalid recovery io size; must be "
-+ "2^^n and less equal BIO_MAX_SECTORS/2");
-+ }
-+
-+ *recover_io_size_parm = *recover_io_size;
-+ if (*recover_io_size == -1)
-+ *recover_io_size = RECOVER_IO_SIZE;
-+
-+ /*
-+ * In case we've got 5 variable raid parameters,
-+ * the recovery io bandwidth is the fifth one
-+ */
-+ if (*raid_parms > 4) {
-+ if (sscanf(argv[i++], "%d", bandwidth) != 1 ||
-+ (*bandwidth != -1 &&
-+ !range_ok(*bandwidth, BANDWIDTH_MIN,
-+ BANDWIDTH_MAX)))
-+ TI_ERR("Invalid recovery bandwidth "
-+ "percentage; must be > 0 and <= 100");
-+ }
-+
-+ *bandwidth_parm = *bandwidth;
-+ if (*bandwidth == -1)
-+ *bandwidth = BANDWIDTH;
-+ }
-+
-+ return 0;
-+}
-+
-+/* Parse optional locking parameters. */
-+static int
-+raid_locking_parms(struct dm_target *ti, char **argv,
-+ unsigned i, int *locking_parms,
-+ struct dm_raid45_locking_type **locking_type)
-+{
-+ *locking_parms = 0;
-+ *locking_type = &locking_none;
-+
-+ if (!strnicmp(argv[i], "none", strlen(argv[i])))
-+ *locking_parms = 1;
-+ else if (!strnicmp(argv[i + 1], "locking", strlen(argv[i + 1]))) {
-+ *locking_type = &locking_none;
-+ *locking_parms = 2;
-+ } else if (!strnicmp(argv[i + 1], "cluster", strlen(argv[i + 1]))) {
-+ *locking_type = &locking_cluster;
-+ /* FIXME: namespace. */
-+ *locking_parms = 3;
-+ }
-+
-+ return *locking_parms == 1 ? -EINVAL : 0;
-+}
-+
-+/* Set backing device information properties of RAID set. */
-+static void rs_set_bdi(struct raid_set *rs, unsigned stripes, unsigned chunks)
-+{
-+ unsigned p, ra_pages;
-+ struct mapped_device *md = dm_table_get_md(rs->ti->table);
-+ struct backing_dev_info *bdi = &dm_disk(md)->queue->backing_dev_info;
-+
-+ /* Set read-ahead for the RAID set and the component devices. */
-+ bdi->ra_pages = stripes * stripe_pages(rs, rs->set.io_size);
-+ ra_pages = chunks * chunk_pages(rs->set.io_size);
-+ for (p = rs->set.raid_devs; p--; ) {
-+ struct request_queue *q = bdev_get_queue(rs->dev[p].dev->bdev);
-+
-+ q->backing_dev_info.ra_pages = ra_pages;
-+ }
-+
-+ /* Set congested function and data. */
-+ bdi->congested_fn = raid_set_congested;
-+ bdi->congested_data = rs;
-+
-+ dm_put(md);
-+}
-+
-+/* Get backing device information properties of RAID set. */
-+static void rs_get_ra(struct raid_set *rs, unsigned *stripes, unsigned *chunks)
-+{
-+ struct mapped_device *md = dm_table_get_md(rs->ti->table);
-+
-+ *stripes = dm_disk(md)->queue->backing_dev_info.ra_pages
-+ / stripe_pages(rs, rs->set.io_size);
-+ *chunks = bdev_get_queue(rs->dev->dev->bdev)->backing_dev_info.ra_pages
-+ / chunk_pages(rs->set.io_size);
-+
-+ dm_put(md);
-+}
-+
-+/*
-+ * Construct a RAID4/5 mapping:
-+ *
-+ * log_type #log_params <log_params> \
-+ * raid_type [#parity_dev] #raid_variable_params <raid_params> \
-+ * [locking "none"/"cluster"]
-+ * #raid_devs #dev_to_initialize [<dev_path> <offset>]{3,}
-+ *
-+ * log_type = "core"/"disk",
-+ * #log_params = 1-3 (1-2 for core dirty log type, 3 for disk dirty log only)
-+ * log_params = [dirty_log_path] region_size [[no]sync])
-+ *
-+ * raid_type = "raid4", "raid5_la", "raid5_ra", "raid5_ls", "raid5_rs"
-+ *
-+ * #parity_dev = N if raid_type = "raid4"
-+ * o N = -1: pick default = last device
-+ * o N >= 0 and < #raid_devs: parity device index
-+ *
-+ * #raid_variable_params = 0-5; raid_params (-1 = default):
-+ * [chunk_size [#stripes [io_size [recover_io_size [%recovery_bandwidth]]]]]
-+ * o chunk_size (unit to calculate drive addresses; must be 2^^n, > 8
-+ * and <= CHUNK_SIZE_MAX)
-+ * o #stripes is number of stripes allocated to stripe cache
-+ * (must be > 1 and < STRIPES_MAX)
-+ * o io_size (io unit size per device in sectors; must be 2^^n and > 8)
-+ * o recover_io_size (io unit size per device for recovery in sectors;
-+ must be 2^^n, > SECTORS_PER_PAGE and <= region_size)
-+ * o %recovery_bandwith is the maximum amount spend for recovery during
-+ * application io (1-100%)
-+ * If raid_variable_params = 0, defaults will be used.
-+ * Any raid_variable_param can be set to -1 to apply a default
-+ *
-+ * #raid_devs = N (N >= 3)
-+ *
-+ * #dev_to_initialize = N
-+ * -1: initialize parity on all devices
-+ * >= 0 and < #raid_devs: initialize raid_path; used to force reconstruction
-+ * of a failed devices content after replacement
-+ *
-+ * <dev_path> = device_path (eg, /dev/sdd1)
-+ * <offset> = begin at offset on <dev_path>
-+ *
-+ */
-+#define MIN_PARMS 13
-+static int raid_ctr(struct dm_target *ti, unsigned argc, char **argv)
-+{
-+ int bandwidth = BANDWIDTH, bandwidth_parm = -1,
-+ chunk_size = CHUNK_SIZE, chunk_size_parm = -1,
-+ dev_to_init, dl_parms, locking_parms, parity_parm, pi = -1,
-+ i, io_size = IO_SIZE, io_size_parm = -1,
-+ r, raid_devs, raid_parms,
-+ recover_io_size = RECOVER_IO_SIZE, recover_io_size_parm = -1,
-+ stripes = STRIPES, stripes_parm = -1;
-+ unsigned speed;
-+ sector_t tmp, sectors_per_dev;
-+ struct dm_raid45_locking_type *locking;
-+ struct raid_set *rs;
-+ struct raid_type *raid_type;
-+
-+ /* Ensure minimum number of parameters. */
-+ if (argc < MIN_PARMS)
-+ TI_ERR("Not enough parameters");
-+
-+ /* Fetch # of dirty log parameters. */
-+ if (sscanf(argv[1], "%d", &dl_parms) != 1
-+ || !range_ok(dl_parms, 1, 4711))
-+ TI_ERR("Bad dirty log parameters number");
-+
-+ /* Check raid_type. */
-+ raid_type = get_raid_type(argv[dl_parms + 2]);
-+ if (!raid_type)
-+ TI_ERR("Bad raid type");
-+
-+ /* In case of RAID4, parity drive is selectable. */
-+ parity_parm = !!(raid_type->level == raid4);
-+
-+ /* Handle variable number of RAID parameters. */
-+ r = raid_variable_parms(ti, argv, dl_parms + parity_parm + 3,
-+ &raid_parms,
-+ &chunk_size, &chunk_size_parm,
-+ &stripes, &stripes_parm,
-+ &io_size, &io_size_parm,
-+ &recover_io_size, &recover_io_size_parm,
-+ &bandwidth, &bandwidth_parm);
-+ if (r)
-+ return r;
-+
-+ r = raid_locking_parms(ti, argv,
-+ dl_parms + parity_parm + raid_parms + 4,
-+ &locking_parms, &locking);
-+ if (r)
-+ return r;
-+
-+ /* # of raid devices. */
-+ i = dl_parms + parity_parm + raid_parms + locking_parms + 4;
-+ if (sscanf(argv[i], "%d", &raid_devs) != 1 ||
-+ raid_devs < raid_type->minimal_devs)
-+ TI_ERR("Invalid number of raid devices");
-+
-+ /* In case of RAID4, check parity drive index is in limits. */
-+ if (raid_type->level == raid4) {
-+ /* Fetch index of parity device. */
-+ if (sscanf(argv[dl_parms + 3], "%d", &pi) != 1 ||
-+ !range_ok(pi, 0, raid_devs - 1))
-+ TI_ERR("Invalid RAID4 parity device index");
-+ }
-+
-+ /*
-+ * Index of device to initialize starts at 0
-+ *
-+ * o -1 -> don't initialize a particular device,
-+ * o 0..raid_devs-1 -> initialize respective device
-+ * (used for reconstruction of a replaced device)
-+ */
-+ if (sscanf
-+ (argv[dl_parms + parity_parm + raid_parms + locking_parms + 5],
-+ "%d", &dev_to_init) != 1
-+ || !range_ok(dev_to_init, -1, raid_devs - 1))
-+ TI_ERR("Invalid number for raid device to initialize");
-+
-+ /* Check # of raid device arguments. */
-+ if (argc - dl_parms - parity_parm - raid_parms - 6 !=
-+ 2 * raid_devs)
-+ TI_ERR("Wrong number of raid device/offset arguments");
-+
-+ /*
-+ * Check that the table length is devisable
-+ * w/o rest by (raid_devs - parity_devs)
-+ */
-+ if (!multiple(ti->len, raid_devs - raid_type->parity_devs,
-+ §ors_per_dev))
-+ TI_ERR
-+ ("Target length not divisable by number of data devices");
-+
-+ /*
-+ * Check that the device size is
-+ * devisable w/o rest by chunk size
-+ */
-+ if (!multiple(sectors_per_dev, chunk_size, &tmp))
-+ TI_ERR("Device length not divisable by chunk_size");
-+
-+ /****************************************************************
-+ * Now that we checked the constructor arguments ->
-+ * let's allocate the RAID set
-+ ****************************************************************/
-+ r = context_alloc(&rs, raid_type, stripes, chunk_size, io_size,
-+ recover_io_size, raid_devs, sectors_per_dev,
-+ ti, dl_parms, argv);
-+ if (r)
-+ return r;
-+
-+ /*
-+ * Set these here in order to avoid passing
-+ * too many arguments to context_alloc()
-+ */
-+ rs->set.dev_to_init_parm = dev_to_init;
-+ rs->set.dev_to_init = dev_to_init;
-+ rs->set.pi_parm = pi;
-+ rs->set.pi = (pi == -1) ? rs->set.data_devs : pi;
-+ rs->set.raid_parms = raid_parms;
-+ rs->set.chunk_size_parm = chunk_size_parm;
-+ rs->set.io_size_parm = io_size_parm;
-+ rs->sc.stripes_parm = stripes_parm;
-+ rs->recover.io_size_parm = recover_io_size_parm;
-+ rs->recover.bandwidth_parm = bandwidth_parm;
-+ recover_set_bandwidth(rs, bandwidth);
-+
-+ /* Use locking type to lock stripe access. */
-+ rs->locking = locking;
-+
-+ /* Get the device/offset tupels. */
-+ argv += dl_parms + 6 + parity_parm + raid_parms;
-+ r = dev_parms(ti, rs, argv, &i);
-+ if (r)
-+ goto err;
-+
-+ /* Initialize recovery. */
-+ rs->recover.start_jiffies = jiffies;
-+ rs->recover.end_jiffies = 0;
-+ recovery_region_reset(rs);
-+
-+ /* Allow for recovery of any nosync regions. */
-+ SetRSRecover(rs);
-+
-+ /* Set backing device information (eg. read ahead). */
-+ rs_set_bdi(rs, chunk_size * 2, io_size * 4);
-+ SetRSCheckOverwrite(rs); /* Allow chunk overwrite checks. */
-+
-+ speed = xor_optimize(rs); /* Select best xor algorithm. */
-+
-+ /* Initialize work queue to handle this RAID set's io. */
-+ r = rs_workqueue_init(rs);
-+ if (r)
-+ goto err;
-+
-+ raid_set_log(rs, speed); /* Log information about RAID set. */
-+
-+ /*
-+ * Make sure that dm core only hands maximum io size
-+ * length down and pays attention to io boundaries.
-+ */
-+ ti->split_io = rs->set.io_size;
-+ ti->private = rs;
-+ return 0;
-+
-+err:
-+ context_free(rs, ti, i);
-+ return r;
-+}
-+
-+/*
-+ * Destruct a raid mapping
-+ */
-+static void raid_dtr(struct dm_target *ti)
-+{
-+ struct raid_set *rs = ti->private;
-+
-+ /* Indicate recovery end so that ios in flight drain. */
-+ ClearRSRecover(rs);
-+
-+ wake_do_raid(rs); /* Wake daemon. */
-+ wait_ios(rs); /* Wait for any io still being processed. */
-+ destroy_workqueue(rs->io.wq);
-+ context_free(rs, ti, rs->set.raid_devs);
-+}
-+
-+/* Queues ios to RAID sets. */
-+static inline void queue_bio(struct raid_set *rs, struct bio *bio)
-+{
-+ int wake;
-+ struct bio_list *in = &rs->io.in;
-+ spinlock_t *in_lock = &rs->io.in_lock;
-+
-+ spin_lock_irq(in_lock);
-+ wake = bio_list_empty(in);
-+ bio_list_add(in, bio);
-+ spin_unlock_irq(in_lock);
-+
-+ /* Wake daemon if input list was empty. */
-+ if (wake)
-+ wake_do_raid(rs);
-+}
-+
-+/* Raid mapping function. */
-+static int raid_map(struct dm_target *ti, struct bio *bio,
-+ union map_info *map_context)
-+{
-+ /* I don't want to waste stripe cache capacity. */
-+ if (bio_rw(bio) == READA)
-+ return -EIO;
-+ else {
-+ struct raid_set *rs = ti->private;
-+
-+ /* REMOVEME: statistics. */
-+ atomic_inc(rs->stats +
-+ (bio_data_dir(bio) == WRITE ?
-+ S_BIOS_WRITE : S_BIOS_READ));
-+
-+ /*
-+ * Get io reference to be waiting for to drop
-+ * to zero on device suspension/destruction.
-+ */
-+ io_get(rs);
-+ bio->bi_sector -= ti->begin; /* Remap sector. */
-+ queue_bio(rs, bio); /* Queue to the daemon. */
-+ return DM_MAPIO_SUBMITTED; /* Handle later. */
-+ }
-+}
-+
-+/* Device suspend. */
-+static void raid_postsuspend(struct dm_target *ti)
-+{
-+ struct raid_set *rs = ti->private;
-+ struct dm_dirty_log *dl = rs->recover.dl;
-+
-+ SetRSSuspended(rs);
-+
-+ if (RSRecover(rs))
-+ dm_rh_stop_recovery(rs->recover.rh); /* Wakes do_raid(). */
-+ else
-+ wake_do_raid(rs);
-+
-+ wait_ios(rs); /* Wait for completion of all ios being processed. */
-+ if (dl->type->postsuspend && dl->type->postsuspend(dl))
-+ /* Suspend dirty log. */
-+ /* FIXME: need better error handling. */
-+ DMWARN("log suspend failed");
-+}
-+
-+/* Device resume. */
-+static void raid_resume(struct dm_target *ti)
-+{
-+ struct raid_set *rs = ti->private;
-+ struct recover *rec = &rs->recover;
-+ struct dm_dirty_log *dl = rec->dl;
-+
-+ if (dl->type->resume && dl->type->resume(dl))
-+ /* Resume dirty log. */
-+ /* FIXME: need better error handling. */
-+ DMWARN("log resume failed");
-+
-+ rec->nr_regions_to_recover =
-+ rec->nr_regions - dl->type->get_sync_count(dl);
-+
-+ ClearRSSuspended(rs);
-+
-+ /* Reset any unfinished recovery. */
-+ if (RSRecover(rs)) {
-+ recovery_region_reset(rs);
-+ dm_rh_start_recovery(rec->rh);/* Calls wake_do_raid(). */
-+ } else
-+ wake_do_raid(rs);
-+}
-+
-+static INLINE unsigned sc_size(struct raid_set *rs)
-+{
-+ return to_sector(atomic_read(&rs->sc.stripes) *
-+ (sizeof(struct stripe) +
-+ (sizeof(struct stripe_set) +
-+ (sizeof(struct page_list) +
-+ to_bytes(rs->set.io_size) *
-+ rs->set.raid_devs)) +
-+ (rs->recover.
-+ end_jiffies ? 0 : to_bytes(rs->set.raid_devs *
-+ rs->recover.
-+ io_size))));
-+}
-+
-+/* REMOVEME: status output for development. */
-+static void
-+raid_devel_stats(struct dm_target *ti, char *result,
-+ unsigned *size, unsigned maxlen)
-+{
-+ unsigned chunks, stripes, sz = *size;
-+ unsigned long j;
-+ char buf[BDEVNAME_SIZE], *p;
-+ struct stats_map *sm, *sm_end = ARRAY_END(stats_map);
-+ struct raid_set *rs = ti->private;
-+ struct recover *rec = &rs->recover;
-+ struct timespec ts;
-+
-+ DMEMIT("%s ", version);
-+ DMEMIT("io_inprocess=%d ", atomic_read(&rs->io.in_process));
-+ DMEMIT("io_inprocess_max=%d ", atomic_read(&rs->io.in_process_max));
-+
-+ for (sm = stats_map; sm < sm_end; sm++)
-+ DMEMIT("%s%d", sm->str, atomic_read(rs->stats + sm->type));
-+
-+ DMEMIT(" overwrite=%s ", RSCheckOverwrite(rs) ? "on" : "off");
-+ DMEMIT("sc=%u/%u/%u/%u/%u ", rs->set.chunk_size, rs->set.io_size,
-+ atomic_read(&rs->sc.stripes), rs->sc.hash.buckets,
-+ sc_size(rs));
-+
-+ j = (rec->end_jiffies ? rec->end_jiffies : jiffies) -
-+ rec->start_jiffies;
-+ jiffies_to_timespec(j, &ts);
-+ sprintf(buf, "%ld.%ld", ts.tv_sec, ts.tv_nsec);
-+ p = strchr(buf, '.');
-+ p[3] = 0;
-+
-+ DMEMIT("rg=%llu%s/%llu/%llu/%u %s ",
-+ (unsigned long long) rec->nr_regions_recovered,
-+ RSRegionGet(rs) ? "+" : "",
-+ (unsigned long long) rec->nr_regions_to_recover,
-+ (unsigned long long) rec->nr_regions, rec->bandwidth, buf);
-+
-+ rs_get_ra(rs, &stripes, &chunks);
-+ DMEMIT("ra=%u/%u ", stripes, chunks);
-+
-+ *size = sz;
-+}
-+
-+static int
-+raid_status(struct dm_target *ti, status_type_t type,
-+ char *result, unsigned maxlen)
-+{
-+ unsigned i, sz = 0;
-+ char buf[BDEVNAME_SIZE];
-+ struct raid_set *rs = ti->private;
-+
-+ switch (type) {
-+ case STATUSTYPE_INFO:
-+ /* REMOVEME: statistics. */
-+ if (RSDevelStats(rs))
-+ raid_devel_stats(ti, result, &sz, maxlen);
-+
-+ DMEMIT("%u ", rs->set.raid_devs);
-+
-+ for (i = 0; i < rs->set.raid_devs; i++)
-+ DMEMIT("%s ",
-+ format_dev_t(buf, rs->dev[i].dev->bdev->bd_dev));
-+
-+ DMEMIT("1 ");
-+ for (i = 0; i < rs->set.raid_devs; i++) {
-+ DMEMIT("%c", dev_operational(rs, i) ? 'A' : 'D');
-+
-+ if (rs->set.raid_type->level == raid4 &&
-+ i == rs->set.pi)
-+ DMEMIT("p");
-+
-+ if (rs->set.dev_to_init == i)
-+ DMEMIT("i");
-+ }
-+
-+ break;
-+
-+ case STATUSTYPE_TABLE:
-+ sz = rs->recover.dl->type->status(rs->recover.dl, type,
-+ result, maxlen);
-+ DMEMIT("%s %u ", rs->set.raid_type->name,
-+ rs->set.raid_parms);
-+
-+ if (rs->set.raid_type->level == raid4)
-+ DMEMIT("%d ", rs->set.pi_parm);
-+
-+ if (rs->set.raid_parms)
-+ DMEMIT("%d ", rs->set.chunk_size_parm);
-+
-+ if (rs->set.raid_parms > 1)
-+ DMEMIT("%d ", rs->sc.stripes_parm);
-+
-+ if (rs->set.raid_parms > 2)
-+ DMEMIT("%d ", rs->set.io_size_parm);
-+
-+ if (rs->set.raid_parms > 3)
-+ DMEMIT("%d ", rs->recover.io_size_parm);
-+
-+ if (rs->set.raid_parms > 4)
-+ DMEMIT("%d ", rs->recover.bandwidth_parm);
-+
-+ DMEMIT("%u %d ", rs->set.raid_devs, rs->set.dev_to_init);
-+
-+ for (i = 0; i < rs->set.raid_devs; i++)
-+ DMEMIT("%s %llu ",
-+ format_dev_t(buf,
-+ rs->dev[i].dev->bdev->bd_dev),
-+ (unsigned long long) rs->dev[i].start);
-+ }
-+
-+ return 0;
-+}
-+
-+/*
-+ * Message interface
-+ */
-+enum raid_msg_actions {
-+ act_bw, /* Recovery bandwidth switch. */
-+ act_dev, /* Device failure switch. */
-+ act_overwrite, /* Stripe overwrite check. */
-+ act_read_ahead, /* Set read ahead. */
-+ act_stats, /* Development statistics switch. */
-+ act_sc, /* Stripe cache switch. */
-+
-+ act_on, /* Set entity on. */
-+ act_off, /* Set entity off. */
-+ act_reset, /* Reset entity. */
-+
-+ act_set = act_on, /* Set # absolute. */
-+ act_grow = act_off, /* Grow # by an amount. */
-+ act_shrink = act_reset, /* Shrink # by an amount. */
-+};
-+
-+/* Turn a delta to absolute. */
-+static int _absolute(unsigned long action, int act, int r)
-+{
-+ /* Make delta absolute. */
-+ if (test_bit(act_set, &action))
-+ ;
-+ else if (test_bit(act_grow, &action))
-+ r += act;
-+ else if (test_bit(act_shrink, &action))
-+ r = act - r;
-+ else
-+ r = -EINVAL;
-+
-+ return r;
-+}
-+
-+ /* Change recovery io bandwidth. */
-+static int bandwidth_change(struct dm_msg *msg, void *context)
-+{
-+ struct raid_set *rs = context;
-+ int act = rs->recover.bandwidth;
-+ int bandwidth = DM_MSG_INT_ARG(msg);
-+
-+ if (range_ok(bandwidth, BANDWIDTH_MIN, BANDWIDTH_MAX)) {
-+ /* Make delta bandwidth absolute. */
-+ bandwidth = _absolute(msg->action, act, bandwidth);
-+
-+ /* Check range. */
-+ if (range_ok(bandwidth, BANDWIDTH_MIN, BANDWIDTH_MAX)) {
-+ recover_set_bandwidth(rs, bandwidth);
-+ return 0;
-+ }
-+ }
-+
-+ set_bit(dm_msg_ret_arg, &msg->ret);
-+ set_bit(dm_msg_ret_inval, &msg->ret);
-+ return -EINVAL;
-+}
-+
-+/* Change state of a device (running/offline). */
-+/* FIXME: this only works while recovering!. */
-+static int device_state(struct dm_msg *msg, void *context)
-+{
-+ int r;
-+ const char *str = "is already ";
-+ union dev_lookup dl = { .dev_name = DM_MSG_STR_ARG(msg) };
-+ struct raid_set *rs = context;
-+
-+ r = raid_dev_lookup(rs, strchr(dl.dev_name, ':') ?
-+ bymajmin : byname, &dl);
-+ if (r == -ENODEV) {
-+ DMERR("device %s is no member of this set", dl.dev_name);
-+ return r;
-+ }
-+
-+ if (test_bit(act_off, &msg->action)) {
-+ if (dev_operational(rs, r))
-+ str = "";
-+ } else if (!dev_operational(rs, r))
-+ str = "";
-+
-+ DMINFO("/dev/%s %s%s", dl.dev_name, str,
-+ test_bit(act_off, &msg->action) ? "offline" : "running");
-+
-+ return test_bit(act_off, &msg->action) ?
-+ raid_set_check_and_degrade(rs, NULL, r) :
-+ raid_set_check_and_upgrade(rs, r);
-+}
-+
-+/* Set/reset development feature flags. */
-+static int devel_flags(struct dm_msg *msg, void *context)
-+{
-+ struct raid_set *rs = context;
-+
-+ if (test_bit(act_on, &msg->action))
-+ return test_and_set_bit(msg->spec->parm,
-+ &rs->io.flags) ? -EPERM : 0;
-+ else if (test_bit(act_off, &msg->action))
-+ return test_and_clear_bit(msg->spec->parm,
-+ &rs->io.flags) ? 0 : -EPERM;
-+ else if (test_bit(act_reset, &msg->action)) {
-+ if (test_bit(act_stats, &msg->action)) {
-+ stats_reset(rs);
-+ goto on;
-+ } else if (test_bit(act_overwrite, &msg->action)) {
-+on:
-+ set_bit(msg->spec->parm, &rs->io.flags);
-+ return 0;
-+ }
-+ }
-+
-+ return -EINVAL;
-+}
-+
-+ /* Set stripe and chunk read ahead pages. */
-+static int read_ahead_set(struct dm_msg *msg, void *context)
-+{
-+ int stripes = DM_MSG_INT_ARGS(msg, 0);
-+ int chunks = DM_MSG_INT_ARGS(msg, 1);
-+
-+ if (range_ok(stripes, 1, 512) &&
-+ range_ok(chunks, 1, 512)) {
-+ rs_set_bdi(context, stripes, chunks);
-+ return 0;
-+ }
-+
-+ set_bit(dm_msg_ret_arg, &msg->ret);
-+ set_bit(dm_msg_ret_inval, &msg->ret);
-+ return -EINVAL;
-+}
-+
-+/* Resize the stripe cache. */
-+static int stripecache_resize(struct dm_msg *msg, void *context)
-+{
-+ int act, stripes;
-+ struct raid_set *rs = context;
-+
-+ /* Deny permission in case the daemon is still shrinking!. */
-+ if (atomic_read(&rs->sc.stripes_to_shrink))
-+ return -EPERM;
-+
-+ stripes = DM_MSG_INT_ARG(msg);
-+ if (stripes > 0) {
-+ act = atomic_read(&rs->sc.stripes);
-+
-+ /* Make delta stripes absolute. */
-+ stripes = _absolute(msg->action, act, stripes);
-+
-+ /*
-+ * Check range and that the # of stripes changes.
-+ * We can grow from gere but need to leave any
-+ * shrinking to the worker for synchronization.
-+ */
-+ if (range_ok(stripes, STRIPES_MIN, STRIPES_MAX)) {
-+ if (stripes > act)
-+ return sc_grow(&rs->sc, stripes - act, SC_GROW);
-+ else if (stripes < act) {
-+ atomic_set(&rs->sc.stripes_to_shrink,
-+ act - stripes);
-+ wake_do_raid(rs);
-+ }
-+
-+ return 0;
-+ }
-+ }
-+
-+ set_bit(dm_msg_ret_arg, &msg->ret);
-+ set_bit(dm_msg_ret_inval, &msg->ret);
-+ return -EINVAL;
-+}
-+
-+/* Parse the RAID message action. */
-+/*
-+ * 'ba[ndwidth] {se[t],g[row],sh[rink]} #' # e.g 'ba se 50'
-+ * 'de{vice] o[ffline]/r[unning] DevName/maj:min' # e.g 'device o /dev/sda'
-+ * "o[verwrite] {on,of[f],r[eset]}' # e.g. 'o of'
-+ * "r[ead_ahead] set #stripes #chunks # e.g. 'r se 3 2'
-+ * 'sta[tistics] {on,of[f],r[eset]}' # e.g. 'stat of'
-+ * 'str[ipecache] {se[t],g[row],sh[rink]} #' # e.g. 'stripe set 1024'
-+ *
-+ */
-+static int
-+raid_message(struct dm_target *ti, unsigned argc, char **argv)
-+{
-+ /* Variables to store the parsed parameters im. */
-+ static int i[2];
-+ static unsigned long *i_arg[] = {
-+ (unsigned long *) i + 0,
-+ (unsigned long *) i + 1,
-+ };
-+ static char *p;
-+ static unsigned long *p_arg[] = { (unsigned long *) &p };
-+
-+ /* Declare all message option strings. */
-+ static char *str_sgs[] = { "set", "grow", "shrink" };
-+ static char *str_dev[] = { "running", "offline" };
-+ static char *str_oor[] = { "on", "off", "reset" };
-+
-+ /* Declare all actions. */
-+ static unsigned long act_sgs[] = { act_set, act_grow, act_shrink };
-+ static unsigned long act_oor[] = { act_on, act_off, act_reset };
-+
-+ /* Bandwidth option. */
-+ static struct dm_message_option bw_opt = { 3, str_sgs, act_sgs };
-+ static struct dm_message_argument bw_args = {
-+ 1, i_arg, { dm_msg_int_t }
-+ };
-+
-+ /* Device option. */
-+ static struct dm_message_option dev_opt = { 2, str_dev, act_oor };
-+ static struct dm_message_argument dev_args = {
-+ 1, p_arg, { dm_msg_base_t }
-+ };
-+
-+ /* Read ahead option. */
-+ static struct dm_message_option ra_opt = { 1, str_sgs, act_sgs };
-+ static struct dm_message_argument ra_args = {
-+ 2, i_arg, { dm_msg_int_t, dm_msg_int_t }
-+ };
-+
-+ static struct dm_message_argument null_args = {
-+ 0, NULL, { dm_msg_int_t }
-+ };
-+
-+ /* Overwrite and statistics option. */
-+ static struct dm_message_option ovr_stats_opt = { 3, str_oor, act_oor };
-+
-+ /* Sripecache option. */
-+ static struct dm_message_option stripe_opt = { 3, str_sgs, act_sgs };
-+
-+ /* Declare messages. */
-+ static struct dm_msg_spec specs[] = {
-+ { "bandwidth", act_bw, &bw_opt, &bw_args,
-+ 0, bandwidth_change },
-+ { "device", act_dev, &dev_opt, &dev_args,
-+ 0, device_state },
-+ { "overwrite", act_overwrite, &ovr_stats_opt, &null_args,
-+ RS_CHECK_OVERWRITE, devel_flags },
-+ { "read_ahead", act_read_ahead, &ra_opt, &ra_args,
-+ 0, read_ahead_set },
-+ { "statistics", act_stats, &ovr_stats_opt, &null_args,
-+ RS_DEVEL_STATS, devel_flags },
-+ { "stripecache", act_sc, &stripe_opt, &bw_args,
-+ 0, stripecache_resize },
-+ };
-+
-+ /* The message for the parser. */
-+ struct dm_msg msg = {
-+ .num_specs = ARRAY_SIZE(specs),
-+ .specs = specs,
-+ };
-+
-+ return dm_message_parse(TARGET, &msg, ti->private, argc, argv);
-+}
-+/*
-+ * END message interface
-+ */
-+
-+static struct target_type raid_target = {
-+ .name = "raid45",
-+ .version = {1, 0, 0},
-+ .module = THIS_MODULE,
-+ .ctr = raid_ctr,
-+ .dtr = raid_dtr,
-+ .map = raid_map,
-+ .postsuspend = raid_postsuspend,
-+ .resume = raid_resume,
-+ .status = raid_status,
-+ .message = raid_message,
-+};
-+
-+static void init_exit(const char *bad_msg, const char *good_msg, int r)
-+{
-+ if (r)
-+ DMERR("Failed to %sregister target [%d]", bad_msg, r);
-+ else
-+ DMINFO("%s %s", good_msg, version);
-+}
-+
-+static int __init dm_raid_init(void)
-+{
-+ int r;
-+
-+ r = dm_register_target(&raid_target);
-+ init_exit("", "initialized", r);
-+ return r;
-+}
-+
-+static void __exit dm_raid_exit(void)
-+{
-+ int r;
-+
-+ r = dm_unregister_target(&raid_target);
-+ init_exit("un", "exit", r);
-+}
-+
-+/* Module hooks. */
-+module_init(dm_raid_init);
-+module_exit(dm_raid_exit);
-+
-+MODULE_DESCRIPTION(DM_NAME " raid4/5 target");
-+MODULE_AUTHOR("Heinz Mauelshagen <hjm@redhat.com>");
-+MODULE_LICENSE("GPL");
---- /dev/null
-+++ b/drivers/md/dm-raid45.h
-@@ -0,0 +1,28 @@
-+/*
-+ * Copyright (C) 2006-2008 Red Hat, Inc. All rights reserved.
-+ *
-+ * Module Author: Heinz Mauelshagen (Mauelshagen@RedHat.com)
-+ *
-+ * Locking definitions for the device-mapper RAID45 target.
-+ *
-+ * This file is released under the GPL.
-+ *
-+ */
-+
-+#ifndef _DM_RAID45_H
-+#define _DM_RAID45_H
-+
-+/* Factor out to dm.h! */
-+#define STR_LEN(ptr, str) (ptr), (str), strlen((ptr))
-+
-+enum dm_lock_type { DM_RAID45_EX, DM_RAID45_SHARED };
-+
-+struct dm_raid45_locking_type {
-+ /* Request a lock on a stripe. */
-+ void* (*lock)(sector_t key, enum dm_lock_type type);
-+
-+ /* Release a lock on a stripe. */
-+ void (*unlock)(void *lock_handle);
-+};
-+
-+#endif
---- /dev/null
-+++ b/drivers/md/dm-regions.c
-@@ -0,0 +1,723 @@
-+/*
-+ * Copyright (C) 2003 Sistina Software Limited.
-+ * Copyright (C) 2004-2008 Red Hat, Inc. All rights reserved.
-+ *
-+ * This file is released under the GPL.
-+ */
-+
-+#include <linux/dm-dirty-log.h>
-+#include <linux/dm-regions.h>
-+
-+#include <linux/ctype.h>
-+#include <linux/init.h>
-+#include <linux/module.h>
-+#include <linux/vmalloc.h>
-+
-+#include "dm.h"
-+#include "dm-bio-list.h"
-+
-+#define DM_MSG_PREFIX "region hash"
-+
-+/*-----------------------------------------------------------------
-+ * Region hash
-+ *
-+ * A storage set (eg. RAID1, RAID5) splits itself up into discrete regions.
-+ * Each region can be in one of three states:
-+ *
-+ * o clean
-+ * o dirty,
-+ * o nosync.
-+ *
-+ * There is no need to put clean regions in the hash.
-+ *
-+ *
-+ * In addition to being present in the hash table a region _may_
-+ * be present on one of three lists.
-+ *
-+ * clean_regions: Regions on this list have no io pending to
-+ * them, they are in sync, we are no longer interested in them,
-+ * they are dull. dm_rh_update_states() will remove them from the
-+ * hash table.
-+ *
-+ * quiesced_regions: These regions have been spun down, ready
-+ * for recovery. dm_rh_recovery_start() will remove regions from
-+ * this list and hand them to the caller, which will schedule the
-+ * recovery io.
-+ *
-+ * recovered_regions: Regions that the caller has successfully
-+ * recovered. dm_rh_update_states() will now schedule any delayed
-+ * io, up the recovery_count, and remove the region from the hash.
-+ *
-+ * There are 2 locks:
-+ * A rw spin lock 'hash_lock' protects just the hash table,
-+ * this is never held in write mode from interrupt context,
-+ * which I believe means that we only have to disable irqs when
-+ * doing a write lock.
-+ *
-+ * An ordinary spin lock 'region_lock' that protects the three
-+ * lists in the region_hash, with the 'state', 'list' and
-+ * 'delayed_bios' fields of the regions. This is used from irq
-+ * context, so all other uses will have to suspend local irqs.
-+ *---------------------------------------------------------------*/
-+struct region_hash {
-+ unsigned max_recovery; /* Max # of regions to recover in parallel */
-+
-+ /* Callback function to dispatch queued writes on recovered regions. */
-+ void (*dispatch)(void *context, struct bio_list *bios, int error);
-+ void *dispatch_context;
-+
-+ /* Callback function to wakeup callers worker thread. */
-+ void (*wake)(void *context);
-+ void *wake_context;
-+
-+ uint32_t region_size;
-+ unsigned region_shift;
-+
-+ /* holds persistent region state */
-+ struct dm_dirty_log *log;
-+
-+ /* hash table */
-+ rwlock_t hash_lock;
-+ mempool_t *region_pool;
-+ unsigned mask;
-+ unsigned nr_buckets;
-+ unsigned prime;
-+ unsigned shift;
-+ struct list_head *buckets;
-+
-+ spinlock_t region_lock;
-+ atomic_t recovery_in_flight;
-+ struct semaphore recovery_count;
-+ struct list_head clean_regions;
-+ struct list_head quiesced_regions;
-+ struct list_head recovered_regions;
-+ struct list_head failed_recovered_regions;
-+};
-+
-+struct region {
-+ region_t key;
-+ enum dm_rh_region_states state;
-+ void *context; /* Caller context. */
-+
-+ struct list_head hash_list;
-+ struct list_head list;
-+
-+ atomic_t pending;
-+ struct bio_list delayed_bios;
-+};
-+
-+/*
-+ * Conversion fns
-+ */
-+region_t dm_rh_sector_to_region(struct dm_rh_client *rh, sector_t sector)
-+{
-+ return sector >> ((struct region_hash *) rh)->region_shift;
-+}
-+EXPORT_SYMBOL_GPL(dm_rh_sector_to_region);
-+
-+region_t dm_rh_bio_to_region(struct dm_rh_client *rh, struct bio *bio)
-+{
-+ return dm_rh_sector_to_region(rh, bio->bi_sector);
-+}
-+EXPORT_SYMBOL_GPL(dm_rh_bio_to_region);
-+
-+sector_t dm_rh_region_to_sector(struct dm_rh_client *rh, region_t region)
-+{
-+ return region << ((struct region_hash *) rh)->region_shift;
-+}
-+EXPORT_SYMBOL_GPL(dm_rh_region_to_sector);
-+
-+/*
-+ * Retrival fns.
-+ */
-+region_t dm_rh_get_region_key(struct dm_region *reg)
-+{
-+ return ((struct region *) reg)->key;
-+}
-+EXPORT_SYMBOL_GPL(dm_rh_get_region_key);
-+
-+sector_t dm_rh_get_region_size(struct dm_rh_client *rh)
-+{
-+ return ((struct region_hash *) rh)->region_size;
-+}
-+EXPORT_SYMBOL_GPL(dm_rh_get_region_size);
-+
-+/* Squirrel a context with a region. */
-+void *dm_rh_reg_get_context(struct dm_region *reg)
-+{
-+ return ((struct region *) reg)->context;
-+}
-+EXPORT_SYMBOL_GPL(dm_rh_reg_get_context);
-+
-+void dm_rh_reg_set_context(struct dm_region *reg, void *context)
-+{
-+ ((struct region *) reg)->context = context;
-+}
-+EXPORT_SYMBOL_GPL(dm_rh_reg_set_context);
-+
-+/*
-+ * Create region hash client.
-+ */
-+#define MIN_REGIONS 64
-+struct dm_rh_client *dm_rh_client_create(
-+ unsigned max_recovery,
-+ void (*dispatch)(void *dispatch_context,
-+ struct bio_list *bios, int error),
-+ void *dispatch_context,
-+ void (*wake)(void *wake_context), void *wake_context,
-+ struct dm_dirty_log *log, uint32_t region_size,
-+ region_t nr_regions)
-+{
-+ unsigned i;
-+ unsigned nr_buckets, max_buckets;
-+ unsigned hash_primes[] = {
-+ /* Table of primes for rh_hash/table size optimization. */
-+ 3, 7, 13, 27, 53, 97, 193, 389, 769,
-+ 1543, 3079, 6151, 12289, 24593,
-+ };
-+ struct region_hash *rh;
-+
-+ if (region_size & (region_size - 1)) {
-+ DMERR("region size must be 2^^n");
-+ return ERR_PTR(-EINVAL);
-+ }
-+
-+ /* Calculate a suitable number of buckets for our hash table. */
-+ max_buckets = nr_regions >> 6;
-+ for (nr_buckets = 128u; nr_buckets < max_buckets; nr_buckets <<= 1)
-+ ;
-+ nr_buckets >>= 1;
-+
-+ rh = kmalloc(sizeof(*rh), GFP_KERNEL);
-+ if (!rh) {
-+ DMERR("unable to allocate region hash memory");
-+ return ERR_PTR(-ENOMEM);
-+ }
-+
-+ rh->max_recovery = max_recovery;
-+ rh->dispatch = dispatch;
-+ rh->dispatch_context = dispatch_context;
-+ rh->wake = wake;
-+ rh->wake_context = wake_context;
-+ rh->log = log;
-+ rh->region_size = region_size;
-+ rh->region_shift = ffs(region_size) - 1;
-+ rwlock_init(&rh->hash_lock);
-+ rh->mask = nr_buckets - 1;
-+ rh->nr_buckets = nr_buckets;
-+ rh->shift = ffs(nr_buckets);
-+
-+ /* Check prime array limits. */
-+ i = rh->shift - 1 > ARRAY_SIZE(hash_primes) ?
-+ ARRAY_SIZE(hash_primes) - 1 : rh->shift - 2;
-+ rh->prime = hash_primes[i];
-+
-+ rh->buckets = vmalloc(nr_buckets * sizeof(*rh->buckets));
-+ if (!rh->buckets) {
-+ DMERR("unable to allocate region hash bucket memory");
-+ kfree(rh);
-+ return ERR_PTR(-ENOMEM);
-+ }
-+
-+ for (i = 0; i < nr_buckets; i++)
-+ INIT_LIST_HEAD(rh->buckets + i);
-+
-+ spin_lock_init(&rh->region_lock);
-+ sema_init(&rh->recovery_count, 0);
-+ atomic_set(&rh->recovery_in_flight, 0);
-+ INIT_LIST_HEAD(&rh->clean_regions);
-+ INIT_LIST_HEAD(&rh->quiesced_regions);
-+ INIT_LIST_HEAD(&rh->recovered_regions);
-+ INIT_LIST_HEAD(&rh->failed_recovered_regions);
-+
-+ rh->region_pool = mempool_create_kmalloc_pool(MIN_REGIONS,
-+ sizeof(struct region));
-+ if (!rh->region_pool) {
-+ vfree(rh->buckets);
-+ kfree(rh);
-+ rh = ERR_PTR(-ENOMEM);
-+ }
-+
-+ return (struct dm_rh_client *) rh;
-+}
-+EXPORT_SYMBOL_GPL(dm_rh_client_create);
-+
-+void dm_rh_client_destroy(struct dm_rh_client *rh_in)
-+{
-+ unsigned h;
-+ struct region_hash *rh = (struct region_hash *) rh_in;
-+ struct region *reg, *tmp;
-+
-+ BUG_ON(!list_empty(&rh->quiesced_regions));
-+
-+ for (h = 0; h < rh->nr_buckets; h++) {
-+ list_for_each_entry_safe(reg, tmp, rh->buckets + h, hash_list) {
-+ BUG_ON(atomic_read(®->pending));
-+ mempool_free(reg, rh->region_pool);
-+ }
-+ }
-+
-+ if (rh->region_pool)
-+ mempool_destroy(rh->region_pool);
-+
-+ vfree(rh->buckets);
-+ kfree(rh);
-+}
-+EXPORT_SYMBOL_GPL(dm_rh_client_destroy);
-+
-+static inline unsigned rh_hash(struct region_hash *rh, region_t region)
-+{
-+ return (unsigned) ((region * rh->prime) >> rh->shift) & rh->mask;
-+}
-+
-+static struct region *__rh_lookup(struct region_hash *rh, region_t region)
-+{
-+ struct region *reg;
-+ struct list_head *bucket = rh->buckets + rh_hash(rh, region);
-+
-+ list_for_each_entry(reg, bucket, hash_list) {
-+ if (reg->key == region)
-+ return reg;
-+ }
-+
-+ return NULL;
-+}
-+
-+static void __rh_insert(struct region_hash *rh, struct region *reg)
-+{
-+ list_add(®->hash_list, rh->buckets + rh_hash(rh, reg->key));
-+}
-+
-+static struct region *__rh_alloc(struct region_hash *rh, region_t region)
-+{
-+ struct region *reg, *nreg;
-+
-+ read_unlock(&rh->hash_lock);
-+ nreg = mempool_alloc(rh->region_pool, GFP_ATOMIC);
-+ if (unlikely(!nreg))
-+ nreg = kmalloc(sizeof(*nreg), GFP_NOIO);
-+
-+ nreg->state = rh->log->type->in_sync(rh->log, region, 1) ?
-+ DM_RH_CLEAN : DM_RH_NOSYNC;
-+ nreg->key = region;
-+ INIT_LIST_HEAD(&nreg->list);
-+ atomic_set(&nreg->pending, 0);
-+ bio_list_init(&nreg->delayed_bios);
-+
-+ write_lock_irq(&rh->hash_lock);
-+ reg = __rh_lookup(rh, region);
-+ if (reg)
-+ /* We lost the race. */
-+ mempool_free(nreg, rh->region_pool);
-+ else {
-+ __rh_insert(rh, nreg);
-+ if (nreg->state == DM_RH_CLEAN) {
-+ spin_lock(&rh->region_lock);
-+ list_add(&nreg->list, &rh->clean_regions);
-+ spin_unlock(&rh->region_lock);
-+ }
-+
-+ reg = nreg;
-+ }
-+
-+ write_unlock_irq(&rh->hash_lock);
-+ read_lock(&rh->hash_lock);
-+ return reg;
-+}
-+
-+static inline struct region *__rh_find(struct region_hash *rh, region_t region)
-+{
-+ struct region *reg;
-+
-+ reg = __rh_lookup(rh, region);
-+ return reg ? reg : __rh_alloc(rh, region);
-+}
-+
-+int dm_rh_get_state(struct dm_rh_client *rh_in, region_t region, int may_block)
-+{
-+ int r;
-+ struct region_hash *rh = (struct region_hash *) rh_in;
-+ struct region *reg;
-+
-+ read_lock(&rh->hash_lock);
-+ reg = __rh_lookup(rh, region);
-+ read_unlock(&rh->hash_lock);
-+
-+ if (reg)
-+ return reg->state;
-+
-+ /*
-+ * The region wasn't in the hash, so we fall back to the dirty log.
-+ */
-+ r = rh->log->type->in_sync(rh->log, region, may_block);
-+
-+ /*
-+ * Any error from the dirty log (eg. -EWOULDBLOCK)
-+ * gets taken as a DM_RH_NOSYNC
-+ */
-+ return r == 1 ? DM_RH_CLEAN : DM_RH_NOSYNC;
-+}
-+EXPORT_SYMBOL_GPL(dm_rh_get_state);
-+
-+void dm_rh_set_state(struct dm_rh_client *rh_in, region_t region,
-+ enum dm_rh_region_states state, int may_block)
-+{
-+ struct region_hash *rh = (struct region_hash *) rh_in;
-+ struct region *reg;
-+ struct dm_dirty_log *log = rh->log;
-+
-+ if (state == DM_RH_NOSYNC)
-+ log->type->set_region_sync(log, region, 0);
-+ else if (state == DM_RH_CLEAN)
-+ log->type->clear_region(log, region);
-+ else if (state == DM_RH_DIRTY)
-+ log->type->mark_region(log, region);
-+
-+ read_lock(&rh->hash_lock);
-+ reg = __rh_find(rh, region);
-+ reg->state = state;
-+ read_unlock(&rh->hash_lock);
-+}
-+EXPORT_SYMBOL_GPL(dm_rh_set_state);
-+
-+void dm_rh_update_states(struct dm_rh_client *rh_in, int errors_handled)
-+{
-+ struct region_hash *rh = (struct region_hash *) rh_in;
-+ struct region *reg, *next;
-+ LIST_HEAD(clean);
-+ LIST_HEAD(recovered);
-+ LIST_HEAD(failed_recovered);
-+
-+ /*
-+ * Quickly grab the lists and remove any regions from hash.
-+ */
-+ write_lock_irq(&rh->hash_lock);
-+ spin_lock(&rh->region_lock);
-+ if (!list_empty(&rh->clean_regions)) {
-+ list_splice_init(&rh->clean_regions, &clean);
-+
-+ list_for_each_entry(reg, &clean, list)
-+ list_del(®->hash_list);
-+ }
-+
-+ if (!list_empty(&rh->recovered_regions)) {
-+ list_splice_init(&rh->recovered_regions, &recovered);
-+
-+ list_for_each_entry(reg, &recovered, list)
-+ list_del(®->hash_list);
-+ }
-+
-+ if (!list_empty(&rh->failed_recovered_regions)) {
-+ list_splice_init(&rh->failed_recovered_regions,
-+ &failed_recovered);
-+
-+ list_for_each_entry(reg, &recovered, list)
-+ list_del(®->hash_list);
-+ }
-+
-+ spin_unlock(&rh->region_lock);
-+ write_unlock_irq(&rh->hash_lock);
-+
-+ /*
-+ * All the regions on the recovered and clean lists have
-+ * now been pulled out of the system, so no need to do
-+ * any more locking.
-+ */
-+ list_for_each_entry_safe(reg, next, &recovered, list) {
-+ rh->log->type->clear_region(rh->log, reg->key);
-+ rh->log->type->set_region_sync(rh->log, reg->key, 1);
-+
-+ if (reg->delayed_bios.head)
-+ rh->dispatch(rh->dispatch_context,
-+ ®->delayed_bios, 0);
-+
-+ up(&rh->recovery_count);
-+ mempool_free(reg, rh->region_pool);
-+ }
-+
-+ list_for_each_entry_safe(reg, next, &failed_recovered, list) {
-+ rh->log->type->set_region_sync(rh->log, reg->key,
-+ errors_handled ? 0 : 1);
-+ if (reg->delayed_bios.head)
-+ rh->dispatch(rh->dispatch_context,
-+ ®->delayed_bios, -EIO);
-+
-+ up(&rh->recovery_count);
-+ mempool_free(reg, rh->region_pool);
-+ }
-+
-+ list_for_each_entry_safe(reg, next, &clean, list) {
-+ rh->log->type->clear_region(rh->log, reg->key);
-+ mempool_free(reg, rh->region_pool);
-+ }
-+
-+ dm_rh_flush(rh_in);
-+}
-+EXPORT_SYMBOL_GPL(dm_rh_update_states);
-+
-+void dm_rh_inc(struct dm_rh_client *rh_in, region_t region)
-+{
-+ struct region_hash *rh = (struct region_hash *) rh_in;
-+ struct region *reg;
-+
-+ read_lock(&rh->hash_lock);
-+ reg = __rh_find(rh, region);
-+ if (reg->state == DM_RH_CLEAN) {
-+ rh->log->type->mark_region(rh->log, reg->key);
-+
-+ spin_lock_irq(&rh->region_lock);
-+ reg->state = DM_RH_DIRTY;
-+ list_del_init(®->list); /* Take off the clean list. */
-+ spin_unlock_irq(&rh->region_lock);
-+ }
-+
-+ atomic_inc(®->pending);
-+ read_unlock(&rh->hash_lock);
-+}
-+EXPORT_SYMBOL_GPL(dm_rh_inc);
-+
-+void dm_rh_inc_pending(struct dm_rh_client *rh_in, struct bio_list *bios)
-+{
-+ struct bio *bio;
-+
-+ for (bio = bios->head; bio; bio = bio->bi_next)
-+ dm_rh_inc(rh_in, dm_rh_bio_to_region(rh_in, bio));
-+}
-+EXPORT_SYMBOL_GPL(dm_rh_inc_pending);
-+
-+int dm_rh_dec(struct dm_rh_client *rh_in, region_t region)
-+{
-+ int r = 0;
-+ struct region_hash *rh = (struct region_hash *) rh_in;
-+ struct region *reg;
-+
-+ read_lock(&rh->hash_lock);
-+ reg = __rh_lookup(rh, region);
-+ read_unlock(&rh->hash_lock);
-+
-+ BUG_ON(!reg);
-+
-+ if (atomic_dec_and_test(®->pending)) {
-+ unsigned long flags;
-+
-+ /*
-+ * There is no pending I/O for this region.
-+ * We can move the region to corresponding list for next action.
-+ * At this point, the region is not yet connected to any list.
-+ *
-+ * If the state is DM_RH_NOSYNC, the region should be kept off
-+ * from clean list.
-+ * The hash entry for DM_RH_NOSYNC will remain in memory
-+ * until the region is recovered or the map is reloaded.
-+ */
-+
-+ spin_lock_irqsave(&rh->region_lock, flags);
-+ if (reg->state == DM_RH_RECOVERING)
-+ list_add_tail(®->list, &rh->quiesced_regions);
-+ else {
-+ reg->state = DM_RH_CLEAN;
-+ list_add(®->list, &rh->clean_regions);
-+ }
-+ spin_unlock_irqrestore(&rh->region_lock, flags);
-+
-+ r = 1;
-+ }
-+
-+ return r;
-+}
-+EXPORT_SYMBOL_GPL(dm_rh_dec);
-+
-+/*
-+ * Starts quiescing a region in preparation for recovery.
-+ */
-+static int __rh_recovery_prepare(struct region_hash *rh)
-+{
-+ int r;
-+ region_t region;
-+ struct region *reg;
-+
-+ /*
-+ * Ask the dirty log what's next.
-+ */
-+ r = rh->log->type->get_resync_work(rh->log, ®ion);
-+ if (r <= 0)
-+ return r;
-+
-+ /*
-+ * Get this region, and start it quiescing
-+ * by setting the recovering flag.
-+ */
-+ read_lock(&rh->hash_lock);
-+ reg = __rh_find(rh, region);
-+ read_unlock(&rh->hash_lock);
-+
-+ spin_lock_irq(&rh->region_lock);
-+
-+ reg->state = DM_RH_RECOVERING;
-+
-+ /* Already quiesced ? */
-+ list_del_init(®->list);
-+ if (!atomic_read(®->pending))
-+ list_add(®->list, &rh->quiesced_regions);
-+
-+ spin_unlock_irq(&rh->region_lock);
-+ return 1;
-+}
-+
-+int dm_rh_recovery_prepare(struct dm_rh_client *rh_in)
-+{
-+ int r = 0;
-+ struct region_hash *rh = (struct region_hash *) rh_in;
-+
-+ /* Extra reference to avoid race with rh_stop_recovery */
-+ atomic_inc(&rh->recovery_in_flight);
-+
-+ while (!down_trylock(&rh->recovery_count)) {
-+ atomic_inc(&rh->recovery_in_flight);
-+
-+ if (__rh_recovery_prepare(rh) <= 0) {
-+ atomic_dec(&rh->recovery_in_flight);
-+ up(&rh->recovery_count);
-+ r = -ENOENT;
-+ break;
-+ }
-+ }
-+
-+ /* Drop the extra reference. */
-+ if (atomic_dec_and_test(&rh->recovery_in_flight))
-+ r = -ESRCH;
-+
-+ return r;
-+}
-+EXPORT_SYMBOL_GPL(dm_rh_recovery_prepare);
-+
-+/*
-+ * Returns any quiesced regions.
-+ */
-+struct dm_region *dm_rh_recovery_start(struct dm_rh_client *rh_in)
-+{
-+ struct region_hash *rh = (struct region_hash *) rh_in;
-+ struct region *reg = NULL;
-+
-+ spin_lock_irq(&rh->region_lock);
-+ if (!list_empty(&rh->quiesced_regions)) {
-+ reg = list_entry(rh->quiesced_regions.next,
-+ struct region, list);
-+ list_del_init(®->list); /* Remove from the quiesced list. */
-+ }
-+
-+ spin_unlock_irq(&rh->region_lock);
-+ return (struct dm_region *) reg;
-+}
-+EXPORT_SYMBOL_GPL(dm_rh_recovery_start);
-+
-+/*
-+ * Put region on list of recovered ones.
-+ */
-+void dm_rh_recovery_end(struct dm_rh_client *rh_in, struct dm_region *reg_in,
-+ int error)
-+{
-+ struct region_hash *rh = (struct region_hash *) rh_in;
-+ struct region *reg = (struct region *) reg_in;
-+
-+ spin_lock_irq(&rh->region_lock);
-+ if (error) {
-+ reg->state = DM_RH_NOSYNC;
-+ list_add(®->list, &rh->failed_recovered_regions);
-+ } else
-+ list_add(®->list, &rh->recovered_regions);
-+
-+ atomic_dec(&rh->recovery_in_flight);
-+ spin_unlock_irq(&rh->region_lock);
-+
-+ rh->wake(rh->wake_context);
-+ BUG_ON(atomic_read(&rh->recovery_in_flight) < 0);
-+}
-+EXPORT_SYMBOL_GPL(dm_rh_recovery_end);
-+
-+/* Return recovery in flight count. */
-+int dm_rh_recovery_in_flight(struct dm_rh_client *rh_in)
-+{
-+ return atomic_read(&((struct region_hash *) rh_in)->recovery_in_flight);
-+}
-+EXPORT_SYMBOL_GPL(dm_rh_recovery_in_flight);
-+
-+int dm_rh_flush(struct dm_rh_client *rh_in)
-+{
-+ struct region_hash *rh = (struct region_hash *) rh_in;
-+
-+ return rh->log->type->flush(rh->log);
-+}
-+EXPORT_SYMBOL_GPL(dm_rh_flush);
-+
-+void dm_rh_delay_by_region(struct dm_rh_client *rh_in,
-+ struct bio *bio, region_t region)
-+{
-+ struct region_hash *rh = (struct region_hash *) rh_in;
-+ struct region *reg;
-+
-+ /* FIXME: locking. */
-+ read_lock(&rh->hash_lock);
-+ reg = __rh_find(rh, region);
-+ bio_list_add(®->delayed_bios, bio);
-+ read_unlock(&rh->hash_lock);
-+}
-+EXPORT_SYMBOL_GPL(dm_rh_delay_by_region);
-+
-+void dm_rh_delay(struct dm_rh_client *rh_in, struct bio *bio)
-+{
-+ return dm_rh_delay_by_region(rh_in, bio,
-+ dm_rh_bio_to_region(rh_in, bio));
-+}
-+EXPORT_SYMBOL_GPL(dm_rh_delay);
-+
-+void dm_rh_dispatch_bios(struct dm_rh_client *rh_in,
-+ region_t region, int error)
-+{
-+ struct region_hash *rh = (struct region_hash *) rh_in;
-+ struct region *reg;
-+ struct bio_list delayed_bios;
-+
-+ /* FIXME: locking. */
-+ read_lock(&rh->hash_lock);
-+ reg = __rh_find(rh, region);
-+ BUG_ON(!reg);
-+ delayed_bios = reg->delayed_bios;
-+ bio_list_init(®->delayed_bios);
-+ read_unlock(&rh->hash_lock);
-+
-+ if (delayed_bios.head)
-+ rh->dispatch(rh->dispatch_context, &delayed_bios, error);
-+
-+ up(&rh->recovery_count);
-+}
-+EXPORT_SYMBOL_GPL(dm_rh_dispatch_bios);
-+
-+void dm_rh_stop_recovery(struct dm_rh_client *rh_in)
-+{
-+ int i;
-+ struct region_hash *rh = (struct region_hash *) rh_in;
-+
-+ rh->wake(rh->wake_context);
-+
-+ /* wait for any recovering regions */
-+ for (i = 0; i < rh->max_recovery; i++)
-+ down(&rh->recovery_count);
-+}
-+EXPORT_SYMBOL_GPL(dm_rh_stop_recovery);
-+
-+void dm_rh_start_recovery(struct dm_rh_client *rh_in)
-+{
-+ int i;
-+ struct region_hash *rh = (struct region_hash *) rh_in;
-+
-+ for (i = 0; i < rh->max_recovery; i++)
-+ up(&rh->recovery_count);
-+
-+ rh->wake(rh->wake_context);
-+}
-+EXPORT_SYMBOL_GPL(dm_rh_start_recovery);
-+
-+MODULE_DESCRIPTION(DM_NAME " region hash");
-+MODULE_AUTHOR("Joe Thornber/Heinz Mauelshagen <hjm@redhat.com>");
-+MODULE_LICENSE("GPL");
---- a/drivers/md/Kconfig
-+++ b/drivers/md/Kconfig
-@@ -269,6 +269,14 @@ config DM_DELAY
-
- If unsure, say N.
-
-+config DM_RAID45
-+ tristate "RAID 4/5 target (EXPERIMENTAL)"
-+ depends on BLK_DEV_DM && EXPERIMENTAL
-+ ---help---
-+ A target that supports RAID4 and RAID5 mappings.
-+
-+ If unsure, say N.
-+
- config DM_UEVENT
- bool "DM uevents (EXPERIMENTAL)"
- depends on BLK_DEV_DM && EXPERIMENTAL
---- a/drivers/md/Makefile
-+++ b/drivers/md/Makefile
-@@ -34,7 +34,9 @@ obj-$(CONFIG_DM_CRYPT) += dm-crypt.o
- obj-$(CONFIG_DM_DELAY) += dm-delay.o
- obj-$(CONFIG_DM_MULTIPATH) += dm-multipath.o dm-round-robin.o
- obj-$(CONFIG_DM_SNAPSHOT) += dm-snapshot.o
--obj-$(CONFIG_DM_MIRROR) += dm-mirror.o dm-log.o
-+obj-$(CONFIG_DM_MIRROR) += dm-mirror.o dm-regions.o dm-log.o
-+obj-$(CONFIG_DM_RAID45) += dm-raid45.o dm-log.o dm-memcache.o \
-+ dm-regions.o dm-message.o
- obj-$(CONFIG_DM_ZERO) += dm-zero.o
-
- quiet_cmd_unroll = UNROLL $@
---- /dev/null
-+++ b/include/linux/dm-regions.h
-@@ -0,0 +1,115 @@
-+/*
-+ * Copyright (C) 2003 Sistina Software Limited.
-+ * Copyright (C) 2004-2008 Red Hat, Inc. All rights reserved.
-+ *
-+ * Device-Mapper dirty region hash interface.
-+ *
-+ * This file is released under the GPL.
-+ */
-+
-+#ifndef DM_REGION_HASH_H
-+#define DM_REGION_HASH_H
-+
-+#include <linux/dm-dirty-log.h>
-+
-+/*-----------------------------------------------------------------
-+ * Region hash
-+ *----------------------------------------------------------------*/
-+struct dm_rh_client;
-+struct dm_region;
-+
-+/*
-+ * States a region can have.
-+ */
-+enum dm_rh_region_states {
-+ DM_RH_CLEAN = 0x01, /* No writes in flight. */
-+ DM_RH_DIRTY = 0x02, /* Writes in flight. */
-+ DM_RH_NOSYNC = 0x04, /* Out of sync. */
-+ DM_RH_RECOVERING = 0x08, /* Under resynchronization. */
-+};
-+
-+/*
-+ * Region hash create/destroy.
-+ */
-+struct bio_list;
-+struct dm_rh_client *dm_rh_client_create(
-+ unsigned max_recovery,
-+ void (*dispatch)(void *dispatch_context,
-+ struct bio_list *bios, int error),
-+ void *dispatch_context,
-+ void (*wake)(void *wake_context), void *wake_context,
-+ struct dm_dirty_log *log, uint32_t region_size,
-+ region_t nr_regions);
-+void dm_rh_client_destroy(struct dm_rh_client *rh);
-+
-+/*
-+ * Conversion fns:
-+ *
-+ * bio -> region
-+ * sector -> region
-+ * region -> sector
-+ */
-+region_t dm_rh_bio_to_region(struct dm_rh_client *rh, struct bio *bio);
-+region_t dm_rh_sector_to_region(struct dm_rh_client *rh, sector_t sector);
-+sector_t dm_rh_region_to_sector(struct dm_rh_client *rh, region_t region);
-+
-+/*
-+ * Functions to set a caller context in a region.
-+ */
-+void *dm_rh_reg_get_context(struct dm_region *reg);
-+void dm_rh_reg_set_context(struct dm_region *reg, void *context);
-+
-+/*
-+ * Get region size and key (ie. number of the region).
-+ */
-+sector_t dm_rh_get_region_size(struct dm_rh_client *rh);
-+sector_t dm_rh_get_region_key(struct dm_region *reg);
-+
-+/*
-+ * Get/set/update region state (and dirty log).
-+ *
-+ * dm_rh_update_states
-+ * @errors_handled != 0 influences
-+ * that the state of the region will be kept NOSYNC
-+ */
-+int dm_rh_get_state(struct dm_rh_client *rh, region_t region, int may_block);
-+void dm_rh_set_state(struct dm_rh_client *rh, region_t region,
-+ enum dm_rh_region_states state, int may_block);
-+void dm_rh_update_states(struct dm_rh_client *rh, int errors_handled);
-+
-+/* Flush the region hash and dirty log. */
-+int dm_rh_flush(struct dm_rh_client *rh);
-+
-+/* Inc/dec pending count on regions. */
-+void dm_rh_inc(struct dm_rh_client *rh, region_t region);
-+void dm_rh_inc_pending(struct dm_rh_client *rh, struct bio_list *bios);
-+int dm_rh_dec(struct dm_rh_client *rh, region_t region);
-+
-+/* Delay bios on regions. */
-+void dm_rh_delay(struct dm_rh_client *rh, struct bio *bio);
-+void dm_rh_delay_by_region(struct dm_rh_client *rh,
-+ struct bio *bio, region_t region);
-+
-+/*
-+ * Normally, the region hash will automatically call the dispatch function.
-+ * dm_rh_dispatch_bios() is for intentional dispatching of bios.
-+ */
-+void dm_rh_dispatch_bios(struct dm_rh_client *rh, region_t region, int error);
-+
-+/*
-+ * Region recovery control.
-+ */
-+/* Prepare some regions for recovery by starting to quiesce them. */
-+int dm_rh_recovery_prepare(struct dm_rh_client *rh);
-+/* Try fetching a quiesced region for recovery. */
-+struct dm_region *dm_rh_recovery_start(struct dm_rh_client *rh);
-+/* Report recovery end on a region. */
-+void dm_rh_recovery_end(struct dm_rh_client *rh, struct dm_region *reg,
-+ int error);
-+/* Check for amount of recoveries in flight. */
-+int dm_rh_recovery_in_flight(struct dm_rh_client *rh);
-+/* Start/stop recovery. */
-+void dm_rh_stop_recovery(struct dm_rh_client *rh);
-+void dm_rh_start_recovery(struct dm_rh_client *rh);
-+
-+#endif /* #ifdef DM_REGION_HASH_H */
projects / ipfire-2.x.git / blobdiff