--- /dev/null
+From: "Heinz Mauelshagen <hjm@redhat.de>
+Subject: DMRAID45 module
+X-URL: http://people.redhat.com/~heinzm/sw/dm/dm-raid45/
+
+ DM-RAID 45 module.
+
+ This driver is used for "Fake RAID" devices.
+
+Acked-by: Jeff Mahoney <jeffm@suse.com>
+
+---
+
+ drivers/md/Kconfig | 8
+ drivers/md/Makefile | 4
+ drivers/md/dm-memcache.c | 301 ++
+ drivers/md/dm-memcache.h | 68
+ drivers/md/dm-message.c | 182 +
+ drivers/md/dm-message.h | 91
+ drivers/md/dm-raid45.c | 4516 +++++++++++++++++++++++++++++++++++++++++++++
+ drivers/md/dm-raid45.h | 28
+ drivers/md/dm-regions.c | 723 +++++++
+ drivers/md/dm.c | 1
+ include/linux/dm-regions.h | 115 +
+ 11 files changed, 6036 insertions(+), 1 deletion(-)
+
+--- a/drivers/md/dm.c
++++ b/drivers/md/dm.c
+@@ -1680,6 +1680,7 @@ struct gendisk *dm_disk(struct mapped_de
+ {
+ return md->disk;
+ }
++EXPORT_SYMBOL_GPL(dm_disk);
+
+ int dm_suspended(struct mapped_device *md)
+ {
+--- /dev/null
++++ b/drivers/md/dm-memcache.c
+@@ -0,0 +1,301 @@
++/*
++ * Copyright (C) 2006-2008 Red Hat, Inc. All rights reserved.
++ *
++ * Module Author: Heinz Mauelshagen <heinzm@redhat.com>
++ *
++ * Device-mapper memory object handling:
++ *
++ * o allocate/free total_pages in a per client page pool.
++ *
++ * o allocate/free memory objects with chunks (1..n) of
++ * pages_per_chunk pages hanging off.
++ *
++ * This file is released under the GPL.
++ */
++
++#define DM_MEM_CACHE_VERSION "0.2"
++
++#include "dm.h"
++#include "dm-memcache.h"
++#include <linux/dm-io.h>
++
++struct dm_mem_cache_client {
++ spinlock_t lock;
++ mempool_t *objs_pool;
++ struct page_list *free_list;
++ unsigned objects;
++ unsigned chunks;
++ unsigned pages_per_chunk;
++ unsigned free_pages;
++ unsigned total_pages;
++};
++
++/*
++ * Free pages and page_list elements of client.
++ */
++static void free_cache_pages(struct page_list *list)
++{
++ while (list) {
++ struct page_list *pl = list;
++
++ list = pl->next;
++ BUG_ON(!pl->page);
++ __free_page(pl->page);
++ kfree(pl);
++ }
++}
++
++/*
++ * Alloc number of pages and page_list elements as required by client.
++ */
++static struct page_list *alloc_cache_pages(unsigned pages)
++{
++ struct page_list *pl, *ret = NULL;
++ struct page *page;
++
++ while (pages--) {
++ page = alloc_page(GFP_NOIO);
++ if (!page)
++ goto err;
++
++ pl = kmalloc(sizeof(*pl), GFP_NOIO);
++ if (!pl) {
++ __free_page(page);
++ goto err;
++ }
++
++ pl->page = page;
++ pl->next = ret;
++ ret = pl;
++ }
++
++ return ret;
++
++err:
++ free_cache_pages(ret);
++ return NULL;
++}
++
++/*
++ * Allocate page_list elements from the pool to chunks of the memory object.
++ */
++static void alloc_chunks(struct dm_mem_cache_client *cl,
++ struct dm_mem_cache_object *obj)
++{
++ unsigned chunks = cl->chunks;
++ unsigned long flags;
++
++ local_irq_save(flags);
++ local_irq_disable();
++ while (chunks--) {
++ unsigned p = cl->pages_per_chunk;
++
++ obj[chunks].pl = NULL;
++
++ while (p--) {
++ struct page_list *pl;
++
++ /* Take next element from free list */
++ spin_lock(&cl->lock);
++ pl = cl->free_list;
++ BUG_ON(!pl);
++ cl->free_list = pl->next;
++ spin_unlock(&cl->lock);
++
++ pl->next = obj[chunks].pl;
++ obj[chunks].pl = pl;
++ }
++ }
++
++ local_irq_restore(flags);
++}
++
++/*
++ * Free page_list elements putting them back onto free list
++ */
++static void free_chunks(struct dm_mem_cache_client *cl,
++ struct dm_mem_cache_object *obj)
++{
++ unsigned chunks = cl->chunks;
++ unsigned long flags;
++ struct page_list *next, *pl;
++
++ local_irq_save(flags);
++ local_irq_disable();
++ while (chunks--) {
++ for (pl = obj[chunks].pl; pl; pl = next) {
++ next = pl->next;
++
++ spin_lock(&cl->lock);
++ pl->next = cl->free_list;
++ cl->free_list = pl;
++ cl->free_pages++;
++ spin_unlock(&cl->lock);
++ }
++ }
++
++ local_irq_restore(flags);
++}
++
++/*
++ * Create/destroy dm memory cache client resources.
++ */
++struct dm_mem_cache_client *
++dm_mem_cache_client_create(unsigned objects, unsigned chunks,
++ unsigned pages_per_chunk)
++{
++ unsigned total_pages = objects * chunks * pages_per_chunk;
++ struct dm_mem_cache_client *client;
++
++ BUG_ON(!total_pages);
++ client = kzalloc(sizeof(*client), GFP_KERNEL);
++ if (!client)
++ return ERR_PTR(-ENOMEM);
++
++ client->objs_pool = mempool_create_kmalloc_pool(objects,
++ chunks * sizeof(struct dm_mem_cache_object));
++ if (!client->objs_pool)
++ goto err;
++
++ client->free_list = alloc_cache_pages(total_pages);
++ if (!client->free_list)
++ goto err1;
++
++ spin_lock_init(&client->lock);
++ client->objects = objects;
++ client->chunks = chunks;
++ client->pages_per_chunk = pages_per_chunk;
++ client->free_pages = client->total_pages = total_pages;
++ return client;
++
++err1:
++ mempool_destroy(client->objs_pool);
++err:
++ kfree(client);
++ return ERR_PTR(-ENOMEM);
++}
++EXPORT_SYMBOL(dm_mem_cache_client_create);
++
++void dm_mem_cache_client_destroy(struct dm_mem_cache_client *cl)
++{
++ BUG_ON(cl->free_pages != cl->total_pages);
++ free_cache_pages(cl->free_list);
++ mempool_destroy(cl->objs_pool);
++ kfree(cl);
++}
++EXPORT_SYMBOL(dm_mem_cache_client_destroy);
++
++/*
++ * Grow a clients cache by an amount of pages.
++ *
++ * Don't call from interrupt context!
++ */
++int dm_mem_cache_grow(struct dm_mem_cache_client *cl, unsigned objects)
++{
++ unsigned pages = objects * cl->chunks * cl->pages_per_chunk;
++ struct page_list *pl, *last;
++
++ BUG_ON(!pages);
++ pl = alloc_cache_pages(pages);
++ if (!pl)
++ return -ENOMEM;
++
++ last = pl;
++ while (last->next)
++ last = last->next;
++
++ spin_lock_irq(&cl->lock);
++ last->next = cl->free_list;
++ cl->free_list = pl;
++ cl->free_pages += pages;
++ cl->total_pages += pages;
++ cl->objects++;
++ spin_unlock_irq(&cl->lock);
++
++ mempool_resize(cl->objs_pool, cl->objects, GFP_NOIO);
++ return 0;
++}
++EXPORT_SYMBOL(dm_mem_cache_grow);
++
++/* Shrink a clients cache by an amount of pages */
++int dm_mem_cache_shrink(struct dm_mem_cache_client *cl, unsigned objects)
++{
++ int r;
++ unsigned pages = objects * cl->chunks * cl->pages_per_chunk, p = pages;
++ unsigned long flags;
++ struct page_list *last = NULL, *pl, *pos;
++
++ BUG_ON(!pages);
++
++ spin_lock_irqsave(&cl->lock, flags);
++ pl = pos = cl->free_list;
++ while (p-- && pos->next) {
++ last = pos;
++ pos = pos->next;
++ }
++
++ if (++p)
++ r = -ENOMEM;
++ else {
++ r = 0;
++ cl->free_list = pos;
++ cl->free_pages -= pages;
++ cl->total_pages -= pages;
++ cl->objects--;
++ last->next = NULL;
++ }
++ spin_unlock_irqrestore(&cl->lock, flags);
++
++ if (!r) {
++ free_cache_pages(pl);
++ mempool_resize(cl->objs_pool, cl->objects, GFP_NOIO);
++ }
++
++ return r;
++}
++EXPORT_SYMBOL(dm_mem_cache_shrink);
++
++/*
++ * Allocate/free a memory object
++ *
++ * Can be called from interrupt context
++ */
++struct dm_mem_cache_object *dm_mem_cache_alloc(struct dm_mem_cache_client *cl)
++{
++ int r = 0;
++ unsigned pages = cl->chunks * cl->pages_per_chunk;
++ unsigned long flags;
++ struct dm_mem_cache_object *obj;
++
++ obj = mempool_alloc(cl->objs_pool, GFP_NOIO);
++ if (!obj)
++ return ERR_PTR(-ENOMEM);
++
++ spin_lock_irqsave(&cl->lock, flags);
++ if (pages > cl->free_pages)
++ r = -ENOMEM;
++ else
++ cl->free_pages -= pages;
++ spin_unlock_irqrestore(&cl->lock, flags);
++
++ if (r) {
++ mempool_free(obj, cl->objs_pool);
++ return ERR_PTR(r);
++ }
++
++ alloc_chunks(cl, obj);
++ return obj;
++}
++EXPORT_SYMBOL(dm_mem_cache_alloc);
++
++void dm_mem_cache_free(struct dm_mem_cache_client *cl,
++ struct dm_mem_cache_object *obj)
++{
++ free_chunks(cl, obj);
++ mempool_free(obj, cl->objs_pool);
++}
++EXPORT_SYMBOL(dm_mem_cache_free);
++
++MODULE_DESCRIPTION(DM_NAME " dm memory cache");
++MODULE_AUTHOR("Heinz Mauelshagen <hjm@redhat.com>");
++MODULE_LICENSE("GPL");
+--- /dev/null
++++ b/drivers/md/dm-memcache.h
+@@ -0,0 +1,68 @@
++/*
++ * Copyright (C) 2006-2008 Red Hat, Inc. All rights reserved.
++ *
++ * Module Author: Heinz Mauelshagen <Mauelshagen@RedHat.com>
++ *
++ * Device-mapper memory object handling:
++ *
++ * o allocate/free total_pages in a per client page pool.
++ *
++ * o allocate/free memory objects with chunks (1..n) of
++ * pages_per_chunk pages hanging off.
++ *
++ * This file is released under the GPL.
++ */
++
++#ifndef _DM_MEM_CACHE_H
++#define _DM_MEM_CACHE_H
++
++#define DM_MEM_CACHE_H_VERSION "0.1"
++
++#include "dm.h"
++#include <linux/dm-io.h>
++
++static inline struct page_list *pl_elem(struct page_list *pl, unsigned p)
++{
++ while (pl && p--)
++ pl = pl->next;
++
++ return pl;
++}
++
++struct dm_mem_cache_object {
++ struct page_list *pl; /* Dynamically allocated array */
++ void *private; /* Caller context reference */
++};
++
++struct dm_mem_cache_client;
++
++/*
++ * Create/destroy dm memory cache client resources.
++ *
++ * On creation, a number of @objects with @chunks of
++ * @pages_per_chunk pages will be allocated.
++ */
++struct dm_mem_cache_client *
++dm_mem_cache_client_create(unsigned objects, unsigned chunks,
++ unsigned pages_per_chunk);
++void dm_mem_cache_client_destroy(struct dm_mem_cache_client *client);
++
++/*
++ * Grow/shrink a dm memory cache client resources
++ * by @objetcs amount of objects.
++ */
++int dm_mem_cache_grow(struct dm_mem_cache_client *client, unsigned objects);
++int dm_mem_cache_shrink(struct dm_mem_cache_client *client, unsigned objects);
++
++/*
++ * Allocate/free a memory object
++ *
++ * On allocation one object with an amount of chunks and
++ * an amount of pages per chunk will be returned on success.
++ */
++struct dm_mem_cache_object *
++dm_mem_cache_alloc(struct dm_mem_cache_client *client);
++void dm_mem_cache_free(struct dm_mem_cache_client *client,
++ struct dm_mem_cache_object *object);
++
++#endif
+--- /dev/null
++++ b/drivers/md/dm-message.c
+@@ -0,0 +1,182 @@
++/*
++ * Copyright (C) 2007,2008 Red Hat Inc. All rights reserved.
++ *
++ * Module Author: Heinz Mauelshagen <heinzm@redhat.com>
++ *
++ * General device-mapper message interface argument parser.
++ *
++ * This file is released under the GPL.
++ *
++ * device-mapper message parser.
++ *
++ */
++
++#include "dm.h"
++#include "dm-message.h"
++#include <linux/kernel.h>
++
++#define DM_MSG_PREFIX "dm_message"
++
++/* Basename of a path. */
++static inline char *
++basename(char *s)
++{
++ char *p = strrchr(s, '/');
++
++ return p ? p + 1 : s;
++}
++
++/* Get an argument depending on type. */
++static void
++message_arguments(struct dm_msg *msg, int argc, char **argv)
++{
++
++ if (argc) {
++ int i;
++ struct dm_message_argument *args = msg->spec->args;
++
++ for (i = 0; i < args->num_args; i++) {
++ int r;
++ unsigned long **ptr = args->ptr;
++ enum dm_message_argument_type type = args->types[i];
++
++ switch (type) {
++ case dm_msg_base_t:
++ ((char **) ptr)[i] = basename(argv[i]);
++ break;
++
++ case dm_msg_str_t:
++ ((char **) ptr)[i] = argv[i];
++ break;
++
++ case dm_msg_int_t:
++ r = sscanf(argv[i], "%d", ((int **) ptr)[i]);
++ goto check;
++
++ case dm_msg_uint_t:
++ r = sscanf(argv[i], "%u",
++ ((unsigned **) ptr)[i]);
++ goto check;
++
++ case dm_msg_uint64_t:
++ r = sscanf(argv[i], "%llu",
++ ((unsigned long long **) ptr)[i]);
++
++check:
++ if (r != 1) {
++ set_bit(dm_msg_ret_undef, &msg->ret);
++ set_bit(dm_msg_ret_arg, &msg->ret);
++ }
++ }
++ }
++ }
++}
++
++/* Parse message options. */
++static void
++message_options_parse(struct dm_msg *msg, int argc, char **argv)
++{
++ int hit = 0;
++ unsigned long *action;
++ size_t l1 = strlen(*argv), l_hit = 0;
++ struct dm_message_option *o = msg->spec->options;
++ char **option, **option_end = o->options + o->num_options;
++
++ for (option = o->options, action = o->actions;
++ option < option_end; option++, action++) {
++ size_t l2 = strlen(*option);
++
++ if (!strnicmp(*argv, *option, min(l1, l2))) {
++ hit++;
++ l_hit = l2;
++ set_bit(*action, &msg->action);
++ }
++ }
++
++ /* Assume error. */
++ msg->ret = 0;
++ set_bit(dm_msg_ret_option, &msg->ret);
++ if (!hit || l1 > l_hit)
++ set_bit(dm_msg_ret_undef, &msg->ret); /* Undefined option. */
++ else if (hit > 1)
++ set_bit(dm_msg_ret_ambiguous, &msg->ret); /* Ambiguous option.*/
++ else {
++ clear_bit(dm_msg_ret_option, &msg->ret); /* Option OK. */
++ message_arguments(msg, --argc, ++argv);
++ }
++}
++
++static inline void
++print_ret(const char *caller, unsigned long ret)
++{
++ struct {
++ unsigned long err;
++ const char *err_str;
++ } static err_msg[] = {
++ { dm_msg_ret_ambiguous, "message ambiguous" },
++ { dm_msg_ret_inval, "message invalid" },
++ { dm_msg_ret_undef, "message undefined" },
++ { dm_msg_ret_arg, "message argument" },
++ { dm_msg_ret_argcount, "message argument count" },
++ { dm_msg_ret_option, "option" },
++ }, *e = ARRAY_END(err_msg);
++
++ while (e-- > err_msg) {
++ if (test_bit(e->err, &ret))
++ DMERR("%s %s", caller, e->err_str);
++ }
++}
++
++/* Parse a message action. */
++int
++dm_message_parse(const char *caller, struct dm_msg *msg, void *context,
++ int argc, char **argv)
++{
++ int hit = 0;
++ size_t l1 = strlen(*argv), l_hit = 0;
++ struct dm_msg_spec *s, *s_hit = NULL,
++ *s_end = msg->specs + msg->num_specs;
++
++ if (argc < 2)
++ return -EINVAL;
++
++ for (s = msg->specs; s < s_end; s++) {
++ size_t l2 = strlen(s->cmd);
++
++ if (!strnicmp(*argv, s->cmd, min(l1, l2))) {
++ hit++;
++ l_hit = l2;
++ s_hit = s;
++ }
++ }
++
++ msg->ret = 0;
++ if (!hit || l1 > l_hit) /* No hit or message string too long. */
++ set_bit(dm_msg_ret_undef, &msg->ret);
++ else if (hit > 1) /* Ambiguous message. */
++ set_bit(dm_msg_ret_ambiguous, &msg->ret);
++ else if (argc - 2 != s_hit->args->num_args) {
++ set_bit(dm_msg_ret_undef, &msg->ret);
++ set_bit(dm_msg_ret_argcount, &msg->ret);
++ }
++
++ if (msg->ret)
++ goto bad;
++
++ msg->action = 0;
++ msg->spec = s_hit;
++ set_bit(s_hit->action, &msg->action);
++ message_options_parse(msg, --argc, ++argv);
++
++ if (!msg->ret)
++ return msg->spec->f(msg, context);
++
++bad:
++ print_ret(caller, msg->ret);
++ return -EINVAL;
++}
++EXPORT_SYMBOL(dm_message_parse);
++
++MODULE_DESCRIPTION(DM_NAME " device-mapper target message parser");
++MODULE_AUTHOR("Heinz Mauelshagen <hjm@redhat.com>");
++MODULE_LICENSE("GPL");
+--- /dev/null
++++ b/drivers/md/dm-message.h
+@@ -0,0 +1,91 @@
++/*
++ * Copyright (C) 2007,2008 Red Hat, Inc. All rights reserved.
++ *
++ * Module Author: Heinz Mauelshagen <Mauelshagen@RedHat.de>
++ *
++ * General device-mapper message interface argument parser.
++ *
++ * This file is released under the GPL.
++ *
++ */
++
++#ifndef DM_MESSAGE_H
++#define DM_MESSAGE_H
++
++/* Factor out to dm.h. */
++/* Reference to array end. */
++#define ARRAY_END(a) ((a) + ARRAY_SIZE(a))
++
++/* Message return bits. */
++enum dm_message_return {
++ dm_msg_ret_ambiguous, /* Action ambiguous. */
++ dm_msg_ret_inval, /* Action invalid. */
++ dm_msg_ret_undef, /* Action undefined. */
++
++ dm_msg_ret_option, /* Option error. */
++ dm_msg_ret_arg, /* Argument error. */
++ dm_msg_ret_argcount, /* Argument count error. */
++};
++
++/* Message argument type conversions. */
++enum dm_message_argument_type {
++ dm_msg_base_t, /* Basename string. */
++ dm_msg_str_t, /* String. */
++ dm_msg_int_t, /* Signed int. */
++ dm_msg_uint_t, /* Unsigned int. */
++ dm_msg_uint64_t, /* Unsigned int 64. */
++};
++
++/* A message option. */
++struct dm_message_option {
++ unsigned num_options;
++ char **options;
++ unsigned long *actions;
++};
++
++/* Message arguments and types. */
++struct dm_message_argument {
++ unsigned num_args;
++ unsigned long **ptr;
++ enum dm_message_argument_type types[];
++};
++
++/* Client message. */
++struct dm_msg {
++ unsigned long action; /* Identified action. */
++ unsigned long ret; /* Return bits. */
++ unsigned num_specs; /* # of sepcifications listed. */
++ struct dm_msg_spec *specs; /* Specification list. */
++ struct dm_msg_spec *spec; /* Specification selected. */
++};
++
++/* Secification of the message. */
++struct dm_msg_spec {
++ const char *cmd; /* Name of the command (i.e. 'bandwidth'). */
++ unsigned long action;
++ struct dm_message_option *options;
++ struct dm_message_argument *args;
++ unsigned long parm; /* Parameter to pass through to callback. */
++ /* Function to process for action. */
++ int (*f) (struct dm_msg *msg, void *context);
++};
++
++/* Parameter access macros. */
++#define DM_MSG_PARM(msg) ((msg)->spec->parm)
++
++#define DM_MSG_STR_ARGS(msg, idx) ((char *) *(msg)->spec->args->ptr[idx])
++#define DM_MSG_INT_ARGS(msg, idx) ((int) *(msg)->spec->args->ptr[idx])
++#define DM_MSG_UINT_ARGS(msg, idx) ((unsigned) DM_MSG_INT_ARG(msg, idx))
++#define DM_MSG_UINT64_ARGS(msg, idx) ((uint64_t) *(msg)->spec->args->ptr[idx])
++
++#define DM_MSG_STR_ARG(msg) DM_MSG_STR_ARGS(msg, 0)
++#define DM_MSG_INT_ARG(msg) DM_MSG_INT_ARGS(msg, 0)
++#define DM_MSG_UINT_ARG(msg) DM_MSG_UINT_ARGS(msg, 0)
++#define DM_MSG_UINT64_ARG(msg) DM_MSG_UINT64_ARGS(msg, 0)
++
++
++/* Parse a message and its options and optionally call a function back. */
++int dm_message_parse(const char *caller, struct dm_msg *msg, void *context,
++ int argc, char **argv);
++
++#endif
+--- /dev/null
++++ b/drivers/md/dm-raid45.c
+@@ -0,0 +1,4516 @@
++/*
++ * Copyright (C) 2005-2008 Red Hat, Inc. All rights reserved.
++ *
++ * Module Author: Heinz Mauelshagen <Mauelshagen@RedHat.com>
++ *
++ * This file is released under the GPL.
++ *
++ *
++ * Linux 2.6 Device Mapper RAID4 and RAID5 target.
++ *
++ * Supports:
++ * o RAID4 with dedicated and selectable parity device
++ * o RAID5 with rotating parity (left+right, symmetric+asymmetric)
++ * o run time optimization of xor algorithm used to calculate parity
++ *
++ *
++ * Thanks to MD for:
++ * o the raid address calculation algorithm
++ * o the base of the biovec <-> page list copier.
++ *
++ *
++ * Uses region hash to keep track of how many writes are in flight to
++ * regions in order to use dirty log to keep state of regions to recover:
++ *
++ * o clean regions (those which are synchronized
++ * and don't have write io in flight)
++ * o dirty regions (those with write io in flight)
++ *
++ *
++ * On startup, any dirty regions are migrated to the 'nosync' state
++ * and are subject to recovery by the daemon.
++ *
++ * See raid_ctr() for table definition.
++ *
++ *
++ * FIXME:
++ * o add virtual interface for locking
++ * o remove instrumentation (REMOVEME:)
++ *
++ */
++
++static const char *version = "v0.2431";
++
++#include "dm.h"
++#include "dm-bio-list.h"
++#include "dm-memcache.h"
++#include "dm-message.h"
++#include "dm-raid45.h"
++
++#include <linux/kernel.h>
++#include <linux/vmalloc.h>
++
++#include <linux/dm-io.h>
++#include <linux/dm-dirty-log.h>
++#include <linux/dm-regions.h>
++
++/* # of parallel recovered regions */
++/* FIXME: cope with multiple recovery stripes in raid_set struct. */
++#define MAX_RECOVER 1 /* needs to be 1! */
++
++/*
++ * Configurable parameters
++ */
++#define INLINE
++
++/* Default # of stripes if not set in constructor. */
++#define STRIPES 64
++
++/* Minimum/maximum # of selectable stripes. */
++#define STRIPES_MIN 8
++#define STRIPES_MAX 16384
++
++/* Default chunk size in sectors if not set in constructor. */
++#define CHUNK_SIZE 64
++
++/* Default io size in sectors if not set in constructor. */
++#define IO_SIZE_MIN SECTORS_PER_PAGE
++#define IO_SIZE IO_SIZE_MIN
++
++/* Maximum setable chunk size in sectors. */
++#define CHUNK_SIZE_MAX 16384
++
++/* Recover io size default in sectors. */
++#define RECOVER_IO_SIZE_MIN 64
++#define RECOVER_IO_SIZE 256
++
++/* Default percentage recover io bandwidth. */
++#define BANDWIDTH 10
++#define BANDWIDTH_MIN 1
++#define BANDWIDTH_MAX 100
++/*
++ * END Configurable parameters
++ */
++
++#define TARGET "dm-raid45"
++#define DAEMON "kraid45d"
++#define DM_MSG_PREFIX TARGET
++
++#define SECTORS_PER_PAGE (PAGE_SIZE >> SECTOR_SHIFT)
++
++/* Amount/size for __xor(). */
++#define SECTORS_PER_XOR SECTORS_PER_PAGE
++#define XOR_SIZE PAGE_SIZE
++
++/* Derive raid_set from stripe_cache pointer. */
++#define RS(x) container_of(x, struct raid_set, sc)
++
++/* Check value in range. */
++#define range_ok(i, min, max) (i >= min && i <= max)
++
++/* Page reference. */
++#define PAGE(stripe, p) ((stripe)->obj[p].pl->page)
++
++/* Bio list reference. */
++#define BL(stripe, p, rw) (stripe->ss[p].bl + rw)
++
++/* Page list reference. */
++#define PL(stripe, p) (stripe->obj[p].pl)
++
++/* Check argument is power of 2. */
++#define POWER_OF_2(a) (!(a & (a - 1)))
++
++/* Factor out to dm-bio-list.h */
++static inline void bio_list_push(struct bio_list *bl, struct bio *bio)
++{
++ bio->bi_next = bl->head;
++ bl->head = bio;
++
++ if (!bl->tail)
++ bl->tail = bio;
++}
++
++/* Factor out to dm.h */
++#define TI_ERR_RET(str, ret) \
++ do { ti->error = DM_MSG_PREFIX ": " str; return ret; } while (0);
++#define TI_ERR(str) TI_ERR_RET(str, -EINVAL)
++
++/*-----------------------------------------------------------------
++ * Stripe cache
++ *
++ * Cache for all reads and writes to raid sets (operational or degraded)
++ *
++ * We need to run all data to and from a RAID set through this cache,
++ * because parity chunks need to get calculated from data chunks
++ * or, in the degraded/resynchronization case, missing chunks need
++ * to be reconstructed using the other chunks of the stripe.
++ *---------------------------------------------------------------*/
++/* Protect kmem cache # counter. */
++static atomic_t _stripe_sc_nr = ATOMIC_INIT(-1); /* kmem cache # counter. */
++
++/* A stripe set (holds bios hanging off). */
++struct stripe_set {
++ struct stripe *stripe; /* Backpointer to stripe for endio(). */
++ struct bio_list bl[3]; /* Reads, writes, and writes merged. */
++#define WRITE_MERGED 2
++};
++
++#if READ != 0 || WRITE != 1
++#error dm-raid45: READ/WRITE != 0/1 used as index!!!
++#endif
++
++/*
++ * Stripe linked list indexes. Keep order, because the stripe
++ * and the stripe cache rely on the first 3!
++ */
++enum list_types {
++ LIST_IO = 0, /* Stripes with io pending. */
++ LIST_ENDIO, /* Stripes to endio. */
++ LIST_LRU, /* Least recently used stripes. */
++ LIST_HASH, /* Hashed stripes. */
++ LIST_RECOVER = LIST_HASH, /* For recovery type stripes only. */
++ NR_LISTS, /* To size array in struct stripe. */
++};
++
++enum lock_types {
++ LOCK_ENDIO = 0, /* Protect endio list. */
++ LOCK_LRU, /* Protect lru list. */
++ NR_LOCKS, /* To size array in struct stripe_cache. */
++};
++
++/* A stripe: the io object to handle all reads and writes to a RAID set. */
++struct stripe {
++ struct stripe_cache *sc; /* Backpointer to stripe cache. */
++
++ sector_t key; /* Hash key. */
++ sector_t region; /* Region stripe is mapped to. */
++
++ /* Reference count. */
++ atomic_t cnt;
++
++ struct {
++ unsigned long flags; /* flags (see below). */
++
++ /*
++ * Pending ios in flight:
++ *
++ * used as a 'lock' to control move of stripe to endio list
++ */
++ atomic_t pending; /* Pending ios in flight. */
++
++ /* Sectors to read and write for multi page stripe sets. */
++ unsigned size;
++ } io;
++
++ /* Lock on stripe (for clustering). */
++ void *lock;
++
++ /*
++ * 4 linked lists:
++ * o io list to flush io
++ * o endio list
++ * o LRU list to put stripes w/o reference count on
++ * o stripe cache hash
++ */
++ struct list_head lists[NR_LISTS];
++
++ struct {
++ unsigned short parity; /* Parity chunk index. */
++ short recover; /* Recovery chunk index. */
++ } idx;
++
++ /* This sets memory cache object (dm-mem-cache). */
++ struct dm_mem_cache_object *obj;
++
++ /* Array of stripe sets (dynamically allocated). */
++ struct stripe_set ss[0];
++};
++
++/* States stripes can be in (flags field). */
++enum stripe_states {
++ STRIPE_ACTIVE, /* Active io on stripe. */
++ STRIPE_ERROR, /* io error on stripe. */
++ STRIPE_MERGED, /* Writes got merged. */
++ STRIPE_READ, /* Read. */
++ STRIPE_RBW, /* Read-before-write. */
++ STRIPE_RECONSTRUCT, /* reconstruct of a missing chunk required. */
++ STRIPE_RECOVER, /* Stripe used for RAID set recovery. */
++};
++
++/* ... and macros to access them. */
++#define BITOPS(name, what, var, flag) \
++static inline int TestClear ## name ## what(struct var *v) \
++{ return test_and_clear_bit(flag, &v->io.flags); } \
++static inline int TestSet ## name ## what(struct var *v) \
++{ return test_and_set_bit(flag, &v->io.flags); } \
++static inline void Clear ## name ## what(struct var *v) \
++{ clear_bit(flag, &v->io.flags); } \
++static inline void Set ## name ## what(struct var *v) \
++{ set_bit(flag, &v->io.flags); } \
++static inline int name ## what(struct var *v) \
++{ return test_bit(flag, &v->io.flags); }
++
++
++BITOPS(Stripe, Active, stripe, STRIPE_ACTIVE)
++BITOPS(Stripe, Merged, stripe, STRIPE_MERGED)
++BITOPS(Stripe, Error, stripe, STRIPE_ERROR)
++BITOPS(Stripe, Read, stripe, STRIPE_READ)
++BITOPS(Stripe, RBW, stripe, STRIPE_RBW)
++BITOPS(Stripe, Reconstruct, stripe, STRIPE_RECONSTRUCT)
++BITOPS(Stripe, Recover, stripe, STRIPE_RECOVER)
++
++/* A stripe hash. */
++struct stripe_hash {
++ struct list_head *hash;
++ unsigned buckets;
++ unsigned mask;
++ unsigned prime;
++ unsigned shift;
++};
++
++/* A stripe cache. */
++struct stripe_cache {
++ /* Stripe hash. */
++ struct stripe_hash hash;
++
++ /* Stripes with io to flush, stripes to endio and LRU lists. */
++ struct list_head lists[3];
++
++ /* Locks to protect endio and lru lists. */
++ spinlock_t locks[NR_LOCKS];
++
++ /* Slab cache to allocate stripes from. */
++ struct {
++ struct kmem_cache *cache; /* Cache itself. */
++ char name[32]; /* Unique name. */
++ } kc;
++
++ struct dm_io_client *dm_io_client; /* dm-io client resource context. */
++
++ /* dm-mem-cache client resource context. */
++ struct dm_mem_cache_client *mem_cache_client;
++
++ int stripes_parm; /* # stripes parameter from constructor. */
++ atomic_t stripes; /* actual # of stripes in cache. */
++ atomic_t stripes_to_shrink; /* # of stripes to shrink cache by. */
++ atomic_t stripes_last; /* last # of stripes in cache. */
++ atomic_t active_stripes; /* actual # of active stripes in cache. */
++
++ /* REMOVEME: */
++ atomic_t max_active_stripes; /* actual # of active stripes in cache. */
++};
++
++/* Flag specs for raid_dev */ ;
++enum raid_dev_flags { DEVICE_FAILED, IO_QUEUED };
++
++/* The raid device in a set. */
++struct raid_dev {
++ struct dm_dev *dev;
++ unsigned long flags; /* raid_dev_flags. */
++ sector_t start; /* offset to map to. */
++};
++
++/* Flags spec for raid_set. */
++enum raid_set_flags {
++ RS_CHECK_OVERWRITE, /* Check for chunk overwrites. */
++ RS_DEAD, /* RAID set inoperational. */
++ RS_DEVEL_STATS, /* REMOVEME: display status information. */
++ RS_IO_ERROR, /* io error on set. */
++ RS_RECOVER, /* Do recovery. */
++ RS_RECOVERY_BANDWIDTH, /* Allow recovery bandwidth (delayed bios). */
++ RS_REGION_GET, /* get a region to recover. */
++ RS_SC_BUSY, /* stripe cache busy -> send an event. */
++ RS_SUSPENDED, /* RAID set suspendedn. */
++};
++
++/* REMOVEME: devel stats counters. */
++enum stats_types {
++ S_BIOS_READ,
++ S_BIOS_ADDED_READ,
++ S_BIOS_ENDIO_READ,
++ S_BIOS_WRITE,
++ S_BIOS_ADDED_WRITE,
++ S_BIOS_ENDIO_WRITE,
++ S_CAN_MERGE,
++ S_CANT_MERGE,
++ S_CONGESTED,
++ S_DM_IO_READ,
++ S_DM_IO_WRITE,
++ S_ACTIVE_READS,
++ S_BANDWIDTH,
++ S_BARRIER,
++ S_BIO_COPY_PL_NEXT,
++ S_DEGRADED,
++ S_DELAYED_BIOS,
++ S_EVICT,
++ S_FLUSHS,
++ S_HITS_1ST,
++ S_IOS_POST,
++ S_INSCACHE,
++ S_MAX_LOOKUP,
++ S_MERGE_PAGE_LOCKED,
++ S_NO_BANDWIDTH,
++ S_NOT_CONGESTED,
++ S_NO_RW,
++ S_NOSYNC,
++ S_PROHIBITPAGEIO,
++ S_RECONSTRUCT_EI,
++ S_RECONSTRUCT_DEV,
++ S_REDO,
++ S_REQUEUE,
++ S_STRIPE_ERROR,
++ S_SUM_DELAYED_BIOS,
++ S_XORS,
++ S_NR_STATS, /* # of stats counters. */
++};
++
++/* Status type -> string mappings. */
++struct stats_map {
++ const enum stats_types type;
++ const char *str;
++};
++
++static struct stats_map stats_map[] = {
++ { S_BIOS_READ, "r=" },
++ { S_BIOS_ADDED_READ, "/" },
++ { S_BIOS_ENDIO_READ, "/" },
++ { S_BIOS_WRITE, " w=" },
++ { S_BIOS_ADDED_WRITE, "/" },
++ { S_BIOS_ENDIO_WRITE, "/" },
++ { S_DM_IO_READ, " rc=" },
++ { S_DM_IO_WRITE, " wc=" },
++ { S_ACTIVE_READS, " active_reads=" },
++ { S_BANDWIDTH, " bandwidth=" },
++ { S_NO_BANDWIDTH, " no_bandwidth=" },
++ { S_BARRIER, " barrier=" },
++ { S_BIO_COPY_PL_NEXT, " bio_copy_pl_next=" },
++ { S_CAN_MERGE, " can_merge=" },
++ { S_MERGE_PAGE_LOCKED, "/page_locked=" },
++ { S_CANT_MERGE, "/cant_merge=" },
++ { S_CONGESTED, " congested=" },
++ { S_NOT_CONGESTED, "/not_congested=" },
++ { S_DEGRADED, " degraded=" },
++ { S_DELAYED_BIOS, " delayed_bios=" },
++ { S_SUM_DELAYED_BIOS, "/sum_delayed_bios=" },
++ { S_EVICT, " evict=" },
++ { S_FLUSHS, " flushs=" },
++ { S_HITS_1ST, " hits_1st=" },
++ { S_IOS_POST, " ios_post=" },
++ { S_INSCACHE, " inscache=" },
++ { S_MAX_LOOKUP, " max_lookup=" },
++ { S_NO_RW, " no_rw=" },
++ { S_NOSYNC, " nosync=" },
++ { S_PROHIBITPAGEIO, " ProhibitPageIO=" },
++ { S_RECONSTRUCT_EI, " reconstruct_ei=" },
++ { S_RECONSTRUCT_DEV, " reconstruct_dev=" },
++ { S_REDO, " redo=" },
++ { S_REQUEUE, " requeue=" },
++ { S_STRIPE_ERROR, " stripe_error=" },
++ { S_XORS, " xors=" },
++};
++
++/*
++ * A RAID set.
++ */
++typedef void (*xor_function_t)(unsigned count, unsigned long **data);
++struct raid_set {
++ struct dm_target *ti; /* Target pointer. */
++
++ struct {
++ unsigned long flags; /* State flags. */
++ spinlock_t in_lock; /* Protects central input list below. */
++ struct bio_list in; /* Pending ios (central input list). */
++ struct bio_list work; /* ios work set. */
++ wait_queue_head_t suspendq; /* suspend synchronization. */
++ atomic_t in_process; /* counter of queued bios (suspendq). */
++ atomic_t in_process_max;/* counter of queued bios max. */
++
++ /* io work. */
++ struct workqueue_struct *wq;
++ struct delayed_work dws;
++ } io;
++
++ /* External locking. */
++ struct dm_raid45_locking_type *locking;
++
++ struct stripe_cache sc; /* Stripe cache for this set. */
++
++ /* Xor optimization. */
++ struct {
++ struct xor_func *f;
++ unsigned chunks;
++ unsigned speed;
++ } xor;
++
++ /* Recovery parameters. */
++ struct recover {
++ struct dm_dirty_log *dl; /* Dirty log. */
++ struct dm_rh_client *rh; /* Region hash. */
++
++ /* dm-mem-cache client resource context for recovery stripes. */
++ struct dm_mem_cache_client *mem_cache_client;
++
++ struct list_head stripes; /* List of recovery stripes. */
++
++ region_t nr_regions;
++ region_t nr_regions_to_recover;
++ region_t nr_regions_recovered;
++ unsigned long start_jiffies;
++ unsigned long end_jiffies;
++
++ unsigned bandwidth; /* Recovery bandwidth [%]. */
++ unsigned bandwidth_work; /* Recovery bandwidth [factor]. */
++ unsigned bandwidth_parm; /* " constructor parm. */
++ unsigned io_size; /* io size <= chunk size. */
++ unsigned io_size_parm; /* io size ctr parameter. */
++
++ /* recovery io throttling. */
++ atomic_t io_count[2]; /* counter recover/regular io. */
++ unsigned long last_jiffies;
++
++ struct dm_region *reg; /* Actual region to recover. */
++ sector_t pos; /* Position within region to recover. */
++ sector_t end; /* End of region to recover. */
++ } recover;
++
++ /* RAID set parameters. */
++ struct {
++ struct raid_type *raid_type; /* RAID type (eg, RAID4). */
++ unsigned raid_parms; /* # variable raid parameters. */
++
++ unsigned chunk_size; /* Sectors per chunk. */
++ unsigned chunk_size_parm;
++ unsigned chunk_mask; /* Mask for amount. */
++ unsigned chunk_shift; /* rsector chunk size shift. */
++
++ unsigned io_size; /* Sectors per io. */
++ unsigned io_size_parm;
++ unsigned io_mask; /* Mask for amount. */
++ unsigned io_shift_mask; /* Mask for raid_address(). */
++ unsigned io_shift; /* rsector io size shift. */
++ unsigned pages_per_io; /* Pages per io. */
++
++ sector_t sectors_per_dev; /* Sectors per device. */
++
++ atomic_t failed_devs; /* Amount of devices failed. */
++
++ /* Index of device to initialize. */
++ int dev_to_init;
++ int dev_to_init_parm;
++
++ /* Raid devices dynamically allocated. */
++ unsigned raid_devs; /* # of RAID devices below. */
++ unsigned data_devs; /* # of RAID data devices. */
++
++ int ei; /* index of failed RAID device. */
++
++ /* index of dedicated parity device (i.e. RAID4). */
++ int pi;
++ int pi_parm; /* constructor parm for status output. */
++ } set;
++
++ /* REMOVEME: devel stats counters. */
++ atomic_t stats[S_NR_STATS];
++
++ /* Dynamically allocated temporary pointers for xor(). */
++ unsigned long **data;
++
++ /* Dynamically allocated RAID devices. Alignment? */
++ struct raid_dev dev[0];
++};
++
++
++BITOPS(RS, Bandwidth, raid_set, RS_RECOVERY_BANDWIDTH)
++BITOPS(RS, CheckOverwrite, raid_set, RS_CHECK_OVERWRITE)
++BITOPS(RS, Dead, raid_set, RS_DEAD)
++BITOPS(RS, DevelStats, raid_set, RS_DEVEL_STATS)
++BITOPS(RS, IoError, raid_set, RS_IO_ERROR)
++BITOPS(RS, Recover, raid_set, RS_RECOVER)
++BITOPS(RS, RegionGet, raid_set, RS_REGION_GET)
++BITOPS(RS, ScBusy, raid_set, RS_SC_BUSY)
++BITOPS(RS, Suspended, raid_set, RS_SUSPENDED)
++#undef BITOPS
++
++#define PageIO(page) PageChecked(page)
++#define AllowPageIO(page) SetPageChecked(page)
++#define ProhibitPageIO(page) ClearPageChecked(page)
++
++/*-----------------------------------------------------------------
++ * Raid-4/5 set structures.
++ *---------------------------------------------------------------*/
++/* RAID level definitions. */
++enum raid_level {
++ raid4,
++ raid5,
++};
++
++/* Symmetric/Asymmetric, Left/Right parity rotating algorithms. */
++enum raid_algorithm {
++ none,
++ left_asym,
++ right_asym,
++ left_sym,
++ right_sym,
++};
++
++struct raid_type {
++ const char *name; /* RAID algorithm. */
++ const char *descr; /* Descriptor text for logging. */
++ const unsigned parity_devs; /* # of parity devices. */
++ const unsigned minimal_devs; /* minimal # of devices in set. */
++ const enum raid_level level; /* RAID level. */
++ const enum raid_algorithm algorithm; /* RAID algorithm. */
++};
++
++/* Supported raid types and properties. */
++static struct raid_type raid_types[] = {
++ {"raid4", "RAID4 (dedicated parity disk)", 1, 3, raid4, none},
++ {"raid5_la", "RAID5 (left asymmetric)", 1, 3, raid5, left_asym},
++ {"raid5_ra", "RAID5 (right asymmetric)", 1, 3, raid5, right_asym},
++ {"raid5_ls", "RAID5 (left symmetric)", 1, 3, raid5, left_sym},
++ {"raid5_rs", "RAID5 (right symmetric)", 1, 3, raid5, right_sym},
++};
++
++/* Address as calculated by raid_address(). */
++struct address {
++ sector_t key; /* Hash key (start address of stripe). */
++ unsigned di, pi; /* Data and parity disks index. */
++};
++
++/* REMOVEME: reset statistics counters. */
++static void stats_reset(struct raid_set *rs)
++{
++ unsigned s = S_NR_STATS;
++
++ while (s--)
++ atomic_set(rs->stats + s, 0);
++}
++
++/*----------------------------------------------------------------
++ * RAID set management routines.
++ *--------------------------------------------------------------*/
++/*
++ * Begin small helper functions.
++ */
++/* Queue (optionally delayed) io work. */
++static void wake_do_raid_delayed(struct raid_set *rs, unsigned long delay)
++{
++ struct delayed_work *dws = &rs->io.dws;
++
++ cancel_delayed_work(dws);
++ queue_delayed_work(rs->io.wq, dws, delay);
++}
++
++/* Queue io work immediately (called from region hash too). */
++static INLINE void wake_do_raid(void *context)
++{
++ wake_do_raid_delayed(context, 0);
++}
++
++/* Wait until all io has been processed. */
++static INLINE void wait_ios(struct raid_set *rs)
++{
++ wait_event(rs->io.suspendq, !atomic_read(&rs->io.in_process));
++}
++
++/* Declare io queued to device. */
++static INLINE void io_dev_queued(struct raid_dev *dev)
++{
++ set_bit(IO_QUEUED, &dev->flags);
++}
++
++/* Io on device and reset ? */
++static inline int io_dev_clear(struct raid_dev *dev)
++{
++ return test_and_clear_bit(IO_QUEUED, &dev->flags);
++}
++
++/* Get an io reference. */
++static INLINE void io_get(struct raid_set *rs)
++{
++ int p = atomic_inc_return(&rs->io.in_process);
++
++ if (p > atomic_read(&rs->io.in_process_max))
++ atomic_set(&rs->io.in_process_max, p); /* REMOVEME: max. */
++}
++
++/* Put the io reference and conditionally wake io waiters. */
++static INLINE void io_put(struct raid_set *rs)
++{
++ /* Intel: rebuild data corrupter? */
++ if (!atomic_read(&rs->io.in_process)) {
++ DMERR("%s would go negative!!!", __func__);
++ return;
++ }
++
++ if (atomic_dec_and_test(&rs->io.in_process))
++ wake_up(&rs->io.suspendq);
++}
++
++/* Calculate device sector offset. */
++static INLINE sector_t _sector(struct raid_set *rs, struct bio *bio)
++{
++ sector_t sector = bio->bi_sector;
++
++ sector_div(sector, rs->set.data_devs);
++ return sector;
++}
++
++/* Test device operational. */
++static INLINE int dev_operational(struct raid_set *rs, unsigned p)
++{
++ return !test_bit(DEVICE_FAILED, &rs->dev[p].flags);
++}
++
++/* Return # of active stripes in stripe cache. */
++static INLINE int sc_active(struct stripe_cache *sc)
++{
++ return atomic_read(&sc->active_stripes);
++}
++
++/* Test io pending on stripe. */
++static INLINE int stripe_io(struct stripe *stripe)
++{
++ return atomic_read(&stripe->io.pending);
++}
++
++static INLINE void stripe_io_inc(struct stripe *stripe)
++{
++ atomic_inc(&stripe->io.pending);
++}
++
++static INLINE void stripe_io_dec(struct stripe *stripe)
++{
++ atomic_dec(&stripe->io.pending);
++}
++
++/* Wrapper needed by for_each_io_dev(). */
++static void _stripe_io_inc(struct stripe *stripe, unsigned p)
++{
++ stripe_io_inc(stripe);
++}
++
++/* Error a stripe. */
++static INLINE void stripe_error(struct stripe *stripe, struct page *page)
++{
++ SetStripeError(stripe);
++ SetPageError(page);
++ atomic_inc(RS(stripe->sc)->stats + S_STRIPE_ERROR);
++}
++
++/* Page IOed ok. */
++enum dirty_type { CLEAN, DIRTY };
++static INLINE void page_set(struct page *page, enum dirty_type type)
++{
++ switch (type) {
++ case DIRTY:
++ SetPageDirty(page);
++ AllowPageIO(page);
++ break;
++
++ case CLEAN:
++ ClearPageDirty(page);
++ break;
++
++ default:
++ BUG();
++ }
++
++ SetPageUptodate(page);
++ ClearPageError(page);
++}
++
++/* Return region state for a sector. */
++static INLINE int
++region_state(struct raid_set *rs, sector_t sector, unsigned long state)
++{
++ struct dm_rh_client *rh = rs->recover.rh;
++
++ return RSRecover(rs) ?
++ (dm_rh_get_state(rh, dm_rh_sector_to_region(rh, sector), 1) &
++ state) : 0;
++}
++
++/* Check maximum devices which may fail in a raid set. */
++static inline int raid_set_degraded(struct raid_set *rs)
++{
++ return RSIoError(rs);
++}
++
++/* Check # of devices which may fail in a raid set. */
++static INLINE int raid_set_operational(struct raid_set *rs)
++{
++ /* Too many failed devices -> BAD. */
++ return atomic_read(&rs->set.failed_devs) <=
++ rs->set.raid_type->parity_devs;
++}
++
++/*
++ * Return true in case a page_list should be read/written
++ *
++ * Conditions to read/write:
++ * o 1st page in list not uptodate
++ * o 1st page in list dirty
++ * o if we optimized io away, we flag it using the pages checked bit.
++ */
++static INLINE unsigned page_io(struct page *page)
++{
++ /* Optimization: page was flagged to need io during first run. */
++ if (PagePrivate(page)) {
++ ClearPagePrivate(page);
++ return 1;
++ }
++
++ /* Avoid io if prohibited or a locked page. */
++ if (!PageIO(page) || PageLocked(page))
++ return 0;
++
++ if (!PageUptodate(page) || PageDirty(page)) {
++ /* Flag page needs io for second run optimization. */
++ SetPagePrivate(page);
++ return 1;
++ }
++
++ return 0;
++}
++
++/* Call a function on each page list needing io. */
++static INLINE unsigned
++for_each_io_dev(struct raid_set *rs, struct stripe *stripe,
++ void (*f_io)(struct stripe *stripe, unsigned p))
++{
++ unsigned p = rs->set.raid_devs, r = 0;
++
++ while (p--) {
++ if (page_io(PAGE(stripe, p))) {
++ f_io(stripe, p);
++ r++;
++ }
++ }
++
++ return r;
++}
++
++/* Reconstruct a particular device ?. */
++static INLINE int dev_to_init(struct raid_set *rs)
++{
++ return rs->set.dev_to_init > -1;
++}
++
++/*
++ * Index of device to calculate parity on.
++ * Either the parity device index *or* the selected device to init
++ * after a spare replacement.
++ */
++static INLINE unsigned dev_for_parity(struct stripe *stripe)
++{
++ struct raid_set *rs = RS(stripe->sc);
++
++ return dev_to_init(rs) ? rs->set.dev_to_init : stripe->idx.parity;
++}
++
++/* Return the index of the device to be recovered. */
++static int idx_get(struct raid_set *rs)
++{
++ /* Avoid to read in the pages to be reconstructed anyway. */
++ if (dev_to_init(rs))
++ return rs->set.dev_to_init;
++ else if (rs->set.raid_type->level == raid4)
++ return rs->set.pi;
++
++ return -1;
++}
++
++/* RAID set congested function. */
++static int raid_set_congested(void *congested_data, int bdi_bits)
++{
++ struct raid_set *rs = congested_data;
++ int r = 0; /* Assume uncongested. */
++ unsigned p = rs->set.raid_devs;
++
++ /* If any of our component devices are overloaded. */
++ while (p--) {
++ struct request_queue *q = bdev_get_queue(rs->dev[p].dev->bdev);
++
++ r |= bdi_congested(&q->backing_dev_info, bdi_bits);
++ }
++
++ /* REMOVEME: statistics. */
++ atomic_inc(rs->stats + (r ? S_CONGESTED : S_NOT_CONGESTED));
++ return r;
++}
++
++/* Display RAID set dead message once. */
++static void raid_set_dead(struct raid_set *rs)
++{
++ if (!TestSetRSDead(rs)) {
++ unsigned p;
++ char buf[BDEVNAME_SIZE];
++
++ DMERR("FATAL: too many devices failed -> RAID set dead");
++
++ for (p = 0; p < rs->set.raid_devs; p++) {
++ if (!dev_operational(rs, p))
++ DMERR("device /dev/%s failed",
++ bdevname(rs->dev[p].dev->bdev, buf));
++ }
++ }
++}
++
++/* RAID set degrade check. */
++static INLINE int
++raid_set_check_and_degrade(struct raid_set *rs,
++ struct stripe *stripe, unsigned p)
++{
++ if (test_and_set_bit(DEVICE_FAILED, &rs->dev[p].flags))
++ return -EPERM;
++
++ /* Through an event in case of member device errors. */
++ dm_table_event(rs->ti->table);
++ atomic_inc(&rs->set.failed_devs);
++
++ /* Only log the first member error. */
++ if (!TestSetRSIoError(rs)) {
++ char buf[BDEVNAME_SIZE];
++
++ /* Store index for recovery. */
++ mb();
++ rs->set.ei = p;
++ mb();
++
++ DMERR("CRITICAL: %sio error on device /dev/%s "
++ "in region=%llu; DEGRADING RAID set",
++ stripe ? "" : "FAKED ",
++ bdevname(rs->dev[p].dev->bdev, buf),
++ (unsigned long long) (stripe ? stripe->key : 0));
++ DMERR("further device error messages suppressed");
++ }
++
++ return 0;
++}
++
++static void
++raid_set_check_degrade(struct raid_set *rs, struct stripe *stripe)
++{
++ unsigned p = rs->set.raid_devs;
++
++ while (p--) {
++ struct page *page = PAGE(stripe, p);
++
++ if (PageError(page)) {
++ ClearPageError(page);
++ raid_set_check_and_degrade(rs, stripe, p);
++ }
++ }
++}
++
++/* RAID set upgrade check. */
++static int raid_set_check_and_upgrade(struct raid_set *rs, unsigned p)
++{
++ if (!test_and_clear_bit(DEVICE_FAILED, &rs->dev[p].flags))
++ return -EPERM;
++
++ if (atomic_dec_and_test(&rs->set.failed_devs)) {
++ ClearRSIoError(rs);
++ rs->set.ei = -1;
++ }
++
++ return 0;
++}
++
++/* Lookup a RAID device by name or by major:minor number. */
++union dev_lookup {
++ const char *dev_name;
++ struct raid_dev *dev;
++};
++enum lookup_type { byname, bymajmin, bynumber };
++static int raid_dev_lookup(struct raid_set *rs, enum lookup_type by,
++ union dev_lookup *dl)
++{
++ unsigned p;
++
++ /*
++ * Must be an incremental loop, because the device array
++ * can have empty slots still on calls from raid_ctr()
++ */
++ for (p = 0; p < rs->set.raid_devs; p++) {
++ char buf[BDEVNAME_SIZE];
++ struct raid_dev *dev = rs->dev + p;
++
++ if (!dev->dev)
++ break;
++
++ /* Format dev string appropriately if necessary. */
++ if (by == byname)
++ bdevname(dev->dev->bdev, buf);
++ else if (by == bymajmin)
++ format_dev_t(buf, dev->dev->bdev->bd_dev);
++
++ /* Do the actual check. */
++ if (by == bynumber) {
++ if (dl->dev->dev->bdev->bd_dev ==
++ dev->dev->bdev->bd_dev)
++ return p;
++ } else if (!strcmp(dl->dev_name, buf))
++ return p;
++ }
++
++ return -ENODEV;
++}
++
++/* End io wrapper. */
++static INLINE void
++_bio_endio(struct raid_set *rs, struct bio *bio, int error)
++{
++ /* REMOVEME: statistics. */
++ atomic_inc(rs->stats + (bio_data_dir(bio) == WRITE ?
++ S_BIOS_ENDIO_WRITE : S_BIOS_ENDIO_READ));
++ bio_endio(bio, error);
++ io_put(rs); /* Wake any suspend waiters. */
++}
++
++/*
++ * End small helper functions.
++ */
++
++
++/*
++ * Stripe hash functions
++ */
++/* Initialize/destroy stripe hash. */
++static int hash_init(struct stripe_hash *hash, unsigned stripes)
++{
++ unsigned buckets = 2, max_buckets = stripes / 4;
++ unsigned hash_primes[] = {
++ /* Table of primes for hash_fn/table size optimization. */
++ 3, 7, 13, 27, 53, 97, 193, 389, 769,
++ 1543, 3079, 6151, 12289, 24593,
++ };
++
++ /* Calculate number of buckets (2^^n <= stripes / 4). */
++ while (buckets < max_buckets)
++ buckets <<= 1;
++
++ /* Allocate stripe hash. */
++ hash->hash = vmalloc(buckets * sizeof(*hash->hash));
++ if (!hash->hash)
++ return -ENOMEM;
++
++ hash->buckets = buckets;
++ hash->mask = buckets - 1;
++ hash->shift = ffs(buckets);
++ if (hash->shift > ARRAY_SIZE(hash_primes) + 1)
++ hash->shift = ARRAY_SIZE(hash_primes) + 1;
++
++ BUG_ON(hash->shift - 2 > ARRAY_SIZE(hash_primes) + 1);
++ hash->prime = hash_primes[hash->shift - 2];
++
++ /* Initialize buckets. */
++ while (buckets--)
++ INIT_LIST_HEAD(hash->hash + buckets);
++
++ return 0;
++}
++
++static INLINE void hash_exit(struct stripe_hash *hash)
++{
++ if (hash->hash) {
++ vfree(hash->hash);
++ hash->hash = NULL;
++ }
++}
++
++/* List add (head/tail/locked/unlocked) inlines. */
++enum list_lock_type { LIST_LOCKED, LIST_UNLOCKED };
++#define LIST_DEL(name, list) \
++static void stripe_ ## name ## _del(struct stripe *stripe, \
++ enum list_lock_type lock) { \
++ struct list_head *lh = stripe->lists + (list); \
++ spinlock_t *l = NULL; \
++\
++ if (lock == LIST_LOCKED) { \
++ l = stripe->sc->locks + LOCK_LRU; \
++ spin_lock_irq(l); \
++ } \
++\
++\
++ if (!list_empty(lh)) \
++ list_del_init(lh); \
++\
++ if (lock == LIST_LOCKED) \
++ spin_unlock_irq(l); \
++}
++
++LIST_DEL(hash, LIST_HASH)
++LIST_DEL(lru, LIST_LRU)
++#undef LIST_DEL
++
++enum list_pos_type { POS_HEAD, POS_TAIL };
++#define LIST_ADD(name, list) \
++static void stripe_ ## name ## _add(struct stripe *stripe, \
++ enum list_pos_type pos, \
++ enum list_lock_type lock) { \
++ struct list_head *lh = stripe->lists + (list); \
++ struct stripe_cache *sc = stripe->sc; \
++ spinlock_t *l = NULL; \
++\
++ if (lock == LIST_LOCKED) { \
++ l = sc->locks + LOCK_LRU; \
++ spin_lock_irq(l); \
++ } \
++\
++ if (list_empty(lh)) { \
++ if (pos == POS_HEAD) \
++ list_add(lh, sc->lists + (list)); \
++ else \
++ list_add_tail(lh, sc->lists + (list)); \
++ } \
++\
++ if (lock == LIST_LOCKED) \
++ spin_unlock_irq(l); \
++}
++
++LIST_ADD(endio, LIST_ENDIO)
++LIST_ADD(io, LIST_IO)
++LIST_ADD(lru, LIST_LRU)
++#undef LIST_ADD
++
++#define POP(list) \
++ do { \
++ if (list_empty(sc->lists + list)) \
++ stripe = NULL; \
++ else { \
++ stripe = list_first_entry(&sc->lists[list], \
++ struct stripe, \
++ lists[list]); \
++ list_del_init(&stripe->lists[list]); \
++ } \
++ } while (0);
++
++/* Pop an available stripe off the lru list. */
++static struct stripe *stripe_lru_pop(struct stripe_cache *sc)
++{
++ struct stripe *stripe;
++ spinlock_t *lock = sc->locks + LOCK_LRU;
++
++ spin_lock_irq(lock);
++ POP(LIST_LRU);
++ spin_unlock_irq(lock);
++
++ if (stripe)
++ /* Remove from hash before reuse. */
++ stripe_hash_del(stripe, LIST_UNLOCKED);
++
++ return stripe;
++}
++
++static inline unsigned hash_fn(struct stripe_hash *hash, sector_t key)
++{
++ return (unsigned) (((key * hash->prime) >> hash->shift) & hash->mask);
++}
++
++static inline struct list_head *
++hash_bucket(struct stripe_hash *hash, sector_t key)
++{
++ return hash->hash + hash_fn(hash, key);
++}
++
++/* Insert an entry into a hash. */
++static inline void hash_insert(struct stripe_hash *hash, struct stripe *stripe)
++{
++ list_add(stripe->lists + LIST_HASH, hash_bucket(hash, stripe->key));
++}
++
++/* Insert an entry into the stripe hash. */
++static inline void
++sc_insert(struct stripe_cache *sc, struct stripe *stripe)
++{
++ hash_insert(&sc->hash, stripe);
++}
++
++/* Lookup an entry in the stripe hash. */
++static inline struct stripe *
++stripe_lookup(struct stripe_cache *sc, sector_t key)
++{
++ unsigned c = 0;
++ struct stripe *stripe;
++ struct list_head *bucket = hash_bucket(&sc->hash, key);
++
++ list_for_each_entry(stripe, bucket, lists[LIST_HASH]) {
++ /* REMOVEME: statisics. */
++ if (++c > atomic_read(RS(sc)->stats + S_MAX_LOOKUP))
++ atomic_set(RS(sc)->stats + S_MAX_LOOKUP, c);
++
++ if (stripe->key == key)
++ return stripe;
++ }
++
++ return NULL;
++}
++
++/* Resize the stripe cache hash on size changes. */
++static int hash_resize(struct stripe_cache *sc)
++{
++ /* Resize threshold reached? */
++ if (atomic_read(&sc->stripes) > 2 * atomic_read(&sc->stripes_last)
++ || atomic_read(&sc->stripes) < atomic_read(&sc->stripes_last) / 4) {
++ int r;
++ struct stripe_hash hash, hash_tmp;
++ spinlock_t *lock;
++
++ r = hash_init(&hash, atomic_read(&sc->stripes));
++ if (r)
++ return r;
++
++ lock = sc->locks + LOCK_LRU;
++ spin_lock_irq(lock);
++ if (sc->hash.hash) {
++ unsigned b = sc->hash.buckets;
++ struct list_head *pos, *tmp;
++
++ /* Walk old buckets and insert into new. */
++ while (b--) {
++ list_for_each_safe(pos, tmp, sc->hash.hash + b)
++ hash_insert(&hash,
++ list_entry(pos, struct stripe,
++ lists[LIST_HASH]));
++ }
++
++ }
++
++ memcpy(&hash_tmp, &sc->hash, sizeof(hash_tmp));
++ memcpy(&sc->hash, &hash, sizeof(sc->hash));
++ atomic_set(&sc->stripes_last, atomic_read(&sc->stripes));
++ spin_unlock_irq(lock);
++
++ hash_exit(&hash_tmp);
++ }
++
++ return 0;
++}
++
++/*
++ * Stripe cache locking functions
++ */
++/* Dummy lock function for local RAID4+5. */
++static void *no_lock(sector_t key, enum dm_lock_type type)
++{
++ return &no_lock;
++}
++
++/* Dummy unlock function for local RAID4+5. */
++static void no_unlock(void *lock_handle)
++{
++}
++
++/* No locking (for local RAID 4+5). */
++static struct dm_raid45_locking_type locking_none = {
++ .lock = no_lock,
++ .unlock = no_unlock,
++};
++
++/* Clustered RAID 4+5. */
++/* FIXME: code this. */
++static struct dm_raid45_locking_type locking_cluster = {
++ .lock = no_lock,
++ .unlock = no_unlock,
++};
++
++/* Lock a stripe (for clustering). */
++static int
++stripe_lock(struct raid_set *rs, struct stripe *stripe, int rw, sector_t key)
++{
++ stripe->lock = rs->locking->lock(key, rw == READ ? DM_RAID45_SHARED :
++ DM_RAID45_EX);
++ return stripe->lock ? 0 : -EPERM;
++}
++
++/* Unlock a stripe (for clustering). */
++static void stripe_unlock(struct raid_set *rs, struct stripe *stripe)
++{
++ rs->locking->unlock(stripe->lock);
++ stripe->lock = NULL;
++}
++
++/*
++ * Stripe cache functions.
++ */
++/*
++ * Invalidate all page lists pages of a stripe.
++ *
++ * I only keep state for the whole list in the first page.
++ */
++static INLINE void
++stripe_pages_invalidate(struct stripe *stripe)
++{
++ unsigned p = RS(stripe->sc)->set.raid_devs;
++
++ while (p--) {
++ struct page *page = PAGE(stripe, p);
++
++ ProhibitPageIO(page);
++ ClearPageChecked(page);
++ ClearPageDirty(page);
++ ClearPageError(page);
++ clear_page_locked(page);
++ ClearPagePrivate(page);
++ ClearPageUptodate(page);
++ }
++}
++
++/* Prepare stripe for (re)use. */
++static INLINE void stripe_invalidate(struct stripe *stripe)
++{
++ stripe->io.flags = 0;
++ stripe_pages_invalidate(stripe);
++}
++
++/* Allow io on all chunks of a stripe. */
++static INLINE void stripe_allow_io(struct stripe *stripe)
++{
++ unsigned p = RS(stripe->sc)->set.raid_devs;
++
++ while (p--)
++ AllowPageIO(PAGE(stripe, p));
++}
++
++/* Initialize a stripe. */
++static void
++stripe_init(struct stripe_cache *sc, struct stripe *stripe)
++{
++ unsigned p = RS(sc)->set.raid_devs;
++ unsigned i;
++
++ /* Work all io chunks. */
++ while (p--) {
++ struct stripe_set *ss = stripe->ss + p;
++
++ stripe->obj[p].private = ss;
++ ss->stripe = stripe;
++
++ i = ARRAY_SIZE(ss->bl);
++ while (i--)
++ bio_list_init(ss->bl + i);
++ }
++
++ stripe->sc = sc;
++
++ i = ARRAY_SIZE(stripe->lists);
++ while (i--)
++ INIT_LIST_HEAD(stripe->lists + i);
++
++ atomic_set(&stripe->cnt, 0);
++ atomic_set(&stripe->io.pending, 0);
++
++ stripe_invalidate(stripe);
++}
++
++/* Number of pages per chunk. */
++static inline unsigned chunk_pages(unsigned io_size)
++{
++ return dm_div_up(io_size, SECTORS_PER_PAGE);
++}
++
++/* Number of pages per stripe. */
++static inline unsigned stripe_pages(struct raid_set *rs, unsigned io_size)
++{
++ return chunk_pages(io_size) * rs->set.raid_devs;
++}
++
++/* Initialize part of page_list (recovery). */
++static INLINE void stripe_zero_pl_part(struct stripe *stripe, unsigned p,
++ unsigned start, unsigned count)
++{
++ unsigned pages = chunk_pages(count);
++ /* Get offset into the page_list. */
++ struct page_list *pl = pl_elem(PL(stripe, p), start / SECTORS_PER_PAGE);
++
++ BUG_ON(!pl);
++ while (pl && pages--) {
++ BUG_ON(!pl->page);
++ memset(page_address(pl->page), 0, PAGE_SIZE);
++ pl = pl->next;
++ }
++}
++
++/* Initialize parity chunk of stripe. */
++static INLINE void stripe_zero_chunk(struct stripe *stripe, unsigned p)
++{
++ stripe_zero_pl_part(stripe, p, 0, stripe->io.size);
++}
++
++/* Return dynamic stripe structure size. */
++static INLINE size_t stripe_size(struct raid_set *rs)
++{
++ return sizeof(struct stripe) +
++ rs->set.raid_devs * sizeof(struct stripe_set);
++}
++
++/* Allocate a stripe and its memory object. */
++/* XXX adjust to cope with stripe cache and recovery stripe caches. */
++enum grow { SC_GROW, SC_KEEP };
++static struct stripe *stripe_alloc(struct stripe_cache *sc,
++ struct dm_mem_cache_client *mc,
++ enum grow grow)
++{
++ int r;
++ struct stripe *stripe;
++
++ stripe = kmem_cache_zalloc(sc->kc.cache, GFP_KERNEL);
++ if (stripe) {
++ /* Grow the dm-mem-cache by one object. */
++ if (grow == SC_GROW) {
++ r = dm_mem_cache_grow(mc, 1);
++ if (r)
++ goto err_free;
++ }
++
++ stripe->obj = dm_mem_cache_alloc(mc);
++ if (!stripe->obj)
++ goto err_shrink;
++
++ stripe_init(sc, stripe);
++ }
++
++ return stripe;
++
++err_shrink:
++ if (grow == SC_GROW)
++ dm_mem_cache_shrink(mc, 1);
++err_free:
++ kmem_cache_free(sc->kc.cache, stripe);
++ return NULL;
++}
++
++/*
++ * Free a stripes memory object, shrink the
++ * memory cache and free the stripe itself
++ */
++static void stripe_free(struct stripe *stripe, struct dm_mem_cache_client *mc)
++{
++ dm_mem_cache_free(mc, stripe->obj);
++ dm_mem_cache_shrink(mc, 1);
++ kmem_cache_free(stripe->sc->kc.cache, stripe);
++}
++
++/* Free the recovery stripe. */
++static void stripe_recover_free(struct raid_set *rs)
++{
++ struct recover *rec = &rs->recover;
++ struct list_head *stripes = &rec->stripes;
++
++ while (!list_empty(stripes)) {
++ struct stripe *stripe = list_first_entry(stripes, struct stripe,
++ lists[LIST_RECOVER]);
++ list_del(stripe->lists + LIST_RECOVER);
++ stripe_free(stripe, rec->mem_cache_client);
++ }
++}
++
++/* Push a stripe safely onto the endio list to be handled by do_endios(). */
++static INLINE void stripe_endio_push(struct stripe *stripe)
++{
++ int wake;
++ unsigned long flags;
++ struct stripe_cache *sc = stripe->sc;
++ spinlock_t *lock = sc->locks + LOCK_ENDIO;
++
++ spin_lock_irqsave(lock, flags);
++ wake = list_empty(sc->lists + LIST_ENDIO);
++ stripe_endio_add(stripe, POS_HEAD, LIST_UNLOCKED);
++ spin_unlock_irqrestore(lock, flags);
++
++ if (wake)
++ wake_do_raid(RS(sc));
++}
++
++/* Protected check for stripe cache endio list empty. */
++static INLINE int stripe_endio_empty(struct stripe_cache *sc)
++{
++ int r;
++ spinlock_t *lock = sc->locks + LOCK_ENDIO;
++
++ spin_lock_irq(lock);
++ r = list_empty(sc->lists + LIST_ENDIO);
++ spin_unlock_irq(lock);
++
++ return r;
++}
++
++/* Pop a stripe off safely off the endio list. */
++static struct stripe *stripe_endio_pop(struct stripe_cache *sc)
++{
++ struct stripe *stripe;
++ spinlock_t *lock = sc->locks + LOCK_ENDIO;
++
++ /* This runs in parallel with endio(). */
++ spin_lock_irq(lock);
++ POP(LIST_ENDIO)
++ spin_unlock_irq(lock);
++ return stripe;
++}
++
++#undef POP
++
++/* Evict stripe from cache. */
++static void stripe_evict(struct stripe *stripe)
++{
++ struct raid_set *rs = RS(stripe->sc);
++ stripe_hash_del(stripe, LIST_UNLOCKED); /* Take off hash. */
++
++ if (list_empty(stripe->lists + LIST_LRU)) {
++ stripe_lru_add(stripe, POS_TAIL, LIST_LOCKED);
++ atomic_inc(rs->stats + S_EVICT); /* REMOVEME: statistics. */
++ }
++}
++
++/* Grow stripe cache. */
++static int
++sc_grow(struct stripe_cache *sc, unsigned stripes, enum grow grow)
++{
++ int r = 0;
++ struct raid_set *rs = RS(sc);
++
++ /* Try to allocate this many (additional) stripes. */
++ while (stripes--) {
++ struct stripe *stripe =
++ stripe_alloc(sc, sc->mem_cache_client, grow);
++
++ if (likely(stripe)) {
++ stripe->io.size = rs->set.io_size;
++ stripe_lru_add(stripe, POS_TAIL, LIST_LOCKED);
++ atomic_inc(&sc->stripes);
++ } else {
++ r = -ENOMEM;
++ break;
++ }
++ }
++
++ ClearRSScBusy(rs);
++ return r ? r : hash_resize(sc);
++}
++
++/* Shrink stripe cache. */
++static int sc_shrink(struct stripe_cache *sc, unsigned stripes)
++{
++ int r = 0;
++
++ /* Try to get unused stripe from LRU list. */
++ while (stripes--) {
++ struct stripe *stripe;
++
++ stripe = stripe_lru_pop(sc);
++ if (stripe) {
++ /* An lru stripe may never have ios pending! */
++ BUG_ON(stripe_io(stripe));
++ stripe_free(stripe, sc->mem_cache_client);
++ atomic_dec(&sc->stripes);
++ } else {
++ r = -ENOENT;
++ break;
++ }
++ }
++
++ /* Check if stats are still sane. */
++ if (atomic_read(&sc->max_active_stripes) >
++ atomic_read(&sc->stripes))
++ atomic_set(&sc->max_active_stripes, 0);
++
++ if (r)
++ return r;
++
++ ClearRSScBusy(RS(sc));
++ return hash_resize(sc);
++}
++
++/* Create stripe cache. */
++static int sc_init(struct raid_set *rs, unsigned stripes)
++{
++ unsigned i, nr;
++ struct stripe_cache *sc = &rs->sc;
++ struct stripe *stripe;
++ struct recover *rec = &rs->recover;
++
++ /* Initialize lists and locks. */
++ i = ARRAY_SIZE(sc->lists);
++ while (i--)
++ INIT_LIST_HEAD(sc->lists + i);
++
++ i = NR_LOCKS;
++ while (i--)
++ spin_lock_init(sc->locks + i);
++
++ /* Initialize atomic variables. */
++ atomic_set(&sc->stripes, 0);
++ atomic_set(&sc->stripes_last, 0);
++ atomic_set(&sc->stripes_to_shrink, 0);
++ atomic_set(&sc->active_stripes, 0);
++ atomic_set(&sc->max_active_stripes, 0); /* REMOVEME: statistics. */
++
++ /*
++ * We need a runtime unique # to suffix the kmem cache name
++ * because we'll have one for each active RAID set.
++ */
++ nr = atomic_inc_return(&_stripe_sc_nr);
++ sprintf(sc->kc.name, "%s_%d", TARGET, nr);
++ sc->kc.cache = kmem_cache_create(sc->kc.name, stripe_size(rs),
++ 0, 0, NULL);
++ if (!sc->kc.cache)
++ return -ENOMEM;
++
++ /* Create memory cache client context for RAID stripe cache. */
++ sc->mem_cache_client =
++ dm_mem_cache_client_create(stripes, rs->set.raid_devs,
++ chunk_pages(rs->set.io_size));
++ if (IS_ERR(sc->mem_cache_client))
++ return PTR_ERR(sc->mem_cache_client);
++
++ /* Create memory cache client context for RAID recovery stripe(s). */
++ rec->mem_cache_client =
++ dm_mem_cache_client_create(MAX_RECOVER, rs->set.raid_devs,
++ chunk_pages(rec->io_size));
++ if (IS_ERR(rec->mem_cache_client))
++ return PTR_ERR(rec->mem_cache_client);
++
++ /* Allocate stripe for set recovery. */
++ /* XXX: cope with MAX_RECOVERY. */
++ INIT_LIST_HEAD(&rec->stripes);
++ for (i = 0; i < MAX_RECOVER; i++) {
++ stripe = stripe_alloc(sc, rec->mem_cache_client, SC_KEEP);
++ if (!stripe)
++ return -ENOMEM;
++
++ SetStripeRecover(stripe);
++ stripe->io.size = rec->io_size;
++ list_add(stripe->lists + LIST_RECOVER, &rec->stripes);
++ }
++
++ /*
++ * Allocate the stripe objetcs from the
++ * cache and add them to the LRU list.
++ */
++ return sc_grow(sc, stripes, SC_KEEP);
++}
++
++/* Destroy the stripe cache. */
++static void sc_exit(struct stripe_cache *sc)
++{
++ if (sc->kc.cache) {
++ BUG_ON(sc_shrink(sc, atomic_read(&sc->stripes)));
++ kmem_cache_destroy(sc->kc.cache);
++ }
++
++ if (sc->mem_cache_client)
++ dm_mem_cache_client_destroy(sc->mem_cache_client);
++
++ ClearRSRecover(RS(sc));
++ stripe_recover_free(RS(sc));
++ if (RS(sc)->recover.mem_cache_client)
++ dm_mem_cache_client_destroy(RS(sc)->recover.mem_cache_client);
++
++ hash_exit(&sc->hash);
++}
++
++/*
++ * Calculate RAID address
++ *
++ * Delivers tuple with the index of the data disk holding the chunk
++ * in the set, the parity disks index and the start of the stripe
++ * within the address space of the set (used as the stripe cache hash key).
++ */
++/* thx MD. */
++static struct address *
++raid_address(struct raid_set *rs, sector_t sector, struct address *addr)
++{
++ unsigned data_devs = rs->set.data_devs, di, pi,
++ raid_devs = rs->set.raid_devs;
++ sector_t stripe, tmp;
++
++ /*
++ * chunk_number = sector / chunk_size
++ * stripe = chunk_number / data_devs
++ * di = stripe % data_devs;
++ */
++ stripe = sector >> rs->set.chunk_shift;
++ di = sector_div(stripe, data_devs);
++
++ switch (rs->set.raid_type->level) {
++ case raid5:
++ tmp = stripe;
++ pi = sector_div(tmp, raid_devs);
++
++ switch (rs->set.raid_type->algorithm) {
++ case left_asym: /* Left asymmetric. */
++ pi = data_devs - pi;
++ case right_asym: /* Right asymmetric. */
++ if (di >= pi)
++ di++;
++ break;
++
++ case left_sym: /* Left symmetric. */
++ pi = data_devs - pi;
++ case right_sym: /* Right symmetric. */
++ di = (pi + di + 1) % raid_devs;
++ break;
++
++ default:
++ DMERR("Unknown RAID algorithm %d",
++ rs->set.raid_type->algorithm);
++ goto out;
++ }
++
++ break;
++
++ case raid4:
++ pi = rs->set.pi;
++ if (di >= pi)
++ di++;
++ break;
++
++ default:
++ DMERR("Unknown RAID level %d", rs->set.raid_type->level);
++ goto out;
++ }
++
++ /*
++ * Hash key = start offset on any single device of the RAID set;
++ * adjusted in case io size differs from chunk size.
++ */
++ addr->key = (stripe << rs->set.chunk_shift) +
++ (sector & rs->set.io_shift_mask);
++ addr->di = di;
++ addr->pi = pi;
++
++out:
++ return addr;
++}
++
++/*
++ * Copy data across between stripe pages and bio vectors.
++ *
++ * Pay attention to data alignment in stripe and bio pages.
++ */
++static void
++bio_copy_page_list(int rw, struct stripe *stripe,
++ struct page_list *pl, struct bio *bio)
++{
++ unsigned i, page_offset;
++ void *page_addr;
++ struct raid_set *rs = RS(stripe->sc);
++ struct bio_vec *bv;
++
++ /* Get start page in page list for this sector. */
++ i = (bio->bi_sector & rs->set.io_mask) / SECTORS_PER_PAGE;
++ pl = pl_elem(pl, i);
++
++ page_addr = page_address(pl->page);
++ page_offset = to_bytes(bio->bi_sector & (SECTORS_PER_PAGE - 1));
++
++ /* Walk all segments and copy data across between bio_vecs and pages. */
++ bio_for_each_segment(bv, bio, i) {
++ int len = bv->bv_len, size;
++ unsigned bio_offset = 0;
++ void *bio_addr = __bio_kmap_atomic(bio, i, KM_USER0);
++redo:
++ size = (page_offset + len > PAGE_SIZE) ?
++ PAGE_SIZE - page_offset : len;
++
++ if (rw == READ)
++ memcpy(bio_addr + bio_offset,
++ page_addr + page_offset, size);
++ else
++ memcpy(page_addr + page_offset,
++ bio_addr + bio_offset, size);
++
++ page_offset += size;
++ if (page_offset == PAGE_SIZE) {
++ /*
++ * We reached the end of the chunk page ->
++ * need refer to the next one to copy more data.
++ */
++ len -= size;
++ if (len) {
++ /* Get next page. */
++ pl = pl->next;
++ BUG_ON(!pl);
++ page_addr = page_address(pl->page);
++ page_offset = 0;
++ bio_offset += size;
++ /* REMOVEME: statistics. */
++ atomic_inc(rs->stats + S_BIO_COPY_PL_NEXT);
++ goto redo;
++ }
++ }
++
++ __bio_kunmap_atomic(bio_addr, KM_USER0);
++ }
++}
++
++/*
++ * Xor optimization macros.
++ */
++/* Xor data pointer declaration and initialization macros. */
++#define DECLARE_2 unsigned long *d0 = data[0], *d1 = data[1]
++#define DECLARE_3 DECLARE_2, *d2 = data[2]
++#define DECLARE_4 DECLARE_3, *d3 = data[3]
++#define DECLARE_5 DECLARE_4, *d4 = data[4]
++#define DECLARE_6 DECLARE_5, *d5 = data[5]
++#define DECLARE_7 DECLARE_6, *d6 = data[6]
++#define DECLARE_8 DECLARE_7, *d7 = data[7]
++
++/* Xor unrole macros. */
++#define D2(n) d0[n] = d0[n] ^ d1[n]
++#define D3(n) D2(n) ^ d2[n]
++#define D4(n) D3(n) ^ d3[n]
++#define D5(n) D4(n) ^ d4[n]
++#define D6(n) D5(n) ^ d5[n]
++#define D7(n) D6(n) ^ d6[n]
++#define D8(n) D7(n) ^ d7[n]
++
++#define X_2(macro, offset) macro(offset); macro(offset + 1);
++#define X_4(macro, offset) X_2(macro, offset); X_2(macro, offset + 2);
++#define X_8(macro, offset) X_4(macro, offset); X_4(macro, offset + 4);
++#define X_16(macro, offset) X_8(macro, offset); X_8(macro, offset + 8);
++#define X_32(macro, offset) X_16(macro, offset); X_16(macro, offset + 16);
++#define X_64(macro, offset) X_32(macro, offset); X_32(macro, offset + 32);
++
++/* Define a _xor_#chunks_#xors_per_run() function. */
++#define _XOR(chunks, xors_per_run) \
++static void _xor ## chunks ## _ ## xors_per_run(unsigned long **data) \
++{ \
++ unsigned end = XOR_SIZE / sizeof(data[0]), i; \
++ DECLARE_ ## chunks; \
++\
++ for (i = 0; i < end; i += xors_per_run) { \
++ X_ ## xors_per_run(D ## chunks, i); \
++ } \
++}
++
++/* Define xor functions for 2 - 8 chunks. */
++#define MAKE_XOR_PER_RUN(xors_per_run) \
++ _XOR(2, xors_per_run); _XOR(3, xors_per_run); \
++ _XOR(4, xors_per_run); _XOR(5, xors_per_run); \
++ _XOR(6, xors_per_run); _XOR(7, xors_per_run); \
++ _XOR(8, xors_per_run);
++
++MAKE_XOR_PER_RUN(8) /* Define _xor_*_8() functions. */
++MAKE_XOR_PER_RUN(16) /* Define _xor_*_16() functions. */
++MAKE_XOR_PER_RUN(32) /* Define _xor_*_32() functions. */
++MAKE_XOR_PER_RUN(64) /* Define _xor_*_64() functions. */
++
++#define MAKE_XOR(xors_per_run) \
++struct { \
++ void (*f)(unsigned long **); \
++} static xor_funcs ## xors_per_run[] = { \
++ { NULL }, \
++ { NULL }, \
++ { _xor2_ ## xors_per_run }, \
++ { _xor3_ ## xors_per_run }, \
++ { _xor4_ ## xors_per_run }, \
++ { _xor5_ ## xors_per_run }, \
++ { _xor6_ ## xors_per_run }, \
++ { _xor7_ ## xors_per_run }, \
++ { _xor8_ ## xors_per_run }, \
++}; \
++\
++static void xor_ ## xors_per_run(unsigned n, unsigned long **data) \
++{ \
++ /* Call respective function for amount of chunks. */ \
++ xor_funcs ## xors_per_run[n].f(data); \
++}
++
++/* Define xor_8() - xor_64 functions. */
++MAKE_XOR(8)
++MAKE_XOR(16)
++MAKE_XOR(32)
++MAKE_XOR(64)
++
++/* Maximum number of chunks, which can be xor'ed in one go. */
++#define XOR_CHUNKS_MAX (ARRAY_SIZE(xor_funcs8) - 1)
++
++struct xor_func {
++ xor_function_t f;
++ const char *name;
++} static xor_funcs[] = {
++ {xor_8, "xor_8"},
++ {xor_16, "xor_16"},
++ {xor_32, "xor_32"},
++ {xor_64, "xor_64"},
++};
++
++/*
++ * Calculate crc.
++ *
++ * This indexes into the page list of the stripe.
++ *
++ * All chunks will be xored into the parity chunk
++ * in maximum groups of xor.chunks.
++ *
++ * FIXME: try mapping the pages on discontiguous memory.
++ */
++static void xor(struct stripe *stripe, unsigned pi, unsigned sector)
++{
++ struct raid_set *rs = RS(stripe->sc);
++ unsigned max_chunks = rs->xor.chunks, n, p;
++ unsigned o = sector / SECTORS_PER_PAGE; /* Offset into the page_list. */
++ unsigned long **d = rs->data;
++ xor_function_t xor_f = rs->xor.f->f;
++
++ /* Address of parity page to xor into. */
++ d[0] = page_address(pl_elem(PL(stripe, pi), o)->page);
++
++ /* Preset pointers to data pages. */
++ for (n = 1, p = rs->set.raid_devs; p--; ) {
++ if (p != pi && PageIO(PAGE(stripe, p)))
++ d[n++] = page_address(pl_elem(PL(stripe, p), o)->page);
++
++ /* If max chunks -> xor .*/
++ if (n == max_chunks) {
++ xor_f(n, d);
++ n = 1;
++ }
++ }
++
++ /* If chunks -> xor. */
++ if (n > 1)
++ xor_f(n, d);
++
++ /* Set parity page uptodate and clean. */
++ page_set(PAGE(stripe, pi), CLEAN);
++}
++
++/* Common xor loop through all stripe page lists. */
++static void common_xor(struct stripe *stripe, sector_t count,
++ unsigned off, unsigned p)
++{
++ unsigned sector;
++
++ for (sector = off; sector < count; sector += SECTORS_PER_XOR)
++ xor(stripe, p, sector);
++
++ atomic_inc(RS(stripe->sc)->stats + S_XORS); /* REMOVEME: statistics. */
++}
++
++/*
++ * Calculate parity sectors on intact stripes.
++ *
++ * Need to calculate raid address for recover stripe, because its
++ * chunk sizes differs and is typically larger than io chunk size.
++ */
++static void parity_xor(struct stripe *stripe)
++{
++ struct raid_set *rs = RS(stripe->sc);
++ unsigned chunk_size = rs->set.chunk_size,
++ io_size = stripe->io.size,
++ xor_size = chunk_size > io_size ? io_size : chunk_size;
++ sector_t off;
++
++ /* This can be the recover stripe with a larger io size. */
++ for (off = 0; off < io_size; off += xor_size) {
++ unsigned pi;
++
++ /*
++ * Recover stripe likely is bigger than regular io
++ * ones and has no precalculated parity disk index ->
++ * need to calculate RAID address.
++ */
++ if (unlikely(StripeRecover(stripe))) {
++ struct address addr;
++
++ raid_address(rs,
++ (stripe->key + off) * rs->set.data_devs,
++ &addr);
++ pi = addr.pi;
++ stripe_zero_pl_part(stripe, pi, off,
++ rs->set.chunk_size);
++ } else
++ pi = stripe->idx.parity;
++
++ common_xor(stripe, xor_size, off, pi);
++ page_set(PAGE(stripe, pi), DIRTY);
++ }
++}
++
++/* Reconstruct missing chunk. */
++static void reconstruct_xor(struct stripe *stripe)
++{
++ struct raid_set *rs = RS(stripe->sc);
++ int p = stripe->idx.recover;
++
++ BUG_ON(p < 0);
++
++ /* REMOVEME: statistics. */
++ atomic_inc(rs->stats + (raid_set_degraded(rs) ?
++ S_RECONSTRUCT_EI : S_RECONSTRUCT_DEV));
++
++ /* Zero chunk to be reconstructed. */
++ stripe_zero_chunk(stripe, p);
++ common_xor(stripe, stripe->io.size, 0, p);
++}
++
++/*
++ * Try getting a stripe either from the hash or from the lru list
++ */
++static inline void _stripe_get(struct stripe *stripe)
++{
++ atomic_inc(&stripe->cnt);
++}
++
++static struct stripe *stripe_get(struct raid_set *rs, struct address *addr)
++{
++ struct stripe_cache *sc = &rs->sc;
++ struct stripe *stripe;
++
++ stripe = stripe_lookup(sc, addr->key);
++ if (stripe) {
++ _stripe_get(stripe);
++ /* Remove from the lru list if on. */
++ stripe_lru_del(stripe, LIST_LOCKED);
++ atomic_inc(rs->stats + S_HITS_1ST); /* REMOVEME: statistics. */
++ } else {
++ /* Second try to get an LRU stripe. */
++ stripe = stripe_lru_pop(sc);
++ if (stripe) {
++ _stripe_get(stripe);
++ /* Invalidate before reinserting with changed key. */
++ stripe_invalidate(stripe);
++ stripe->key = addr->key;
++ stripe->region = dm_rh_sector_to_region(rs->recover.rh,
++ addr->key);
++ stripe->idx.parity = addr->pi;
++ sc_insert(sc, stripe);
++ /* REMOVEME: statistics. */
++ atomic_inc(rs->stats + S_INSCACHE);
++ }
++ }
++
++ return stripe;
++}
++
++/*
++ * Decrement reference count on a stripe.
++ *
++ * Move it to list of LRU stripes if zero.
++ */
++static void stripe_put(struct stripe *stripe)
++{
++ if (atomic_dec_and_test(&stripe->cnt)) {
++ if (TestClearStripeActive(stripe))
++ atomic_dec(&stripe->sc->active_stripes);
++
++ /* Put stripe onto the LRU list. */
++ stripe_lru_add(stripe, POS_TAIL, LIST_LOCKED);
++ }
++
++ BUG_ON(atomic_read(&stripe->cnt) < 0);
++}
++
++/*
++ * Process end io
++ *
++ * I need to do it here because I can't in interrupt
++ *
++ * Read and write functions are split in order to avoid
++ * conditionals in the main loop for performamce reasons.
++ */
++
++/* Helper read bios on a page list. */
++static void _bio_copy_page_list(struct stripe *stripe, struct page_list *pl,
++ struct bio *bio)
++{
++ bio_copy_page_list(READ, stripe, pl, bio);
++}
++
++/* Helper write bios on a page list. */
++static void _rh_dec(struct stripe *stripe, struct page_list *pl,
++ struct bio *bio)
++{
++ dm_rh_dec(RS(stripe->sc)->recover.rh, stripe->region);
++}
++
++/* End io all bios on a page list. */
++static inline int
++page_list_endio(int rw, struct stripe *stripe, unsigned p, unsigned *count)
++{
++ int r = 0;
++ struct bio_list *bl = BL(stripe, p, rw);
++
++ if (!bio_list_empty(bl)) {
++ struct page_list *pl = PL(stripe, p);
++ struct page *page = pl->page;
++
++ if (PageLocked(page))
++ r = -EBUSY;
++ /*
++ * FIXME: PageUptodate() not cleared
++ * properly for missing chunks ?
++ */
++ else if (PageUptodate(page)) {
++ struct bio *bio;
++ struct raid_set *rs = RS(stripe->sc);
++ void (*h_f)(struct stripe *, struct page_list *,
++ struct bio *) =
++ (rw == READ) ? _bio_copy_page_list : _rh_dec;
++
++ while ((bio = bio_list_pop(bl))) {
++ h_f(stripe, pl, bio);
++ _bio_endio(rs, bio, 0);
++ stripe_put(stripe);
++ if (count)
++ (*count)++;
++ }
++ } else
++ r = -EAGAIN;
++ }
++
++ return r;
++}
++
++/*
++ * End io all reads/writes on a stripe copying
++ * read date accross from stripe to bios.
++ */
++static int stripe_endio(int rw, struct stripe *stripe, unsigned *count)
++{
++ int r = 0;
++ unsigned p = RS(stripe->sc)->set.raid_devs;
++
++ while (p--) {
++ int rr = page_list_endio(rw, stripe, p, count);
++
++ if (rr && r != -EIO)
++ r = rr;
++ }
++
++ return r;
++}
++
++/* Fail all ios on a bio list and return # of bios. */
++static unsigned
++bio_list_fail(struct raid_set *rs, struct stripe *stripe, struct bio_list *bl)
++{
++ unsigned r;
++ struct bio *bio;
++
++ raid_set_dead(rs);
++
++ /* Update region counters. */
++ if (stripe) {
++ struct dm_rh_client *rh = rs->recover.rh;
++
++ bio_list_for_each(bio, bl) {
++ if (bio_data_dir(bio) == WRITE)
++ dm_rh_dec(rh, stripe->region);
++ }
++ }
++
++ /* Error end io all bios. */
++ for (r = 0; (bio = bio_list_pop(bl)); r++)
++ _bio_endio(rs, bio, -EIO);
++
++ return r;
++}
++
++/* Fail all ios of a bio list of a stripe and drop io pending count. */
++static void
++stripe_bio_list_fail(struct raid_set *rs, struct stripe *stripe,
++ struct bio_list *bl)
++{
++ unsigned put = bio_list_fail(rs, stripe, bl);
++
++ while (put--)
++ stripe_put(stripe);
++}
++
++/* Fail all ios hanging off all bio lists of a stripe. */
++static void stripe_fail_io(struct stripe *stripe)
++{
++ struct raid_set *rs = RS(stripe->sc);
++ unsigned p = rs->set.raid_devs;
++
++ stripe_evict(stripe);
++
++ while (p--) {
++ struct stripe_set *ss = stripe->ss + p;
++ int i = ARRAY_SIZE(ss->bl);
++
++ while (i--)
++ stripe_bio_list_fail(rs, stripe, ss->bl + i);
++ }
++}
++
++/*
++ * Handle all stripes by handing them to the daemon, because we can't
++ * map their pages to copy the data in interrupt context.
++ *
++ * We don't want to handle them here either, while interrupts are disabled.
++ */
++
++/* Read/write endio function for dm-io (interrupt context). */
++static void endio(unsigned long error, void *context)
++{
++ struct dm_mem_cache_object *obj = context;
++ struct stripe_set *ss = obj->private;
++ struct stripe *stripe = ss->stripe;
++ struct page *page = obj->pl->page;
++
++ if (unlikely(error))
++ stripe_error(stripe, page);
++ else
++ page_set(page, CLEAN);
++
++ clear_page_locked(page);
++ stripe_io_dec(stripe);
++
++ /* Add stripe to endio list and wake daemon. */
++ stripe_endio_push(stripe);
++}
++
++/*
++ * Recovery io throttling
++ */
++/* Conditionally reset io counters. */
++enum count_type { IO_WORK = 0, IO_RECOVER };
++static int recover_io_reset(struct raid_set *rs)
++{
++ unsigned long j = jiffies;
++
++ /* Pay attention to jiffies overflows. */
++ if (j > rs->recover.last_jiffies + HZ
++ || j < rs->recover.last_jiffies) {
++ rs->recover.last_jiffies = j;
++ atomic_set(rs->recover.io_count + IO_WORK, 0);
++ atomic_set(rs->recover.io_count + IO_RECOVER, 0);
++ return 1;
++ }
++
++ return 0;
++}
++
++/* Count ios. */
++static INLINE void
++recover_io_count(struct raid_set *rs, struct stripe *stripe)
++{
++ if (RSRecover(rs)) {
++ recover_io_reset(rs);
++ atomic_inc(rs->recover.io_count +
++ (StripeRecover(stripe) ? IO_RECOVER : IO_WORK));
++ }
++}
++
++/* Read/Write a page_list asynchronously. */
++static void page_list_rw(struct stripe *stripe, unsigned p)
++{
++ struct stripe_cache *sc = stripe->sc;
++ struct raid_set *rs = RS(sc);
++ struct dm_mem_cache_object *obj = stripe->obj + p;
++ struct page_list *pl = obj->pl;
++ struct page *page = pl->page;
++ struct raid_dev *dev = rs->dev + p;
++ struct dm_io_region io = {
++ .bdev = dev->dev->bdev,
++ .sector = stripe->key,
++ .count = stripe->io.size,
++ };
++ struct dm_io_request control = {
++ .bi_rw = PageDirty(page) ? WRITE : READ,
++ .mem.type = DM_IO_PAGE_LIST,
++ .mem.ptr.pl = pl,
++ .mem.offset = 0,
++ .notify.fn = endio,
++ .notify.context = obj,
++ .client = sc->dm_io_client,
++ };
++
++ BUG_ON(PageLocked(page));
++
++ /*
++ * Don't rw past end of device, which can happen, because
++ * typically sectors_per_dev isn't divisable by io_size.
++ */
++ if (unlikely(io.sector + io.count > rs->set.sectors_per_dev))
++ io.count = rs->set.sectors_per_dev - io.sector;
++
++ io.sector += dev->start; /* Add <offset>. */
++ recover_io_count(rs, stripe); /* Recovery io accounting. */
++
++ /* REMOVEME: statistics. */
++ atomic_inc(rs->stats +
++ (PageDirty(page) ? S_DM_IO_WRITE : S_DM_IO_READ));
++
++ ClearPageError(page);
++ set_page_locked(page);
++ io_dev_queued(dev);
++ BUG_ON(dm_io(&control, 1, &io, NULL));
++}
++
++/*
++ * Write dirty / read not uptodate page lists of a stripe.
++ */
++static unsigned stripe_page_lists_rw(struct raid_set *rs, struct stripe *stripe)
++{
++ unsigned r;
++
++ /*
++ * Increment the pending count on the stripe
++ * first, so that we don't race in endio().
++ *
++ * An inc (IO) is needed for any page:
++ *
++ * o not uptodate
++ * o dirtied by writes merged
++ * o dirtied by parity calculations
++ */
++ r = for_each_io_dev(rs, stripe, _stripe_io_inc);
++ if (r) {
++ /* io needed: chunks are not uptodate/dirty. */
++ int max; /* REMOVEME: */
++ struct stripe_cache *sc = &rs->sc;
++
++ if (!TestSetStripeActive(stripe))
++ atomic_inc(&sc->active_stripes);
++
++ /* Take off the lru list in case it got added there. */
++ stripe_lru_del(stripe, LIST_LOCKED);
++
++ /* Submit actual io. */
++ for_each_io_dev(rs, stripe, page_list_rw);
++
++ /* REMOVEME: statistics */
++ max = sc_active(sc);
++ if (atomic_read(&sc->max_active_stripes) < max)
++ atomic_set(&sc->max_active_stripes, max);
++
++ atomic_inc(rs->stats + S_FLUSHS);
++ /* END REMOVEME: statistics */
++ }
++
++ return r;
++}
++
++/* Work in all pending writes. */
++static INLINE void _writes_merge(struct stripe *stripe, unsigned p)
++{
++ struct bio_list *write = BL(stripe, p, WRITE);
++
++ if (!bio_list_empty(write)) {
++ struct page_list *pl = stripe->obj[p].pl;
++ struct bio *bio;
++ struct bio_list *write_merged = BL(stripe, p, WRITE_MERGED);
++
++ /*
++ * We can play with the lists without holding a lock,
++ * because it is just us accessing them anyway.
++ */
++ bio_list_for_each(bio, write)
++ bio_copy_page_list(WRITE, stripe, pl, bio);
++
++ bio_list_merge(write_merged, write);
++ bio_list_init(write);
++ page_set(pl->page, DIRTY);
++ }
++}
++
++/* Merge in all writes hence dirtying respective pages. */
++static INLINE void writes_merge(struct stripe *stripe)
++{
++ unsigned p = RS(stripe->sc)->set.raid_devs;
++
++ while (p--)
++ _writes_merge(stripe, p);
++}
++
++/* Check, if a chunk gets completely overwritten. */
++static INLINE int stripe_check_overwrite(struct stripe *stripe, unsigned p)
++{
++ unsigned sectors = 0;
++ struct bio *bio;
++ struct bio_list *bl = BL(stripe, p, WRITE);
++
++ bio_list_for_each(bio, bl)
++ sectors += bio_sectors(bio);
++
++ return sectors == RS(stripe->sc)->set.io_size;
++}
++
++/*
++ * Prepare stripe to avoid io on broken/reconstructed
++ * drive in order to reconstruct date on endio.
++ */
++enum prepare_type { IO_ALLOW, IO_PROHIBIT };
++static void stripe_prepare(struct stripe *stripe, unsigned p,
++ enum prepare_type type)
++{
++ struct page *page = PAGE(stripe, p);
++
++ switch (type) {
++ case IO_PROHIBIT:
++ /*
++ * In case we prohibit, we gotta make sure, that
++ * io on all other chunks than the one which failed
++ * or is being reconstructed is allowed and that it
++ * doesn't have state uptodate.
++ */
++ stripe_allow_io(stripe);
++ ClearPageUptodate(page);
++ ProhibitPageIO(page);
++
++ /* REMOVEME: statistics. */
++ atomic_inc(RS(stripe->sc)->stats + S_PROHIBITPAGEIO);
++ stripe->idx.recover = p;
++ SetStripeReconstruct(stripe);
++ break;
++
++ case IO_ALLOW:
++ AllowPageIO(page);
++ stripe->idx.recover = -1;
++ ClearStripeReconstruct(stripe);
++ break;
++
++ default:
++ BUG();
++ }
++}
++
++/*
++ * Degraded/reconstruction mode.
++ *
++ * Check stripe state to figure which chunks don't need IO.
++ */
++static INLINE void stripe_check_reconstruct(struct stripe *stripe,
++ int prohibited)
++{
++ struct raid_set *rs = RS(stripe->sc);
++
++ /*
++ * Degraded mode (device(s) failed) ->
++ * avoid io on the failed device.
++ */
++ if (unlikely(raid_set_degraded(rs))) {
++ /* REMOVEME: statistics. */
++ atomic_inc(rs->stats + S_DEGRADED);
++ stripe_prepare(stripe, rs->set.ei, IO_PROHIBIT);
++ return;
++ } else {
++ /*
++ * Reconstruction mode (ie. a particular device or
++ * some (rotating) parity chunk is being resynchronized) ->
++ * o make sure all needed pages are read in
++ * o writes are allowed to go through
++ */
++ int r = region_state(rs, stripe->key, DM_RH_NOSYNC);
++
++ if (r) {
++ /* REMOVEME: statistics. */
++ atomic_inc(rs->stats + S_NOSYNC);
++ stripe_prepare(stripe, dev_for_parity(stripe),
++ IO_PROHIBIT);
++ return;
++ }
++ }
++
++ /*
++ * All disks good. Avoid reading parity chunk and reconstruct it
++ * unless we have prohibited io to chunk(s).
++ */
++ if (!prohibited) {
++ if (StripeMerged(stripe))
++ stripe_prepare(stripe, stripe->idx.parity, IO_ALLOW);
++ else {
++ stripe_prepare(stripe, stripe->idx.parity, IO_PROHIBIT);
++
++ /*
++ * Overrule stripe_prepare to reconstruct the
++ * parity chunk, because it'll be created new anyway.
++ */
++ ClearStripeReconstruct(stripe);
++ }
++ }
++}
++
++/* Check, if stripe is ready to merge writes. */
++static INLINE int stripe_check_merge(struct stripe *stripe)
++{
++ struct raid_set *rs = RS(stripe->sc);
++ int prohibited = 0;
++ unsigned chunks = 0, p = rs->set.raid_devs;
++
++ /* Walk all chunks. */
++ while (p--) {
++ struct page *page = PAGE(stripe, p);
++
++ /* Can't merge active chunks. */
++ if (PageLocked(page)) {
++ /* REMOVEME: statistics. */
++ atomic_inc(rs->stats + S_MERGE_PAGE_LOCKED);
++ break;
++ }
++
++ /* Can merge uptodate chunks and have to count parity chunk. */
++ if (PageUptodate(page) || p == stripe->idx.parity) {
++ chunks++;
++ continue;
++ }
++
++ /* Read before write ordering. */
++ if (RSCheckOverwrite(rs) &&
++ bio_list_empty(BL(stripe, p, READ))) {
++ int r = stripe_check_overwrite(stripe, p);
++
++ if (r) {
++ chunks++;
++ /* REMOVEME: statistics. */
++ atomic_inc(RS(stripe->sc)->stats +
++ S_PROHIBITPAGEIO);
++ ProhibitPageIO(page);
++ prohibited = 1;
++ }
++ }
++ }
++
++ if (chunks == rs->set.raid_devs) {
++ /* All pages are uptodate or get written over or mixture. */
++ /* REMOVEME: statistics. */
++ atomic_inc(rs->stats + S_CAN_MERGE);
++ return 0;
++ } else
++ /* REMOVEME: statistics.*/
++ atomic_inc(rs->stats + S_CANT_MERGE);
++
++ return prohibited ? 1 : -EPERM;
++}
++
++/* Check, if stripe is ready to merge writes. */
++static INLINE int stripe_check_read(struct stripe *stripe)
++{
++ int r = 0;
++ unsigned p = RS(stripe->sc)->set.raid_devs;
++
++ /* Walk all chunks. */
++ while (p--) {
++ struct page *page = PAGE(stripe, p);
++
++ if (!PageLocked(page) &&
++ bio_list_empty(BL(stripe, p, READ))) {
++ ProhibitPageIO(page);
++ r = 1;
++ }
++ }
++
++ return r;
++}
++
++/*
++ * Read/write a stripe.
++ *
++ * All stripe read/write activity goes through this function.
++ *
++ * States to cover:
++ * o stripe to read and/or write
++ * o stripe with error to reconstruct
++ */
++static int stripe_rw(struct stripe *stripe)
++{
++ struct raid_set *rs = RS(stripe->sc);
++ int prohibited = 0, r;
++
++ /*
++ * Check the state of the RAID set and if degraded (or
++ * resynchronizing for reads), read in all other chunks but
++ * the one on the dead/resynchronizing device in order to be
++ * able to reconstruct the missing one.
++ *
++ * Merge all writes hanging off uptodate pages of the stripe.
++ */
++
++ /* Initially allow io on all chunks and prohibit below, if necessary. */
++ stripe_allow_io(stripe);
++
++ if (StripeRBW(stripe)) {
++ r = stripe_check_merge(stripe);
++ if (!r) {
++ /*
++ * If I could rely on valid parity (which would only
++ * be sure in case of a full synchronization),
++ * I could xor a fraction of chunks out of
++ * parity and back in.
++ *
++ * For the time being, I got to redo parity...
++ */
++ /* parity_xor(stripe); */ /* Xor chunks out. */
++ stripe_zero_chunk(stripe, stripe->idx.parity);
++ writes_merge(stripe); /* Merge writes in. */
++ parity_xor(stripe); /* Update parity. */
++ ClearStripeRBW(stripe); /* Disable RBW. */
++ SetStripeMerged(stripe); /* Writes merged. */
++ }
++
++ if (r > 0)
++ prohibited = 1;
++ } else if (!raid_set_degraded(rs))
++ /* Only allow for read avoidance if not degraded. */
++ prohibited = stripe_check_read(stripe);
++
++ /*
++ * Check, if io needs to be allowed/prohibeted on certain chunks
++ * because of a degraded set or reconstruction on a region.
++ */
++ stripe_check_reconstruct(stripe, prohibited);
++
++ /* Now submit any reads/writes. */
++ r = stripe_page_lists_rw(rs, stripe);
++ if (!r) {
++ /*
++ * No io submitted because of chunk io prohibited or
++ * locked pages -> push to end io list for processing.
++ */
++ atomic_inc(rs->stats + S_NO_RW); /* REMOVEME: statistics. */
++ stripe_endio_push(stripe);
++ wake_do_raid(rs); /* Wake myself. */
++ }
++
++ return 0;
++}
++
++/* Flush stripe either via flush list or imeediately. */
++enum flush_type { FLUSH_DELAY, FLUSH_NOW };
++static int stripe_flush(struct stripe *stripe, enum flush_type type)
++{
++ int r = 0;
++
++ stripe_lru_del(stripe, LIST_LOCKED);
++
++ /* Immediately flush. */
++ if (type == FLUSH_NOW) {
++ if (likely(raid_set_operational(RS(stripe->sc))))
++ r = stripe_rw(stripe); /* Read/write stripe. */
++ else
++ /* Optimization: Fail early on failed sets. */
++ stripe_fail_io(stripe);
++ /* Delay flush by putting it on io list for later processing. */
++ } else if (type == FLUSH_DELAY)
++ stripe_io_add(stripe, POS_TAIL, LIST_UNLOCKED);
++ else
++ BUG();
++
++ return r;
++}
++
++/*
++ * Queue reads and writes to a stripe by hanging
++ * their bios off the stripsets read/write lists.
++ *
++ * Endio reads on uptodate chunks.
++ */
++static INLINE int stripe_queue_bio(struct raid_set *rs, struct bio *bio,
++ struct bio_list *reject)
++{
++ int r = 0;
++ struct address addr;
++ struct stripe *stripe =
++ stripe_get(rs, raid_address(rs, bio->bi_sector, &addr));
++
++ if (stripe) {
++ int rr, rw = bio_data_dir(bio);
++
++ rr = stripe_lock(rs, stripe, rw, addr.key); /* Lock stripe */
++ if (rr) {
++ stripe_put(stripe);
++ goto out;
++ }
++
++ /* Distinguish read and write cases. */
++ bio_list_add(BL(stripe, addr.di, rw), bio);
++
++ /* REMOVEME: statistics */
++ atomic_inc(rs->stats + (rw == WRITE ?
++ S_BIOS_ADDED_WRITE : S_BIOS_ADDED_READ));
++
++ if (rw == READ)
++ SetStripeRead(stripe);
++ else {
++ SetStripeRBW(stripe);
++
++ /* Inrement pending write count on region. */
++ dm_rh_inc(rs->recover.rh, stripe->region);
++ r = 1; /* Region hash needs a flush. */
++ }
++
++ /*
++ * Optimize stripe flushing:
++ *
++ * o directly start io for read stripes.
++ *
++ * o put stripe onto stripe caches io_list for RBW,
++ * so that do_flush() can belabour it after we put
++ * more bios to the stripe for overwrite optimization.
++ */
++ stripe_flush(stripe,
++ StripeRead(stripe) ? FLUSH_NOW : FLUSH_DELAY);
++
++ /* Got no stripe from cache -> reject bio. */
++ } else {
++out:
++ bio_list_add(reject, bio);
++ /* REMOVEME: statistics. */
++ atomic_inc(rs->stats + S_IOS_POST);
++ }
++
++ return r;
++}
++
++/*
++ * Recovery functions
++ */
++/* Read a stripe off a raid set for recovery. */
++static int recover_read(struct raid_set *rs, struct stripe *stripe, int idx)
++{
++ /* Invalidate all pages so that they get read in. */
++ stripe_pages_invalidate(stripe);
++
++ /* Allow io on all recovery chunks. */
++ stripe_allow_io(stripe);
++
++ if (idx > -1)
++ ProhibitPageIO(PAGE(stripe, idx));
++
++ stripe->key = rs->recover.pos;
++ return stripe_page_lists_rw(rs, stripe);
++}
++
++/* Write a stripe to a raid set for recovery. */
++static int recover_write(struct raid_set *rs, struct stripe *stripe, int idx)
++{
++ /*
++ * If this is a reconstruct of a particular device, then
++ * reconstruct the respective page(s), else create parity page(s).
++ */
++ if (idx > -1) {
++ struct page *page = PAGE(stripe, idx);
++
++ AllowPageIO(page);
++ stripe_zero_chunk(stripe, idx);
++ common_xor(stripe, stripe->io.size, 0, idx);
++ page_set(page, DIRTY);
++ } else
++ parity_xor(stripe);
++
++ return stripe_page_lists_rw(rs, stripe);
++}
++
++/* Recover bandwidth available ?. */
++static int recover_bandwidth(struct raid_set *rs)
++{
++ int r, work;
++
++ /* On reset -> allow recovery. */
++ r = recover_io_reset(rs);
++ if (r || RSBandwidth(rs))
++ goto out;
++
++ work = atomic_read(rs->recover.io_count + IO_WORK);
++ if (work) {
++ /* Pay attention to larger recover stripe size. */
++ int recover =
++ atomic_read(rs->recover.io_count + IO_RECOVER) *
++ rs->recover.io_size /
++ rs->set.io_size;
++
++ /*
++ * Don't use more than given bandwidth of
++ * the work io for recovery.
++ */
++ if (recover > work / rs->recover.bandwidth_work) {
++ /* REMOVEME: statistics. */
++ atomic_inc(rs->stats + S_NO_BANDWIDTH);
++ return 0;
++ }
++ }
++
++out:
++ atomic_inc(rs->stats + S_BANDWIDTH); /* REMOVEME: statistics. */
++ return 1;
++}
++
++/* Try to get a region to recover. */
++static int recover_get_region(struct raid_set *rs)
++{
++ struct recover *rec = &rs->recover;
++ struct dm_rh_client *rh = rec->rh;
++
++ /* Start quiescing some regions. */
++ if (!RSRegionGet(rs)) {
++ int r = recover_bandwidth(rs); /* Enough bandwidth ?. */
++
++ if (r) {
++ r = dm_rh_recovery_prepare(rh);
++ if (r < 0) {
++ DMINFO("No %sregions to recover",
++ rec->nr_regions_to_recover ?
++ "more " : "");
++ return -ENOENT;
++ }
++ } else
++ return -EAGAIN;
++
++ SetRSRegionGet(rs);
++ }
++
++ if (!rec->reg) {
++ rec->reg = dm_rh_recovery_start(rh);
++ if (rec->reg) {
++ /*
++ * A reference for the the region I'll
++ * keep till I've completely synced it.
++ */
++ io_get(rs);
++ rec->pos = dm_rh_region_to_sector(rh,
++ dm_rh_get_region_key(rec->reg));
++ rec->end = rec->pos + dm_rh_get_region_size(rh);
++ return 1;
++ } else
++ return -EAGAIN;
++ }
++
++ return 0;
++}
++
++/* Read/write a recovery stripe. */
++static INLINE int recover_stripe_rw(struct raid_set *rs, struct stripe *stripe)
++{
++ /* Read/write flip-flop. */
++ if (TestClearStripeRBW(stripe)) {
++ SetStripeRead(stripe);
++ return recover_read(rs, stripe, idx_get(rs));
++ } else if (TestClearStripeRead(stripe))
++ return recover_write(rs, stripe, idx_get(rs));
++
++ return 0;
++}
++
++/* Reset recovery variables. */
++static void recovery_region_reset(struct raid_set *rs)
++{
++ rs->recover.reg = NULL;
++ ClearRSRegionGet(rs);
++}
++
++/* Update region hash state. */
++static void recover_rh_update(struct raid_set *rs, int error)
++{
++ struct recover *rec = &rs->recover;
++ struct dm_rh_client *rh = rec->rh;
++ struct dm_region *reg = rec->reg;
++
++ if (reg) {
++ dm_rh_recovery_end(rh, reg, error);
++ if (!error)
++ rec->nr_regions_recovered++;
++
++ recovery_region_reset(rs);
++ }
++
++ dm_rh_update_states(rh, 1);
++ dm_rh_flush(rh);
++ io_put(rs); /* Release the io reference for the region. */
++}
++
++/* Called by main io daemon to recover regions. */
++/* FIXME: cope with MAX_RECOVER > 1. */
++static INLINE void _do_recovery(struct raid_set *rs, struct stripe *stripe)
++{
++ int r;
++ struct recover *rec = &rs->recover;
++
++ /* If recovery is active -> return. */
++ if (StripeActive(stripe))
++ return;
++
++ /* io error is fatal for recovery -> stop it. */
++ if (unlikely(StripeError(stripe)))
++ goto err;
++
++ /* Get a region to recover. */
++ r = recover_get_region(rs);
++ switch (r) {
++ case 1: /* Got a new region. */
++ /* Flag read before write. */
++ ClearStripeRead(stripe);
++ SetStripeRBW(stripe);
++ break;
++
++ case 0:
++ /* Got a region in the works. */
++ r = recover_bandwidth(rs);
++ if (r) /* Got enough bandwidth. */
++ break;
++
++ case -EAGAIN:
++ /* No bandwidth/quiesced region yet, try later. */
++ wake_do_raid_delayed(rs, HZ / 10);
++ return;
++
++ case -ENOENT: /* No more regions. */
++ dm_table_event(rs->ti->table);
++ goto free;
++ }
++
++ /* Read/write a recover stripe. */
++ r = recover_stripe_rw(rs, stripe);
++ if (r) {
++ /* IO initiated, get another reference for the IO. */
++ io_get(rs);
++ return;
++ }
++
++ /* Update recovery position within region. */
++ rec->pos += stripe->io.size;
++
++ /* If we're at end of region, update region hash. */
++ if (rec->pos >= rec->end ||
++ rec->pos >= rs->set.sectors_per_dev)
++ recover_rh_update(rs, 0);
++ else
++ SetStripeRBW(stripe);
++
++ /* Schedule myself for another round... */
++ wake_do_raid(rs);
++ return;
++
++err:
++ raid_set_check_degrade(rs, stripe);
++
++ {
++ char buf[BDEVNAME_SIZE];
++
++ DMERR("stopping recovery due to "
++ "ERROR on /dev/%s, stripe at offset %llu",
++ bdevname(rs->dev[rs->set.ei].dev->bdev, buf),
++ (unsigned long long) stripe->key);
++
++ }
++
++ /* Make sure, that all quiesced regions get released. */
++ do {
++ if (rec->reg)
++ dm_rh_recovery_end(rec->rh, rec->reg, -EIO);
++
++ rec->reg = dm_rh_recovery_start(rec->rh);
++ } while (rec->reg);
++
++ recover_rh_update(rs, -EIO);
++free:
++ rs->set.dev_to_init = -1;
++
++ /* Check for jiffies overrun. */
++ rs->recover.end_jiffies = jiffies;
++ if (rs->recover.end_jiffies < rs->recover.start_jiffies)
++ rs->recover.end_jiffies = ~0;
++
++ ClearRSRecover(rs);
++}
++
++static INLINE void do_recovery(struct raid_set *rs)
++{
++ struct stripe *stripe;
++
++ list_for_each_entry(stripe, &rs->recover.stripes, lists[LIST_RECOVER])
++ _do_recovery(rs, stripe);
++
++ if (!RSRecover(rs))
++ stripe_recover_free(rs);
++}
++
++/*
++ * END recovery functions
++ */
++
++/* End io process all stripes handed in by endio() callback. */
++static void do_endios(struct raid_set *rs)
++{
++ struct stripe_cache *sc = &rs->sc;
++ struct stripe *stripe;
++
++ while ((stripe = stripe_endio_pop(sc))) {
++ unsigned count;
++
++ /* Recovery stripe special case. */
++ if (unlikely(StripeRecover(stripe))) {
++ if (stripe_io(stripe))
++ continue;
++
++ io_put(rs); /* Release region io reference. */
++ ClearStripeActive(stripe);
++
++ /* REMOVEME: statistics*/
++ atomic_dec(&sc->active_stripes);
++ continue;
++ }
++
++ /* Early end io all reads on any uptodate chunks. */
++ stripe_endio(READ, stripe, (count = 0, &count));
++ if (stripe_io(stripe)) {
++ if (count) /* REMOVEME: statistics. */
++ atomic_inc(rs->stats + S_ACTIVE_READS);
++
++ continue;
++ }
++
++ /* Set stripe inactive after all io got processed. */
++ if (TestClearStripeActive(stripe))
++ atomic_dec(&sc->active_stripes);
++
++ /* Unlock stripe (for clustering). */
++ stripe_unlock(rs, stripe);
++
++ /*
++ * If an io error on a stripe occured and the RAID set
++ * is still operational, requeue the stripe for io.
++ */
++ if (TestClearStripeError(stripe)) {
++ raid_set_check_degrade(rs, stripe);
++ ClearStripeReconstruct(stripe);
++
++ if (!StripeMerged(stripe) &&
++ raid_set_operational(rs)) {
++ stripe_pages_invalidate(stripe);
++ stripe_flush(stripe, FLUSH_DELAY);
++ /* REMOVEME: statistics. */
++ atomic_inc(rs->stats + S_REQUEUE);
++ continue;
++ }
++ }
++
++ /* Check if the RAID set is inoperational to error ios. */
++ if (!raid_set_operational(rs)) {
++ ClearStripeReconstruct(stripe);
++ stripe_fail_io(stripe);
++ BUG_ON(atomic_read(&stripe->cnt));
++ continue;
++ }
++
++ /* Got to reconstruct a missing chunk. */
++ if (TestClearStripeReconstruct(stripe))
++ reconstruct_xor(stripe);
++
++ /*
++ * Now that we've got a complete stripe, we can
++ * process the rest of the end ios on reads.
++ */
++ BUG_ON(stripe_endio(READ, stripe, NULL));
++ ClearStripeRead(stripe);
++
++ /*
++ * Read-before-write stripes need to be flushed again in
++ * order to work the write data into the pages *after*
++ * they were read in.
++ */
++ if (TestClearStripeMerged(stripe))
++ /* End io all bios which got merged already. */
++ BUG_ON(stripe_endio(WRITE_MERGED, stripe, NULL));
++
++ /* Got to put on flush list because of new writes. */
++ if (StripeRBW(stripe))
++ stripe_flush(stripe, FLUSH_DELAY);
++ }
++}
++
++/*
++ * Stripe cache shrinking.
++ */
++static INLINE void do_sc_shrink(struct raid_set *rs)
++{
++ unsigned shrink = atomic_read(&rs->sc.stripes_to_shrink);
++
++ if (shrink) {
++ unsigned cur = atomic_read(&rs->sc.stripes);
++
++ sc_shrink(&rs->sc, shrink);
++ shrink -= cur - atomic_read(&rs->sc.stripes);
++ atomic_set(&rs->sc.stripes_to_shrink, shrink);
++
++ /*
++ * Wake myself up in case we failed to shrink the
++ * requested amount in order to try again later.
++ */
++ if (shrink)
++ wake_do_raid(rs);
++ }
++}
++
++
++/*
++ * Process all ios
++ *
++ * We do different things with the io depending on the
++ * state of the region that it's in:
++ *
++ * o reads: hang off stripe cache or postpone if full
++ *
++ * o writes:
++ *
++ * CLEAN/DIRTY/NOSYNC: increment pending and hang io off stripe's stripe set.
++ * In case stripe cache is full or busy, postpone the io.
++ *
++ * RECOVERING: delay the io until recovery of the region completes.
++ *
++ */
++static INLINE void do_ios(struct raid_set *rs, struct bio_list *ios)
++{
++ int r;
++ unsigned flush = 0;
++ struct dm_rh_client *rh = rs->recover.rh;
++ struct bio *bio;
++ struct bio_list delay, reject;
++
++ bio_list_init(&delay);
++ bio_list_init(&reject);
++
++ /*
++ * Classify each io:
++ * o delay to recovering regions
++ * o queue to all other regions
++ */
++ while ((bio = bio_list_pop(ios))) {
++ /*
++ * In case we get a barrier bio, push it back onto
++ * the input queue unless all work queues are empty
++ * and the stripe cache is inactive.
++ */
++ if (unlikely(bio_barrier(bio))) {
++ /* REMOVEME: statistics. */
++ atomic_inc(rs->stats + S_BARRIER);
++ if (!list_empty(rs->sc.lists + LIST_IO) ||
++ !bio_list_empty(&delay) ||
++ !bio_list_empty(&reject) ||
++ sc_active(&rs->sc)) {
++ bio_list_push(ios, bio);
++ break;
++ }
++ }
++
++ r = region_state(rs, _sector(rs, bio), DM_RH_RECOVERING);
++ if (unlikely(r)) {
++ /* Got to wait for recovering regions. */
++ bio_list_add(&delay, bio);
++ SetRSBandwidth(rs);
++ } else {
++ /*
++ * Process ios to non-recovering regions by queueing
++ * them to stripes (does rh_inc()) for writes).
++ */
++ flush += stripe_queue_bio(rs, bio, &reject);
++ }
++ }
++
++ if (flush) {
++ r = dm_rh_flush(rh); /* Writes got queued -> flush dirty log. */
++ if (r)
++ DMERR("dirty log flush");
++ }
++
++ /* Delay ios to regions which are recovering. */
++ while ((bio = bio_list_pop(&delay))) {
++ /* REMOVEME: statistics.*/
++ atomic_inc(rs->stats + S_DELAYED_BIOS);
++ atomic_inc(rs->stats + S_SUM_DELAYED_BIOS);
++ dm_rh_delay_by_region(rh, bio,
++ dm_rh_sector_to_region(rh, _sector(rs, bio)));
++
++ }
++
++ /* Merge any rejected bios back to the head of the input list. */
++ bio_list_merge_head(ios, &reject);
++}
++
++/* Flush any stripes on the io list. */
++static INLINE void do_flush(struct raid_set *rs)
++{
++ struct list_head *list = rs->sc.lists + LIST_IO, *pos, *tmp;
++
++ list_for_each_safe(pos, tmp, list) {
++ int r = stripe_flush(list_entry(pos, struct stripe,
++ lists[LIST_IO]), FLUSH_NOW);
++
++ /* Remove from the list only if the stripe got processed. */
++ if (!r)
++ list_del_init(pos);
++ }
++}
++
++/* Send an event in case we're getting too busy. */
++static INLINE void do_busy_event(struct raid_set *rs)
++{
++ if ((sc_active(&rs->sc) > atomic_read(&rs->sc.stripes) * 4 / 5)) {
++ if (!TestSetRSScBusy(rs))
++ dm_table_event(rs->ti->table);
++ } else
++ ClearRSScBusy(rs);
++}
++
++/* Unplug: let the io role on the sets devices. */
++static INLINE void do_unplug(struct raid_set *rs)
++{
++ struct raid_dev *dev = rs->dev + rs->set.raid_devs;
++
++ while (dev-- > rs->dev) {
++ /* Only call any device unplug function, if io got queued. */
++ if (io_dev_clear(dev))
++ blk_unplug(bdev_get_queue(dev->dev->bdev));
++ }
++}
++
++/*-----------------------------------------------------------------
++ * RAID daemon
++ *---------------------------------------------------------------*/
++/*
++ * o belabour all end ios
++ * o optionally shrink the stripe cache
++ * o update the region hash states
++ * o optionally do recovery
++ * o grab the input queue
++ * o work an all requeued or new ios and perform stripe cache flushs
++ * unless the RAID set is inoperational (when we error ios)
++ * o check, if the stripe cache gets too busy and throw an event if so
++ * o unplug any component raid devices with queued bios
++ */
++static void do_raid(struct work_struct *ws)
++{
++ struct raid_set *rs = container_of(ws, struct raid_set, io.dws.work);
++ struct bio_list *ios = &rs->io.work, *ios_in = &rs->io.in;
++ spinlock_t *lock = &rs->io.in_lock;
++
++ /*
++ * We always need to end io, so that ios
++ * can get errored in case the set failed
++ * and the region counters get decremented
++ * before we update the region hash states.
++ */
++redo:
++ do_endios(rs);
++
++ /*
++ * Now that we've end io'd, which may have put stripes on
++ * the LRU list, we shrink the stripe cache if requested.
++ */
++ do_sc_shrink(rs);
++
++ /* Update region hash states before we go any further. */
++ dm_rh_update_states(rs->recover.rh, 1);
++
++ /* Try to recover regions. */
++ if (RSRecover(rs))
++ do_recovery(rs);
++
++ /* More endios -> process. */
++ if (!stripe_endio_empty(&rs->sc)) {
++ atomic_inc(rs->stats + S_REDO);
++ goto redo;
++ }
++
++ /* Quickly grab all new ios queued and add them to the work list. */
++ spin_lock_irq(lock);
++ bio_list_merge(ios, ios_in);
++ bio_list_init(ios_in);
++ spin_unlock_irq(lock);
++
++ /* Let's assume we're operational most of the time ;-). */
++ if (likely(raid_set_operational(rs))) {
++ /* If we got ios, work them into the cache. */
++ if (!bio_list_empty(ios)) {
++ do_ios(rs, ios);
++ do_unplug(rs); /* Unplug the sets device queues. */
++ }
++
++ do_flush(rs); /* Flush any stripes on io list. */
++ do_unplug(rs); /* Unplug the sets device queues. */
++ do_busy_event(rs); /* Check if we got too busy. */
++
++ /* More endios -> process. */
++ if (!stripe_endio_empty(&rs->sc)) {
++ atomic_inc(rs->stats + S_REDO);
++ goto redo;
++ }
++ } else
++ /* No way to reconstruct data with too many devices failed. */
++ bio_list_fail(rs, NULL, ios);
++}
++
++/*
++ * Callback for region hash to dispatch
++ * delayed bios queued to recovered regions
++ * (Gets called via rh_update_states()).
++ */
++static void dispatch_delayed_bios(void *context, struct bio_list *bl, int dummy)
++{
++ struct raid_set *rs = context;
++ struct bio *bio;
++
++ /* REMOVEME: decrement pending delayed bios counter. */
++ bio_list_for_each(bio, bl)
++ atomic_dec(rs->stats + S_DELAYED_BIOS);
++
++ /* Merge region hash private list to work list. */
++ bio_list_merge_head(&rs->io.work, bl);
++ bio_list_init(bl);
++ ClearRSBandwidth(rs);
++}
++
++/*************************************************************
++ * Constructor helpers
++ *************************************************************/
++/* Calculate MB/sec. */
++static INLINE unsigned mbpers(struct raid_set *rs, unsigned speed)
++{
++ return to_bytes(speed * rs->set.data_devs *
++ rs->recover.io_size * HZ >> 10) >> 10;
++}
++
++/*
++ * Discover fastest xor algorithm and # of chunks combination.
++ */
++/* Calculate speed for algorithm and # of chunks. */
++static INLINE unsigned xor_speed(struct stripe *stripe)
++{
++ unsigned r = 0;
++ unsigned long j;
++
++ /* Wait for next tick. */
++ for (j = jiffies; j == jiffies;)
++ ;
++
++ /* Do xors for a full tick. */
++ for (j = jiffies; j == jiffies;) {
++ mb();
++ common_xor(stripe, stripe->io.size, 0, 0);
++ mb();
++ r++;
++ mb();
++ }
++
++ return r;
++}
++
++/* Optimize xor algorithm for this RAID set. */
++static unsigned xor_optimize(struct raid_set *rs)
++{
++ unsigned chunks_max = 2, speed_max = 0;
++ struct xor_func *f = ARRAY_END(xor_funcs), *f_max = NULL;
++ struct stripe *stripe;
++
++ BUG_ON(list_empty(&rs->recover.stripes));
++ stripe = list_first_entry(&rs->recover.stripes, struct stripe,
++ lists[LIST_RECOVER]);
++
++ /*
++ * Got to allow io on all chunks, so that
++ * xor() will actually work on them.
++ */
++ stripe_allow_io(stripe);
++
++ /* Try all xor functions. */
++ while (f-- > xor_funcs) {
++ unsigned speed;
++
++ /* Set actual xor function for common_xor(). */
++ rs->xor.f = f;
++ rs->xor.chunks = XOR_CHUNKS_MAX + 1;
++
++ while (rs->xor.chunks-- > 2) {
++ speed = xor_speed(stripe);
++ if (speed > speed_max) {
++ speed_max = speed;
++ chunks_max = rs->xor.chunks;
++ f_max = f;
++ }
++ }
++ }
++
++ /* Memorize optimum parameters. */
++ rs->xor.f = f_max;
++ rs->xor.chunks = chunks_max;
++ return speed_max;
++}
++
++/*
++ * Allocate a RAID context (a RAID set)
++ */
++static int
++context_alloc(struct raid_set **raid_set, struct raid_type *raid_type,
++ unsigned stripes, unsigned chunk_size, unsigned io_size,
++ unsigned recover_io_size, unsigned raid_devs,
++ sector_t sectors_per_dev,
++ struct dm_target *ti, unsigned dl_parms, char **argv)
++{
++ int r;
++ unsigned p;
++ size_t len;
++ sector_t region_size, ti_len;
++ struct raid_set *rs = NULL;
++ struct dm_dirty_log *dl;
++ struct recover *rec;
++
++ /*
++ * Create the dirty log
++ *
++ * We need to change length for the dirty log constructor,
++ * because we want an amount of regions for all stripes derived
++ * from the single device size, so that we can keep region
++ * size = 2^^n independant of the number of devices
++ */
++ ti_len = ti->len;
++ ti->len = sectors_per_dev;
++ dl = dm_dirty_log_create(argv[0], ti, dl_parms, argv + 2);
++ ti->len = ti_len;
++ if (!dl)
++ goto bad_dirty_log;
++
++ /* Chunk size *must* be smaller than region size. */
++ region_size = dl->type->get_region_size(dl);
++ if (chunk_size > region_size)
++ goto bad_chunk_size;
++
++ /* Recover io size *must* be smaller than region size as well. */
++ if (recover_io_size > region_size)
++ goto bad_recover_io_size;
++
++ /* Size and allocate the RAID set structure. */
++ len = sizeof(*rs->data) + sizeof(*rs->dev);
++ if (array_too_big(sizeof(*rs), len, raid_devs))
++ goto bad_array;
++
++ len = sizeof(*rs) + raid_devs * len;
++ rs = kzalloc(len, GFP_KERNEL);
++ if (!rs)
++ goto bad_alloc;
++
++ rec = &rs->recover;
++ atomic_set(&rs->io.in_process, 0);
++ atomic_set(&rs->io.in_process_max, 0);
++ rec->io_size = recover_io_size;
++
++ /* Pointer to data array. */
++ rs->data = (unsigned long **)
++ ((void *) rs->dev + raid_devs * sizeof(*rs->dev));
++ rec->dl = dl;
++ rs->set.raid_devs = p = raid_devs;
++ rs->set.data_devs = raid_devs - raid_type->parity_devs;
++ rs->set.raid_type = raid_type;
++
++ /*
++ * Set chunk and io size and respective shifts
++ * (used to avoid divisions)
++ */
++ rs->set.chunk_size = chunk_size;
++ rs->set.chunk_mask = chunk_size - 1;
++ rs->set.chunk_shift = ffs(chunk_size) - 1;
++
++ rs->set.io_size = io_size;
++ rs->set.io_mask = io_size - 1;
++ rs->set.io_shift = ffs(io_size) - 1;
++ rs->set.io_shift_mask = rs->set.chunk_mask & ~rs->set.io_mask;
++
++ rs->set.pages_per_io = chunk_pages(io_size);
++ rs->set.sectors_per_dev = sectors_per_dev;
++
++ rs->set.ei = -1; /* Indicate no failed device. */
++ atomic_set(&rs->set.failed_devs, 0);
++
++ rs->ti = ti;
++
++ atomic_set(rec->io_count + IO_WORK, 0);
++ atomic_set(rec->io_count + IO_RECOVER, 0);
++
++ /* Initialize io lock and queues. */
++ spin_lock_init(&rs->io.in_lock);
++ bio_list_init(&rs->io.in);
++ bio_list_init(&rs->io.work);
++
++ init_waitqueue_head(&rs->io.suspendq); /* Suspend waiters (dm-io). */
++
++ rec->nr_regions = dm_sector_div_up(sectors_per_dev, region_size);
++ rec->rh = dm_rh_client_create(MAX_RECOVER, dispatch_delayed_bios, rs,
++ wake_do_raid, rs, dl, region_size,
++ rs->recover.nr_regions);
++ if (IS_ERR(rec->rh))
++ goto bad_rh;
++
++ /* Initialize stripe cache. */
++ r = sc_init(rs, stripes);
++ if (r)
++ goto bad_sc;
++
++ /* Create dm-io client context. */
++ rs->sc.dm_io_client = dm_io_client_create(rs->set.raid_devs *
++ rs->set.pages_per_io);
++ if (IS_ERR(rs->sc.dm_io_client))
++ goto bad_dm_io_client;
++
++ /* REMOVEME: statistics. */
++ stats_reset(rs);
++ ClearRSDevelStats(rs); /* Disnable development status. */
++
++ *raid_set = rs;
++ return 0;
++
++bad_dirty_log:
++ TI_ERR_RET("Error creating dirty log", -ENOMEM);
++
++
++bad_chunk_size:
++ dm_dirty_log_destroy(dl);
++ TI_ERR("Chunk size larger than region size");
++
++bad_recover_io_size:
++ dm_dirty_log_destroy(dl);
++ TI_ERR("Recover stripe io size larger than region size");
++
++bad_array:
++ dm_dirty_log_destroy(dl);
++ TI_ERR("Arry too big");
++
++bad_alloc:
++ dm_dirty_log_destroy(dl);
++ TI_ERR_RET("Cannot allocate raid context", -ENOMEM);
++
++bad_rh:
++ dm_dirty_log_destroy(dl);
++ ti->error = DM_MSG_PREFIX "Error creating dirty region hash";
++ goto free_rs;
++
++bad_sc:
++ ti->error = DM_MSG_PREFIX "Error creating stripe cache";
++ goto free;
++
++bad_dm_io_client:
++ ti->error = DM_MSG_PREFIX "Error allocating dm-io resources";
++free:
++ dm_rh_client_destroy(rec->rh);
++ sc_exit(&rs->sc);
++ dm_rh_client_destroy(rec->rh); /* Destroys dirty log as well. */
++free_rs:
++ kfree(rs);
++ return -ENOMEM;
++}
++
++/* Free a RAID context (a RAID set). */
++static void
++context_free(struct raid_set *rs, struct dm_target *ti, unsigned r)
++{
++ while (r--)
++ dm_put_device(ti, rs->dev[r].dev);
++
++ dm_io_client_destroy(rs->sc.dm_io_client);
++ sc_exit(&rs->sc);
++ dm_rh_client_destroy(rs->recover.rh);
++ dm_dirty_log_destroy(rs->recover.dl);
++ kfree(rs);
++}
++
++/* Create work queue and initialize work. */
++static int rs_workqueue_init(struct raid_set *rs)
++{
++ struct dm_target *ti = rs->ti;
++
++ rs->io.wq = create_singlethread_workqueue(DAEMON);
++ if (!rs->io.wq)
++ TI_ERR_RET("failed to create " DAEMON, -ENOMEM);
++
++ INIT_DELAYED_WORK(&rs->io.dws, do_raid);
++ return 0;
++}
++
++/* Return pointer to raid_type structure for raid name. */
++static struct raid_type *get_raid_type(char *name)
++{
++ struct raid_type *r = ARRAY_END(raid_types);
++
++ while (r-- > raid_types) {
++ if (!strnicmp(STR_LEN(r->name, name)))
++ return r;
++ }
++
++ return NULL;
++}
++
++/* FIXME: factor out to dm core. */
++static int multiple(sector_t a, sector_t b, sector_t *n)
++{
++ sector_t r = a;
++
++ sector_div(r, b);
++ *n = r;
++ return a == r * b;
++}
++
++/* Log RAID set information to kernel log. */
++static void raid_set_log(struct raid_set *rs, unsigned speed)
++{
++ unsigned p;
++ char buf[BDEVNAME_SIZE];
++
++ for (p = 0; p < rs->set.raid_devs; p++)
++ DMINFO("/dev/%s is raid disk %u",
++ bdevname(rs->dev[p].dev->bdev, buf), p);
++
++ DMINFO("%d/%d/%d sectors chunk/io/recovery size, %u stripes",
++ rs->set.chunk_size, rs->set.io_size, rs->recover.io_size,
++ atomic_read(&rs->sc.stripes));
++ DMINFO("algorithm \"%s\", %u chunks with %uMB/s", rs->xor.f->name,
++ rs->xor.chunks, mbpers(rs, speed));
++ DMINFO("%s set with net %u/%u devices", rs->set.raid_type->descr,
++ rs->set.data_devs, rs->set.raid_devs);
++}
++
++/* Get all devices and offsets. */
++static int
++dev_parms(struct dm_target *ti, struct raid_set *rs,
++ char **argv, int *p)
++{
++ for (*p = 0; *p < rs->set.raid_devs; (*p)++, argv += 2) {
++ int r;
++ unsigned long long tmp;
++ struct raid_dev *dev = rs->dev + *p;
++ union dev_lookup dl = {.dev = dev };
++
++ /* Get offset and device. */
++ r = sscanf(argv[1], "%llu", &tmp);
++ if (r != 1)
++ TI_ERR("Invalid RAID device offset parameter");
++
++ dev->start = tmp;
++ r = dm_get_device(ti, argv[0], dev->start,
++ rs->set.sectors_per_dev,
++ dm_table_get_mode(ti->table), &dev->dev);
++ if (r)
++ TI_ERR_RET("RAID device lookup failure", r);
++
++ r = raid_dev_lookup(rs, bynumber, &dl);
++ if (r != -ENODEV && r < *p) {
++ (*p)++; /* Ensure dm_put_device() on actual device. */
++ TI_ERR_RET("Duplicate RAID device", -ENXIO);
++ }
++ }
++
++ return 0;
++}
++
++/* Set recovery bandwidth. */
++static INLINE void
++recover_set_bandwidth(struct raid_set *rs, unsigned bandwidth)
++{
++ rs->recover.bandwidth = bandwidth;
++ rs->recover.bandwidth_work = 100 / bandwidth;
++}
++
++/* Handle variable number of RAID parameters. */
++static int
++raid_variable_parms(struct dm_target *ti, char **argv,
++ unsigned i, int *raid_parms,
++ int *chunk_size, int *chunk_size_parm,
++ int *stripes, int *stripes_parm,
++ int *io_size, int *io_size_parm,
++ int *recover_io_size, int *recover_io_size_parm,
++ int *bandwidth, int *bandwidth_parm)
++{
++ /* Fetch # of variable raid parameters. */
++ if (sscanf(argv[i++], "%d", raid_parms) != 1 ||
++ !range_ok(*raid_parms, 0, 5))
++ TI_ERR("Bad variable raid parameters number");
++
++ if (*raid_parms) {
++ /*
++ * If we've got variable RAID parameters,
++ * chunk size is the first one
++ */
++ if (sscanf(argv[i++], "%d", chunk_size) != 1 ||
++ (*chunk_size != -1 &&
++ (!POWER_OF_2(*chunk_size) ||
++ !range_ok(*chunk_size, IO_SIZE_MIN, CHUNK_SIZE_MAX))))
++ TI_ERR("Invalid chunk size; must be 2^^n and <= 16384");
++
++ *chunk_size_parm = *chunk_size;
++ if (*chunk_size == -1)
++ *chunk_size = CHUNK_SIZE;
++
++ /*
++ * In case we've got 2 or more variable raid
++ * parameters, the number of stripes is the second one
++ */
++ if (*raid_parms > 1) {
++ if (sscanf(argv[i++], "%d", stripes) != 1 ||
++ (*stripes != -1 &&
++ !range_ok(*stripes, STRIPES_MIN,
++ STRIPES_MAX)))
++ TI_ERR("Invalid number of stripes: must "
++ "be >= 8 and <= 8192");
++ }
++
++ *stripes_parm = *stripes;
++ if (*stripes == -1)
++ *stripes = STRIPES;
++
++ /*
++ * In case we've got 3 or more variable raid
++ * parameters, the io size is the third one.
++ */
++ if (*raid_parms > 2) {
++ if (sscanf(argv[i++], "%d", io_size) != 1 ||
++ (*io_size != -1 &&
++ (!POWER_OF_2(*io_size) ||
++ !range_ok(*io_size, IO_SIZE_MIN,
++ min(BIO_MAX_SECTORS / 2,
++ *chunk_size)))))
++ TI_ERR("Invalid io size; must "
++ "be 2^^n and less equal "
++ "min(BIO_MAX_SECTORS/2, chunk size)");
++ } else
++ *io_size = *chunk_size;
++
++ *io_size_parm = *io_size;
++ if (*io_size == -1)
++ *io_size = *chunk_size;
++
++ /*
++ * In case we've got 4 variable raid parameters,
++ * the recovery stripe io_size is the fourth one
++ */
++ if (*raid_parms > 3) {
++ if (sscanf(argv[i++], "%d", recover_io_size) != 1 ||
++ (*recover_io_size != -1 &&
++ (!POWER_OF_2(*recover_io_size) ||
++ !range_ok(*recover_io_size, RECOVER_IO_SIZE_MIN,
++ BIO_MAX_SECTORS / 2))))
++ TI_ERR("Invalid recovery io size; must be "
++ "2^^n and less equal BIO_MAX_SECTORS/2");
++ }
++
++ *recover_io_size_parm = *recover_io_size;
++ if (*recover_io_size == -1)
++ *recover_io_size = RECOVER_IO_SIZE;
++
++ /*
++ * In case we've got 5 variable raid parameters,
++ * the recovery io bandwidth is the fifth one
++ */
++ if (*raid_parms > 4) {
++ if (sscanf(argv[i++], "%d", bandwidth) != 1 ||
++ (*bandwidth != -1 &&
++ !range_ok(*bandwidth, BANDWIDTH_MIN,
++ BANDWIDTH_MAX)))
++ TI_ERR("Invalid recovery bandwidth "
++ "percentage; must be > 0 and <= 100");
++ }
++
++ *bandwidth_parm = *bandwidth;
++ if (*bandwidth == -1)
++ *bandwidth = BANDWIDTH;
++ }
++
++ return 0;
++}
++
++/* Parse optional locking parameters. */
++static int
++raid_locking_parms(struct dm_target *ti, char **argv,
++ unsigned i, int *locking_parms,
++ struct dm_raid45_locking_type **locking_type)
++{
++ *locking_parms = 0;
++ *locking_type = &locking_none;
++
++ if (!strnicmp(argv[i], "none", strlen(argv[i])))
++ *locking_parms = 1;
++ else if (!strnicmp(argv[i + 1], "locking", strlen(argv[i + 1]))) {
++ *locking_type = &locking_none;
++ *locking_parms = 2;
++ } else if (!strnicmp(argv[i + 1], "cluster", strlen(argv[i + 1]))) {
++ *locking_type = &locking_cluster;
++ /* FIXME: namespace. */
++ *locking_parms = 3;
++ }
++
++ return *locking_parms == 1 ? -EINVAL : 0;
++}
++
++/* Set backing device information properties of RAID set. */
++static void rs_set_bdi(struct raid_set *rs, unsigned stripes, unsigned chunks)
++{
++ unsigned p, ra_pages;
++ struct mapped_device *md = dm_table_get_md(rs->ti->table);
++ struct backing_dev_info *bdi = &dm_disk(md)->queue->backing_dev_info;
++
++ /* Set read-ahead for the RAID set and the component devices. */
++ bdi->ra_pages = stripes * stripe_pages(rs, rs->set.io_size);
++ ra_pages = chunks * chunk_pages(rs->set.io_size);
++ for (p = rs->set.raid_devs; p--; ) {
++ struct request_queue *q = bdev_get_queue(rs->dev[p].dev->bdev);
++
++ q->backing_dev_info.ra_pages = ra_pages;
++ }
++
++ /* Set congested function and data. */
++ bdi->congested_fn = raid_set_congested;
++ bdi->congested_data = rs;
++
++ dm_put(md);
++}
++
++/* Get backing device information properties of RAID set. */
++static void rs_get_ra(struct raid_set *rs, unsigned *stripes, unsigned *chunks)
++{
++ struct mapped_device *md = dm_table_get_md(rs->ti->table);
++
++ *stripes = dm_disk(md)->queue->backing_dev_info.ra_pages
++ / stripe_pages(rs, rs->set.io_size);
++ *chunks = bdev_get_queue(rs->dev->dev->bdev)->backing_dev_info.ra_pages
++ / chunk_pages(rs->set.io_size);
++
++ dm_put(md);
++}
++
++/*
++ * Construct a RAID4/5 mapping:
++ *
++ * log_type #log_params <log_params> \
++ * raid_type [#parity_dev] #raid_variable_params <raid_params> \
++ * [locking "none"/"cluster"]
++ * #raid_devs #dev_to_initialize [<dev_path> <offset>]{3,}
++ *
++ * log_type = "core"/"disk",
++ * #log_params = 1-3 (1-2 for core dirty log type, 3 for disk dirty log only)
++ * log_params = [dirty_log_path] region_size [[no]sync])
++ *
++ * raid_type = "raid4", "raid5_la", "raid5_ra", "raid5_ls", "raid5_rs"
++ *
++ * #parity_dev = N if raid_type = "raid4"
++ * o N = -1: pick default = last device
++ * o N >= 0 and < #raid_devs: parity device index
++ *
++ * #raid_variable_params = 0-5; raid_params (-1 = default):
++ * [chunk_size [#stripes [io_size [recover_io_size [%recovery_bandwidth]]]]]
++ * o chunk_size (unit to calculate drive addresses; must be 2^^n, > 8
++ * and <= CHUNK_SIZE_MAX)
++ * o #stripes is number of stripes allocated to stripe cache
++ * (must be > 1 and < STRIPES_MAX)
++ * o io_size (io unit size per device in sectors; must be 2^^n and > 8)
++ * o recover_io_size (io unit size per device for recovery in sectors;
++ must be 2^^n, > SECTORS_PER_PAGE and <= region_size)
++ * o %recovery_bandwith is the maximum amount spend for recovery during
++ * application io (1-100%)
++ * If raid_variable_params = 0, defaults will be used.
++ * Any raid_variable_param can be set to -1 to apply a default
++ *
++ * #raid_devs = N (N >= 3)
++ *
++ * #dev_to_initialize = N
++ * -1: initialize parity on all devices
++ * >= 0 and < #raid_devs: initialize raid_path; used to force reconstruction
++ * of a failed devices content after replacement
++ *
++ * <dev_path> = device_path (eg, /dev/sdd1)
++ * <offset> = begin at offset on <dev_path>
++ *
++ */
++#define MIN_PARMS 13
++static int raid_ctr(struct dm_target *ti, unsigned argc, char **argv)
++{
++ int bandwidth = BANDWIDTH, bandwidth_parm = -1,
++ chunk_size = CHUNK_SIZE, chunk_size_parm = -1,
++ dev_to_init, dl_parms, locking_parms, parity_parm, pi = -1,
++ i, io_size = IO_SIZE, io_size_parm = -1,
++ r, raid_devs, raid_parms,
++ recover_io_size = RECOVER_IO_SIZE, recover_io_size_parm = -1,
++ stripes = STRIPES, stripes_parm = -1;
++ unsigned speed;
++ sector_t tmp, sectors_per_dev;
++ struct dm_raid45_locking_type *locking;
++ struct raid_set *rs;
++ struct raid_type *raid_type;
++
++ /* Ensure minimum number of parameters. */
++ if (argc < MIN_PARMS)
++ TI_ERR("Not enough parameters");
++
++ /* Fetch # of dirty log parameters. */
++ if (sscanf(argv[1], "%d", &dl_parms) != 1
++ || !range_ok(dl_parms, 1, 4711))
++ TI_ERR("Bad dirty log parameters number");
++
++ /* Check raid_type. */
++ raid_type = get_raid_type(argv[dl_parms + 2]);
++ if (!raid_type)
++ TI_ERR("Bad raid type");
++
++ /* In case of RAID4, parity drive is selectable. */
++ parity_parm = !!(raid_type->level == raid4);
++
++ /* Handle variable number of RAID parameters. */
++ r = raid_variable_parms(ti, argv, dl_parms + parity_parm + 3,
++ &raid_parms,
++ &chunk_size, &chunk_size_parm,
++ &stripes, &stripes_parm,
++ &io_size, &io_size_parm,
++ &recover_io_size, &recover_io_size_parm,
++ &bandwidth, &bandwidth_parm);
++ if (r)
++ return r;
++
++ r = raid_locking_parms(ti, argv,
++ dl_parms + parity_parm + raid_parms + 4,
++ &locking_parms, &locking);
++ if (r)
++ return r;
++
++ /* # of raid devices. */
++ i = dl_parms + parity_parm + raid_parms + locking_parms + 4;
++ if (sscanf(argv[i], "%d", &raid_devs) != 1 ||
++ raid_devs < raid_type->minimal_devs)
++ TI_ERR("Invalid number of raid devices");
++
++ /* In case of RAID4, check parity drive index is in limits. */
++ if (raid_type->level == raid4) {
++ /* Fetch index of parity device. */
++ if (sscanf(argv[dl_parms + 3], "%d", &pi) != 1 ||
++ !range_ok(pi, 0, raid_devs - 1))
++ TI_ERR("Invalid RAID4 parity device index");
++ }
++
++ /*
++ * Index of device to initialize starts at 0
++ *
++ * o -1 -> don't initialize a particular device,
++ * o 0..raid_devs-1 -> initialize respective device
++ * (used for reconstruction of a replaced device)
++ */
++ if (sscanf
++ (argv[dl_parms + parity_parm + raid_parms + locking_parms + 5],
++ "%d", &dev_to_init) != 1
++ || !range_ok(dev_to_init, -1, raid_devs - 1))
++ TI_ERR("Invalid number for raid device to initialize");
++
++ /* Check # of raid device arguments. */
++ if (argc - dl_parms - parity_parm - raid_parms - 6 !=
++ 2 * raid_devs)
++ TI_ERR("Wrong number of raid device/offset arguments");
++
++ /*
++ * Check that the table length is devisable
++ * w/o rest by (raid_devs - parity_devs)
++ */
++ if (!multiple(ti->len, raid_devs - raid_type->parity_devs,
++ §ors_per_dev))
++ TI_ERR
++ ("Target length not divisable by number of data devices");
++
++ /*
++ * Check that the device size is
++ * devisable w/o rest by chunk size
++ */
++ if (!multiple(sectors_per_dev, chunk_size, &tmp))
++ TI_ERR("Device length not divisable by chunk_size");
++
++ /****************************************************************
++ * Now that we checked the constructor arguments ->
++ * let's allocate the RAID set
++ ****************************************************************/
++ r = context_alloc(&rs, raid_type, stripes, chunk_size, io_size,
++ recover_io_size, raid_devs, sectors_per_dev,
++ ti, dl_parms, argv);
++ if (r)
++ return r;
++
++ /*
++ * Set these here in order to avoid passing
++ * too many arguments to context_alloc()
++ */
++ rs->set.dev_to_init_parm = dev_to_init;
++ rs->set.dev_to_init = dev_to_init;
++ rs->set.pi_parm = pi;
++ rs->set.pi = (pi == -1) ? rs->set.data_devs : pi;
++ rs->set.raid_parms = raid_parms;
++ rs->set.chunk_size_parm = chunk_size_parm;
++ rs->set.io_size_parm = io_size_parm;
++ rs->sc.stripes_parm = stripes_parm;
++ rs->recover.io_size_parm = recover_io_size_parm;
++ rs->recover.bandwidth_parm = bandwidth_parm;
++ recover_set_bandwidth(rs, bandwidth);
++
++ /* Use locking type to lock stripe access. */
++ rs->locking = locking;
++
++ /* Get the device/offset tupels. */
++ argv += dl_parms + 6 + parity_parm + raid_parms;
++ r = dev_parms(ti, rs, argv, &i);
++ if (r)
++ goto err;
++
++ /* Initialize recovery. */
++ rs->recover.start_jiffies = jiffies;
++ rs->recover.end_jiffies = 0;
++ recovery_region_reset(rs);
++
++ /* Allow for recovery of any nosync regions. */
++ SetRSRecover(rs);
++
++ /* Set backing device information (eg. read ahead). */
++ rs_set_bdi(rs, chunk_size * 2, io_size * 4);
++ SetRSCheckOverwrite(rs); /* Allow chunk overwrite checks. */
++
++ speed = xor_optimize(rs); /* Select best xor algorithm. */
++
++ /* Initialize work queue to handle this RAID set's io. */
++ r = rs_workqueue_init(rs);
++ if (r)
++ goto err;
++
++ raid_set_log(rs, speed); /* Log information about RAID set. */
++
++ /*
++ * Make sure that dm core only hands maximum io size
++ * length down and pays attention to io boundaries.
++ */
++ ti->split_io = rs->set.io_size;
++ ti->private = rs;
++ return 0;
++
++err:
++ context_free(rs, ti, i);
++ return r;
++}
++
++/*
++ * Destruct a raid mapping
++ */
++static void raid_dtr(struct dm_target *ti)
++{
++ struct raid_set *rs = ti->private;
++
++ /* Indicate recovery end so that ios in flight drain. */
++ ClearRSRecover(rs);
++
++ wake_do_raid(rs); /* Wake daemon. */
++ wait_ios(rs); /* Wait for any io still being processed. */
++ destroy_workqueue(rs->io.wq);
++ context_free(rs, ti, rs->set.raid_devs);
++}
++
++/* Queues ios to RAID sets. */
++static inline void queue_bio(struct raid_set *rs, struct bio *bio)
++{
++ int wake;
++ struct bio_list *in = &rs->io.in;
++ spinlock_t *in_lock = &rs->io.in_lock;
++
++ spin_lock_irq(in_lock);
++ wake = bio_list_empty(in);
++ bio_list_add(in, bio);
++ spin_unlock_irq(in_lock);
++
++ /* Wake daemon if input list was empty. */
++ if (wake)
++ wake_do_raid(rs);
++}
++
++/* Raid mapping function. */
++static int raid_map(struct dm_target *ti, struct bio *bio,
++ union map_info *map_context)
++{
++ /* I don't want to waste stripe cache capacity. */
++ if (bio_rw(bio) == READA)
++ return -EIO;
++ else {
++ struct raid_set *rs = ti->private;
++
++ /* REMOVEME: statistics. */
++ atomic_inc(rs->stats +
++ (bio_data_dir(bio) == WRITE ?
++ S_BIOS_WRITE : S_BIOS_READ));
++
++ /*
++ * Get io reference to be waiting for to drop
++ * to zero on device suspension/destruction.
++ */
++ io_get(rs);
++ bio->bi_sector -= ti->begin; /* Remap sector. */
++ queue_bio(rs, bio); /* Queue to the daemon. */
++ return DM_MAPIO_SUBMITTED; /* Handle later. */
++ }
++}
++
++/* Device suspend. */
++static void raid_postsuspend(struct dm_target *ti)
++{
++ struct raid_set *rs = ti->private;
++ struct dm_dirty_log *dl = rs->recover.dl;
++
++ SetRSSuspended(rs);
++
++ if (RSRecover(rs))
++ dm_rh_stop_recovery(rs->recover.rh); /* Wakes do_raid(). */
++ else
++ wake_do_raid(rs);
++
++ wait_ios(rs); /* Wait for completion of all ios being processed. */
++ if (dl->type->postsuspend && dl->type->postsuspend(dl))
++ /* Suspend dirty log. */
++ /* FIXME: need better error handling. */
++ DMWARN("log suspend failed");
++}
++
++/* Device resume. */
++static void raid_resume(struct dm_target *ti)
++{
++ struct raid_set *rs = ti->private;
++ struct recover *rec = &rs->recover;
++ struct dm_dirty_log *dl = rec->dl;
++
++ if (dl->type->resume && dl->type->resume(dl))
++ /* Resume dirty log. */
++ /* FIXME: need better error handling. */
++ DMWARN("log resume failed");
++
++ rec->nr_regions_to_recover =
++ rec->nr_regions - dl->type->get_sync_count(dl);
++
++ ClearRSSuspended(rs);
++
++ /* Reset any unfinished recovery. */
++ if (RSRecover(rs)) {
++ recovery_region_reset(rs);
++ dm_rh_start_recovery(rec->rh);/* Calls wake_do_raid(). */
++ } else
++ wake_do_raid(rs);
++}
++
++static INLINE unsigned sc_size(struct raid_set *rs)
++{
++ return to_sector(atomic_read(&rs->sc.stripes) *
++ (sizeof(struct stripe) +
++ (sizeof(struct stripe_set) +
++ (sizeof(struct page_list) +
++ to_bytes(rs->set.io_size) *
++ rs->set.raid_devs)) +
++ (rs->recover.
++ end_jiffies ? 0 : to_bytes(rs->set.raid_devs *
++ rs->recover.
++ io_size))));
++}
++
++/* REMOVEME: status output for development. */
++static void
++raid_devel_stats(struct dm_target *ti, char *result,
++ unsigned *size, unsigned maxlen)
++{
++ unsigned chunks, stripes, sz = *size;
++ unsigned long j;
++ char buf[BDEVNAME_SIZE], *p;
++ struct stats_map *sm, *sm_end = ARRAY_END(stats_map);
++ struct raid_set *rs = ti->private;
++ struct recover *rec = &rs->recover;
++ struct timespec ts;
++
++ DMEMIT("%s ", version);
++ DMEMIT("io_inprocess=%d ", atomic_read(&rs->io.in_process));
++ DMEMIT("io_inprocess_max=%d ", atomic_read(&rs->io.in_process_max));
++
++ for (sm = stats_map; sm < sm_end; sm++)
++ DMEMIT("%s%d", sm->str, atomic_read(rs->stats + sm->type));
++
++ DMEMIT(" overwrite=%s ", RSCheckOverwrite(rs) ? "on" : "off");
++ DMEMIT("sc=%u/%u/%u/%u/%u ", rs->set.chunk_size, rs->set.io_size,
++ atomic_read(&rs->sc.stripes), rs->sc.hash.buckets,
++ sc_size(rs));
++
++ j = (rec->end_jiffies ? rec->end_jiffies : jiffies) -
++ rec->start_jiffies;
++ jiffies_to_timespec(j, &ts);
++ sprintf(buf, "%ld.%ld", ts.tv_sec, ts.tv_nsec);
++ p = strchr(buf, '.');
++ p[3] = 0;
++
++ DMEMIT("rg=%llu%s/%llu/%llu/%u %s ",
++ (unsigned long long) rec->nr_regions_recovered,
++ RSRegionGet(rs) ? "+" : "",
++ (unsigned long long) rec->nr_regions_to_recover,
++ (unsigned long long) rec->nr_regions, rec->bandwidth, buf);
++
++ rs_get_ra(rs, &stripes, &chunks);
++ DMEMIT("ra=%u/%u ", stripes, chunks);
++
++ *size = sz;
++}
++
++static int
++raid_status(struct dm_target *ti, status_type_t type,
++ char *result, unsigned maxlen)
++{
++ unsigned i, sz = 0;
++ char buf[BDEVNAME_SIZE];
++ struct raid_set *rs = ti->private;
++
++ switch (type) {
++ case STATUSTYPE_INFO:
++ /* REMOVEME: statistics. */
++ if (RSDevelStats(rs))
++ raid_devel_stats(ti, result, &sz, maxlen);
++
++ DMEMIT("%u ", rs->set.raid_devs);
++
++ for (i = 0; i < rs->set.raid_devs; i++)
++ DMEMIT("%s ",
++ format_dev_t(buf, rs->dev[i].dev->bdev->bd_dev));
++
++ DMEMIT("1 ");
++ for (i = 0; i < rs->set.raid_devs; i++) {
++ DMEMIT("%c", dev_operational(rs, i) ? 'A' : 'D');
++
++ if (rs->set.raid_type->level == raid4 &&
++ i == rs->set.pi)
++ DMEMIT("p");
++
++ if (rs->set.dev_to_init == i)
++ DMEMIT("i");
++ }
++
++ break;
++
++ case STATUSTYPE_TABLE:
++ sz = rs->recover.dl->type->status(rs->recover.dl, type,
++ result, maxlen);
++ DMEMIT("%s %u ", rs->set.raid_type->name,
++ rs->set.raid_parms);
++
++ if (rs->set.raid_type->level == raid4)
++ DMEMIT("%d ", rs->set.pi_parm);
++
++ if (rs->set.raid_parms)
++ DMEMIT("%d ", rs->set.chunk_size_parm);
++
++ if (rs->set.raid_parms > 1)
++ DMEMIT("%d ", rs->sc.stripes_parm);
++
++ if (rs->set.raid_parms > 2)
++ DMEMIT("%d ", rs->set.io_size_parm);
++
++ if (rs->set.raid_parms > 3)
++ DMEMIT("%d ", rs->recover.io_size_parm);
++
++ if (rs->set.raid_parms > 4)
++ DMEMIT("%d ", rs->recover.bandwidth_parm);
++
++ DMEMIT("%u %d ", rs->set.raid_devs, rs->set.dev_to_init);
++
++ for (i = 0; i < rs->set.raid_devs; i++)
++ DMEMIT("%s %llu ",
++ format_dev_t(buf,
++ rs->dev[i].dev->bdev->bd_dev),
++ (unsigned long long) rs->dev[i].start);
++ }
++
++ return 0;
++}
++
++/*
++ * Message interface
++ */
++enum raid_msg_actions {
++ act_bw, /* Recovery bandwidth switch. */
++ act_dev, /* Device failure switch. */
++ act_overwrite, /* Stripe overwrite check. */
++ act_read_ahead, /* Set read ahead. */
++ act_stats, /* Development statistics switch. */
++ act_sc, /* Stripe cache switch. */
++
++ act_on, /* Set entity on. */
++ act_off, /* Set entity off. */
++ act_reset, /* Reset entity. */
++
++ act_set = act_on, /* Set # absolute. */
++ act_grow = act_off, /* Grow # by an amount. */
++ act_shrink = act_reset, /* Shrink # by an amount. */
++};
++
++/* Turn a delta to absolute. */
++static int _absolute(unsigned long action, int act, int r)
++{
++ /* Make delta absolute. */
++ if (test_bit(act_set, &action))
++ ;
++ else if (test_bit(act_grow, &action))
++ r += act;
++ else if (test_bit(act_shrink, &action))
++ r = act - r;
++ else
++ r = -EINVAL;
++
++ return r;
++}
++
++ /* Change recovery io bandwidth. */
++static int bandwidth_change(struct dm_msg *msg, void *context)
++{
++ struct raid_set *rs = context;
++ int act = rs->recover.bandwidth;
++ int bandwidth = DM_MSG_INT_ARG(msg);
++
++ if (range_ok(bandwidth, BANDWIDTH_MIN, BANDWIDTH_MAX)) {
++ /* Make delta bandwidth absolute. */
++ bandwidth = _absolute(msg->action, act, bandwidth);
++
++ /* Check range. */
++ if (range_ok(bandwidth, BANDWIDTH_MIN, BANDWIDTH_MAX)) {
++ recover_set_bandwidth(rs, bandwidth);
++ return 0;
++ }
++ }
++
++ set_bit(dm_msg_ret_arg, &msg->ret);
++ set_bit(dm_msg_ret_inval, &msg->ret);
++ return -EINVAL;
++}
++
++/* Change state of a device (running/offline). */
++/* FIXME: this only works while recovering!. */
++static int device_state(struct dm_msg *msg, void *context)
++{
++ int r;
++ const char *str = "is already ";
++ union dev_lookup dl = { .dev_name = DM_MSG_STR_ARG(msg) };
++ struct raid_set *rs = context;
++
++ r = raid_dev_lookup(rs, strchr(dl.dev_name, ':') ?
++ bymajmin : byname, &dl);
++ if (r == -ENODEV) {
++ DMERR("device %s is no member of this set", dl.dev_name);
++ return r;
++ }
++
++ if (test_bit(act_off, &msg->action)) {
++ if (dev_operational(rs, r))
++ str = "";
++ } else if (!dev_operational(rs, r))
++ str = "";
++
++ DMINFO("/dev/%s %s%s", dl.dev_name, str,
++ test_bit(act_off, &msg->action) ? "offline" : "running");
++
++ return test_bit(act_off, &msg->action) ?
++ raid_set_check_and_degrade(rs, NULL, r) :
++ raid_set_check_and_upgrade(rs, r);
++}
++
++/* Set/reset development feature flags. */
++static int devel_flags(struct dm_msg *msg, void *context)
++{
++ struct raid_set *rs = context;
++
++ if (test_bit(act_on, &msg->action))
++ return test_and_set_bit(msg->spec->parm,
++ &rs->io.flags) ? -EPERM : 0;
++ else if (test_bit(act_off, &msg->action))
++ return test_and_clear_bit(msg->spec->parm,
++ &rs->io.flags) ? 0 : -EPERM;
++ else if (test_bit(act_reset, &msg->action)) {
++ if (test_bit(act_stats, &msg->action)) {
++ stats_reset(rs);
++ goto on;
++ } else if (test_bit(act_overwrite, &msg->action)) {
++on:
++ set_bit(msg->spec->parm, &rs->io.flags);
++ return 0;
++ }
++ }
++
++ return -EINVAL;
++}
++
++ /* Set stripe and chunk read ahead pages. */
++static int read_ahead_set(struct dm_msg *msg, void *context)
++{
++ int stripes = DM_MSG_INT_ARGS(msg, 0);
++ int chunks = DM_MSG_INT_ARGS(msg, 1);
++
++ if (range_ok(stripes, 1, 512) &&
++ range_ok(chunks, 1, 512)) {
++ rs_set_bdi(context, stripes, chunks);
++ return 0;
++ }
++
++ set_bit(dm_msg_ret_arg, &msg->ret);
++ set_bit(dm_msg_ret_inval, &msg->ret);
++ return -EINVAL;
++}
++
++/* Resize the stripe cache. */
++static int stripecache_resize(struct dm_msg *msg, void *context)
++{
++ int act, stripes;
++ struct raid_set *rs = context;
++
++ /* Deny permission in case the daemon is still shrinking!. */
++ if (atomic_read(&rs->sc.stripes_to_shrink))
++ return -EPERM;
++
++ stripes = DM_MSG_INT_ARG(msg);
++ if (stripes > 0) {
++ act = atomic_read(&rs->sc.stripes);
++
++ /* Make delta stripes absolute. */
++ stripes = _absolute(msg->action, act, stripes);
++
++ /*
++ * Check range and that the # of stripes changes.
++ * We can grow from gere but need to leave any
++ * shrinking to the worker for synchronization.
++ */
++ if (range_ok(stripes, STRIPES_MIN, STRIPES_MAX)) {
++ if (stripes > act)
++ return sc_grow(&rs->sc, stripes - act, SC_GROW);
++ else if (stripes < act) {
++ atomic_set(&rs->sc.stripes_to_shrink,
++ act - stripes);
++ wake_do_raid(rs);
++ }
++
++ return 0;
++ }
++ }
++
++ set_bit(dm_msg_ret_arg, &msg->ret);
++ set_bit(dm_msg_ret_inval, &msg->ret);
++ return -EINVAL;
++}
++
++/* Parse the RAID message action. */
++/*
++ * 'ba[ndwidth] {se[t],g[row],sh[rink]} #' # e.g 'ba se 50'
++ * 'de{vice] o[ffline]/r[unning] DevName/maj:min' # e.g 'device o /dev/sda'
++ * "o[verwrite] {on,of[f],r[eset]}' # e.g. 'o of'
++ * "r[ead_ahead] set #stripes #chunks # e.g. 'r se 3 2'
++ * 'sta[tistics] {on,of[f],r[eset]}' # e.g. 'stat of'
++ * 'str[ipecache] {se[t],g[row],sh[rink]} #' # e.g. 'stripe set 1024'
++ *
++ */
++static int
++raid_message(struct dm_target *ti, unsigned argc, char **argv)
++{
++ /* Variables to store the parsed parameters im. */
++ static int i[2];
++ static unsigned long *i_arg[] = {
++ (unsigned long *) i + 0,
++ (unsigned long *) i + 1,
++ };
++ static char *p;
++ static unsigned long *p_arg[] = { (unsigned long *) &p };
++
++ /* Declare all message option strings. */
++ static char *str_sgs[] = { "set", "grow", "shrink" };
++ static char *str_dev[] = { "running", "offline" };
++ static char *str_oor[] = { "on", "off", "reset" };
++
++ /* Declare all actions. */
++ static unsigned long act_sgs[] = { act_set, act_grow, act_shrink };
++ static unsigned long act_oor[] = { act_on, act_off, act_reset };
++
++ /* Bandwidth option. */
++ static struct dm_message_option bw_opt = { 3, str_sgs, act_sgs };
++ static struct dm_message_argument bw_args = {
++ 1, i_arg, { dm_msg_int_t }
++ };
++
++ /* Device option. */
++ static struct dm_message_option dev_opt = { 2, str_dev, act_oor };
++ static struct dm_message_argument dev_args = {
++ 1, p_arg, { dm_msg_base_t }
++ };
++
++ /* Read ahead option. */
++ static struct dm_message_option ra_opt = { 1, str_sgs, act_sgs };
++ static struct dm_message_argument ra_args = {
++ 2, i_arg, { dm_msg_int_t, dm_msg_int_t }
++ };
++
++ static struct dm_message_argument null_args = {
++ 0, NULL, { dm_msg_int_t }
++ };
++
++ /* Overwrite and statistics option. */
++ static struct dm_message_option ovr_stats_opt = { 3, str_oor, act_oor };
++
++ /* Sripecache option. */
++ static struct dm_message_option stripe_opt = { 3, str_sgs, act_sgs };
++
++ /* Declare messages. */
++ static struct dm_msg_spec specs[] = {
++ { "bandwidth", act_bw, &bw_opt, &bw_args,
++ 0, bandwidth_change },
++ { "device", act_dev, &dev_opt, &dev_args,
++ 0, device_state },
++ { "overwrite", act_overwrite, &ovr_stats_opt, &null_args,
++ RS_CHECK_OVERWRITE, devel_flags },
++ { "read_ahead", act_read_ahead, &ra_opt, &ra_args,
++ 0, read_ahead_set },
++ { "statistics", act_stats, &ovr_stats_opt, &null_args,
++ RS_DEVEL_STATS, devel_flags },
++ { "stripecache", act_sc, &stripe_opt, &bw_args,
++ 0, stripecache_resize },
++ };
++
++ /* The message for the parser. */
++ struct dm_msg msg = {
++ .num_specs = ARRAY_SIZE(specs),
++ .specs = specs,
++ };
++
++ return dm_message_parse(TARGET, &msg, ti->private, argc, argv);
++}
++/*
++ * END message interface
++ */
++
++static struct target_type raid_target = {
++ .name = "raid45",
++ .version = {1, 0, 0},
++ .module = THIS_MODULE,
++ .ctr = raid_ctr,
++ .dtr = raid_dtr,
++ .map = raid_map,
++ .postsuspend = raid_postsuspend,
++ .resume = raid_resume,
++ .status = raid_status,
++ .message = raid_message,
++};
++
++static void init_exit(const char *bad_msg, const char *good_msg, int r)
++{
++ if (r)
++ DMERR("Failed to %sregister target [%d]", bad_msg, r);
++ else
++ DMINFO("%s %s", good_msg, version);
++}
++
++static int __init dm_raid_init(void)
++{
++ int r;
++
++ r = dm_register_target(&raid_target);
++ init_exit("", "initialized", r);
++ return r;
++}
++
++static void __exit dm_raid_exit(void)
++{
++ int r;
++
++ r = dm_unregister_target(&raid_target);
++ init_exit("un", "exit", r);
++}
++
++/* Module hooks. */
++module_init(dm_raid_init);
++module_exit(dm_raid_exit);
++
++MODULE_DESCRIPTION(DM_NAME " raid4/5 target");
++MODULE_AUTHOR("Heinz Mauelshagen <hjm@redhat.com>");
++MODULE_LICENSE("GPL");
+--- /dev/null
++++ b/drivers/md/dm-raid45.h
+@@ -0,0 +1,28 @@
++/*
++ * Copyright (C) 2006-2008 Red Hat, Inc. All rights reserved.
++ *
++ * Module Author: Heinz Mauelshagen (Mauelshagen@RedHat.com)
++ *
++ * Locking definitions for the device-mapper RAID45 target.
++ *
++ * This file is released under the GPL.
++ *
++ */
++
++#ifndef _DM_RAID45_H
++#define _DM_RAID45_H
++
++/* Factor out to dm.h! */
++#define STR_LEN(ptr, str) (ptr), (str), strlen((ptr))
++
++enum dm_lock_type { DM_RAID45_EX, DM_RAID45_SHARED };
++
++struct dm_raid45_locking_type {
++ /* Request a lock on a stripe. */
++ void* (*lock)(sector_t key, enum dm_lock_type type);
++
++ /* Release a lock on a stripe. */
++ void (*unlock)(void *lock_handle);
++};
++
++#endif
+--- /dev/null
++++ b/drivers/md/dm-regions.c
+@@ -0,0 +1,723 @@
++/*
++ * Copyright (C) 2003 Sistina Software Limited.
++ * Copyright (C) 2004-2008 Red Hat, Inc. All rights reserved.
++ *
++ * This file is released under the GPL.
++ */
++
++#include <linux/dm-dirty-log.h>
++#include <linux/dm-regions.h>
++
++#include <linux/ctype.h>
++#include <linux/init.h>
++#include <linux/module.h>
++#include <linux/vmalloc.h>
++
++#include "dm.h"
++#include "dm-bio-list.h"
++
++#define DM_MSG_PREFIX "region hash"
++
++/*-----------------------------------------------------------------
++ * Region hash
++ *
++ * A storage set (eg. RAID1, RAID5) splits itself up into discrete regions.
++ * Each region can be in one of three states:
++ *
++ * o clean
++ * o dirty,
++ * o nosync.
++ *
++ * There is no need to put clean regions in the hash.
++ *
++ *
++ * In addition to being present in the hash table a region _may_
++ * be present on one of three lists.
++ *
++ * clean_regions: Regions on this list have no io pending to
++ * them, they are in sync, we are no longer interested in them,
++ * they are dull. dm_rh_update_states() will remove them from the
++ * hash table.
++ *
++ * quiesced_regions: These regions have been spun down, ready
++ * for recovery. dm_rh_recovery_start() will remove regions from
++ * this list and hand them to the caller, which will schedule the
++ * recovery io.
++ *
++ * recovered_regions: Regions that the caller has successfully
++ * recovered. dm_rh_update_states() will now schedule any delayed
++ * io, up the recovery_count, and remove the region from the hash.
++ *
++ * There are 2 locks:
++ * A rw spin lock 'hash_lock' protects just the hash table,
++ * this is never held in write mode from interrupt context,
++ * which I believe means that we only have to disable irqs when
++ * doing a write lock.
++ *
++ * An ordinary spin lock 'region_lock' that protects the three
++ * lists in the region_hash, with the 'state', 'list' and
++ * 'delayed_bios' fields of the regions. This is used from irq
++ * context, so all other uses will have to suspend local irqs.
++ *---------------------------------------------------------------*/
++struct region_hash {
++ unsigned max_recovery; /* Max # of regions to recover in parallel */
++
++ /* Callback function to dispatch queued writes on recovered regions. */
++ void (*dispatch)(void *context, struct bio_list *bios, int error);
++ void *dispatch_context;
++
++ /* Callback function to wakeup callers worker thread. */
++ void (*wake)(void *context);
++ void *wake_context;
++
++ uint32_t region_size;
++ unsigned region_shift;
++
++ /* holds persistent region state */
++ struct dm_dirty_log *log;
++
++ /* hash table */
++ rwlock_t hash_lock;
++ mempool_t *region_pool;
++ unsigned mask;
++ unsigned nr_buckets;
++ unsigned prime;
++ unsigned shift;
++ struct list_head *buckets;
++
++ spinlock_t region_lock;
++ atomic_t recovery_in_flight;
++ struct semaphore recovery_count;
++ struct list_head clean_regions;
++ struct list_head quiesced_regions;
++ struct list_head recovered_regions;
++ struct list_head failed_recovered_regions;
++};
++
++struct region {
++ region_t key;
++ enum dm_rh_region_states state;
++ void *context; /* Caller context. */
++
++ struct list_head hash_list;
++ struct list_head list;
++
++ atomic_t pending;
++ struct bio_list delayed_bios;
++};
++
++/*
++ * Conversion fns
++ */
++region_t dm_rh_sector_to_region(struct dm_rh_client *rh, sector_t sector)
++{
++ return sector >> ((struct region_hash *) rh)->region_shift;
++}
++EXPORT_SYMBOL_GPL(dm_rh_sector_to_region);
++
++region_t dm_rh_bio_to_region(struct dm_rh_client *rh, struct bio *bio)
++{
++ return dm_rh_sector_to_region(rh, bio->bi_sector);
++}
++EXPORT_SYMBOL_GPL(dm_rh_bio_to_region);
++
++sector_t dm_rh_region_to_sector(struct dm_rh_client *rh, region_t region)
++{
++ return region << ((struct region_hash *) rh)->region_shift;
++}
++EXPORT_SYMBOL_GPL(dm_rh_region_to_sector);
++
++/*
++ * Retrival fns.
++ */
++region_t dm_rh_get_region_key(struct dm_region *reg)
++{
++ return ((struct region *) reg)->key;
++}
++EXPORT_SYMBOL_GPL(dm_rh_get_region_key);
++
++sector_t dm_rh_get_region_size(struct dm_rh_client *rh)
++{
++ return ((struct region_hash *) rh)->region_size;
++}
++EXPORT_SYMBOL_GPL(dm_rh_get_region_size);
++
++/* Squirrel a context with a region. */
++void *dm_rh_reg_get_context(struct dm_region *reg)
++{
++ return ((struct region *) reg)->context;
++}
++EXPORT_SYMBOL_GPL(dm_rh_reg_get_context);
++
++void dm_rh_reg_set_context(struct dm_region *reg, void *context)
++{
++ ((struct region *) reg)->context = context;
++}
++EXPORT_SYMBOL_GPL(dm_rh_reg_set_context);
++
++/*
++ * Create region hash client.
++ */
++#define MIN_REGIONS 64
++struct dm_rh_client *dm_rh_client_create(
++ unsigned max_recovery,
++ void (*dispatch)(void *dispatch_context,
++ struct bio_list *bios, int error),
++ void *dispatch_context,
++ void (*wake)(void *wake_context), void *wake_context,
++ struct dm_dirty_log *log, uint32_t region_size,
++ region_t nr_regions)
++{
++ unsigned i;
++ unsigned nr_buckets, max_buckets;
++ unsigned hash_primes[] = {
++ /* Table of primes for rh_hash/table size optimization. */
++ 3, 7, 13, 27, 53, 97, 193, 389, 769,
++ 1543, 3079, 6151, 12289, 24593,
++ };
++ struct region_hash *rh;
++
++ if (region_size & (region_size - 1)) {
++ DMERR("region size must be 2^^n");
++ return ERR_PTR(-EINVAL);
++ }
++
++ /* Calculate a suitable number of buckets for our hash table. */
++ max_buckets = nr_regions >> 6;
++ for (nr_buckets = 128u; nr_buckets < max_buckets; nr_buckets <<= 1)
++ ;
++ nr_buckets >>= 1;
++
++ rh = kmalloc(sizeof(*rh), GFP_KERNEL);
++ if (!rh) {
++ DMERR("unable to allocate region hash memory");
++ return ERR_PTR(-ENOMEM);
++ }
++
++ rh->max_recovery = max_recovery;
++ rh->dispatch = dispatch;
++ rh->dispatch_context = dispatch_context;
++ rh->wake = wake;
++ rh->wake_context = wake_context;
++ rh->log = log;
++ rh->region_size = region_size;
++ rh->region_shift = ffs(region_size) - 1;
++ rwlock_init(&rh->hash_lock);
++ rh->mask = nr_buckets - 1;
++ rh->nr_buckets = nr_buckets;
++ rh->shift = ffs(nr_buckets);
++
++ /* Check prime array limits. */
++ i = rh->shift - 1 > ARRAY_SIZE(hash_primes) ?
++ ARRAY_SIZE(hash_primes) - 1 : rh->shift - 2;
++ rh->prime = hash_primes[i];
++
++ rh->buckets = vmalloc(nr_buckets * sizeof(*rh->buckets));
++ if (!rh->buckets) {
++ DMERR("unable to allocate region hash bucket memory");
++ kfree(rh);
++ return ERR_PTR(-ENOMEM);
++ }
++
++ for (i = 0; i < nr_buckets; i++)
++ INIT_LIST_HEAD(rh->buckets + i);
++
++ spin_lock_init(&rh->region_lock);
++ sema_init(&rh->recovery_count, 0);
++ atomic_set(&rh->recovery_in_flight, 0);
++ INIT_LIST_HEAD(&rh->clean_regions);
++ INIT_LIST_HEAD(&rh->quiesced_regions);
++ INIT_LIST_HEAD(&rh->recovered_regions);
++ INIT_LIST_HEAD(&rh->failed_recovered_regions);
++
++ rh->region_pool = mempool_create_kmalloc_pool(MIN_REGIONS,
++ sizeof(struct region));
++ if (!rh->region_pool) {
++ vfree(rh->buckets);
++ kfree(rh);
++ rh = ERR_PTR(-ENOMEM);
++ }
++
++ return (struct dm_rh_client *) rh;
++}
++EXPORT_SYMBOL_GPL(dm_rh_client_create);
++
++void dm_rh_client_destroy(struct dm_rh_client *rh_in)
++{
++ unsigned h;
++ struct region_hash *rh = (struct region_hash *) rh_in;
++ struct region *reg, *tmp;
++
++ BUG_ON(!list_empty(&rh->quiesced_regions));
++
++ for (h = 0; h < rh->nr_buckets; h++) {
++ list_for_each_entry_safe(reg, tmp, rh->buckets + h, hash_list) {
++ BUG_ON(atomic_read(®->pending));
++ mempool_free(reg, rh->region_pool);
++ }
++ }
++
++ if (rh->region_pool)
++ mempool_destroy(rh->region_pool);
++
++ vfree(rh->buckets);
++ kfree(rh);
++}
++EXPORT_SYMBOL_GPL(dm_rh_client_destroy);
++
++static inline unsigned rh_hash(struct region_hash *rh, region_t region)
++{
++ return (unsigned) ((region * rh->prime) >> rh->shift) & rh->mask;
++}
++
++static struct region *__rh_lookup(struct region_hash *rh, region_t region)
++{
++ struct region *reg;
++ struct list_head *bucket = rh->buckets + rh_hash(rh, region);
++
++ list_for_each_entry(reg, bucket, hash_list) {
++ if (reg->key == region)
++ return reg;
++ }
++
++ return NULL;
++}
++
++static void __rh_insert(struct region_hash *rh, struct region *reg)
++{
++ list_add(®->hash_list, rh->buckets + rh_hash(rh, reg->key));
++}
++
++static struct region *__rh_alloc(struct region_hash *rh, region_t region)
++{
++ struct region *reg, *nreg;
++
++ read_unlock(&rh->hash_lock);
++ nreg = mempool_alloc(rh->region_pool, GFP_ATOMIC);
++ if (unlikely(!nreg))
++ nreg = kmalloc(sizeof(*nreg), GFP_NOIO);
++
++ nreg->state = rh->log->type->in_sync(rh->log, region, 1) ?
++ DM_RH_CLEAN : DM_RH_NOSYNC;
++ nreg->key = region;
++ INIT_LIST_HEAD(&nreg->list);
++ atomic_set(&nreg->pending, 0);
++ bio_list_init(&nreg->delayed_bios);
++
++ write_lock_irq(&rh->hash_lock);
++ reg = __rh_lookup(rh, region);
++ if (reg)
++ /* We lost the race. */
++ mempool_free(nreg, rh->region_pool);
++ else {
++ __rh_insert(rh, nreg);
++ if (nreg->state == DM_RH_CLEAN) {
++ spin_lock(&rh->region_lock);
++ list_add(&nreg->list, &rh->clean_regions);
++ spin_unlock(&rh->region_lock);
++ }
++
++ reg = nreg;
++ }
++
++ write_unlock_irq(&rh->hash_lock);
++ read_lock(&rh->hash_lock);
++ return reg;
++}
++
++static inline struct region *__rh_find(struct region_hash *rh, region_t region)
++{
++ struct region *reg;
++
++ reg = __rh_lookup(rh, region);
++ return reg ? reg : __rh_alloc(rh, region);
++}
++
++int dm_rh_get_state(struct dm_rh_client *rh_in, region_t region, int may_block)
++{
++ int r;
++ struct region_hash *rh = (struct region_hash *) rh_in;
++ struct region *reg;
++
++ read_lock(&rh->hash_lock);
++ reg = __rh_lookup(rh, region);
++ read_unlock(&rh->hash_lock);
++
++ if (reg)
++ return reg->state;
++
++ /*
++ * The region wasn't in the hash, so we fall back to the dirty log.
++ */
++ r = rh->log->type->in_sync(rh->log, region, may_block);
++
++ /*
++ * Any error from the dirty log (eg. -EWOULDBLOCK)
++ * gets taken as a DM_RH_NOSYNC
++ */
++ return r == 1 ? DM_RH_CLEAN : DM_RH_NOSYNC;
++}
++EXPORT_SYMBOL_GPL(dm_rh_get_state);
++
++void dm_rh_set_state(struct dm_rh_client *rh_in, region_t region,
++ enum dm_rh_region_states state, int may_block)
++{
++ struct region_hash *rh = (struct region_hash *) rh_in;
++ struct region *reg;
++ struct dm_dirty_log *log = rh->log;
++
++ if (state == DM_RH_NOSYNC)
++ log->type->set_region_sync(log, region, 0);
++ else if (state == DM_RH_CLEAN)
++ log->type->clear_region(log, region);
++ else if (state == DM_RH_DIRTY)
++ log->type->mark_region(log, region);
++
++ read_lock(&rh->hash_lock);
++ reg = __rh_find(rh, region);
++ reg->state = state;
++ read_unlock(&rh->hash_lock);
++}
++EXPORT_SYMBOL_GPL(dm_rh_set_state);
++
++void dm_rh_update_states(struct dm_rh_client *rh_in, int errors_handled)
++{
++ struct region_hash *rh = (struct region_hash *) rh_in;
++ struct region *reg, *next;
++ LIST_HEAD(clean);
++ LIST_HEAD(recovered);
++ LIST_HEAD(failed_recovered);
++
++ /*
++ * Quickly grab the lists and remove any regions from hash.
++ */
++ write_lock_irq(&rh->hash_lock);
++ spin_lock(&rh->region_lock);
++ if (!list_empty(&rh->clean_regions)) {
++ list_splice_init(&rh->clean_regions, &clean);
++
++ list_for_each_entry(reg, &clean, list)
++ list_del(®->hash_list);
++ }
++
++ if (!list_empty(&rh->recovered_regions)) {
++ list_splice_init(&rh->recovered_regions, &recovered);
++
++ list_for_each_entry(reg, &recovered, list)
++ list_del(®->hash_list);
++ }
++
++ if (!list_empty(&rh->failed_recovered_regions)) {
++ list_splice_init(&rh->failed_recovered_regions,
++ &failed_recovered);
++
++ list_for_each_entry(reg, &recovered, list)
++ list_del(®->hash_list);
++ }
++
++ spin_unlock(&rh->region_lock);
++ write_unlock_irq(&rh->hash_lock);
++
++ /*
++ * All the regions on the recovered and clean lists have
++ * now been pulled out of the system, so no need to do
++ * any more locking.
++ */
++ list_for_each_entry_safe(reg, next, &recovered, list) {
++ rh->log->type->clear_region(rh->log, reg->key);
++ rh->log->type->set_region_sync(rh->log, reg->key, 1);
++
++ if (reg->delayed_bios.head)
++ rh->dispatch(rh->dispatch_context,
++ ®->delayed_bios, 0);
++
++ up(&rh->recovery_count);
++ mempool_free(reg, rh->region_pool);
++ }
++
++ list_for_each_entry_safe(reg, next, &failed_recovered, list) {
++ rh->log->type->set_region_sync(rh->log, reg->key,
++ errors_handled ? 0 : 1);
++ if (reg->delayed_bios.head)
++ rh->dispatch(rh->dispatch_context,
++ ®->delayed_bios, -EIO);
++
++ up(&rh->recovery_count);
++ mempool_free(reg, rh->region_pool);
++ }
++
++ list_for_each_entry_safe(reg, next, &clean, list) {
++ rh->log->type->clear_region(rh->log, reg->key);
++ mempool_free(reg, rh->region_pool);
++ }
++
++ dm_rh_flush(rh_in);
++}
++EXPORT_SYMBOL_GPL(dm_rh_update_states);
++
++void dm_rh_inc(struct dm_rh_client *rh_in, region_t region)
++{
++ struct region_hash *rh = (struct region_hash *) rh_in;
++ struct region *reg;
++
++ read_lock(&rh->hash_lock);
++ reg = __rh_find(rh, region);
++ if (reg->state == DM_RH_CLEAN) {
++ rh->log->type->mark_region(rh->log, reg->key);
++
++ spin_lock_irq(&rh->region_lock);
++ reg->state = DM_RH_DIRTY;
++ list_del_init(®->list); /* Take off the clean list. */
++ spin_unlock_irq(&rh->region_lock);
++ }
++
++ atomic_inc(®->pending);
++ read_unlock(&rh->hash_lock);
++}
++EXPORT_SYMBOL_GPL(dm_rh_inc);
++
++void dm_rh_inc_pending(struct dm_rh_client *rh_in, struct bio_list *bios)
++{
++ struct bio *bio;
++
++ for (bio = bios->head; bio; bio = bio->bi_next)
++ dm_rh_inc(rh_in, dm_rh_bio_to_region(rh_in, bio));
++}
++EXPORT_SYMBOL_GPL(dm_rh_inc_pending);
++
++int dm_rh_dec(struct dm_rh_client *rh_in, region_t region)
++{
++ int r = 0;
++ struct region_hash *rh = (struct region_hash *) rh_in;
++ struct region *reg;
++
++ read_lock(&rh->hash_lock);
++ reg = __rh_lookup(rh, region);
++ read_unlock(&rh->hash_lock);
++
++ BUG_ON(!reg);
++
++ if (atomic_dec_and_test(®->pending)) {
++ unsigned long flags;
++
++ /*
++ * There is no pending I/O for this region.
++ * We can move the region to corresponding list for next action.
++ * At this point, the region is not yet connected to any list.
++ *
++ * If the state is DM_RH_NOSYNC, the region should be kept off
++ * from clean list.
++ * The hash entry for DM_RH_NOSYNC will remain in memory
++ * until the region is recovered or the map is reloaded.
++ */
++
++ spin_lock_irqsave(&rh->region_lock, flags);
++ if (reg->state == DM_RH_RECOVERING)
++ list_add_tail(®->list, &rh->quiesced_regions);
++ else {
++ reg->state = DM_RH_CLEAN;
++ list_add(®->list, &rh->clean_regions);
++ }
++ spin_unlock_irqrestore(&rh->region_lock, flags);
++
++ r = 1;
++ }
++
++ return r;
++}
++EXPORT_SYMBOL_GPL(dm_rh_dec);
++
++/*
++ * Starts quiescing a region in preparation for recovery.
++ */
++static int __rh_recovery_prepare(struct region_hash *rh)
++{
++ int r;
++ region_t region;
++ struct region *reg;
++
++ /*
++ * Ask the dirty log what's next.
++ */
++ r = rh->log->type->get_resync_work(rh->log, ®ion);
++ if (r <= 0)
++ return r;
++
++ /*
++ * Get this region, and start it quiescing
++ * by setting the recovering flag.
++ */
++ read_lock(&rh->hash_lock);
++ reg = __rh_find(rh, region);
++ read_unlock(&rh->hash_lock);
++
++ spin_lock_irq(&rh->region_lock);
++
++ reg->state = DM_RH_RECOVERING;
++
++ /* Already quiesced ? */
++ list_del_init(®->list);
++ if (!atomic_read(®->pending))
++ list_add(®->list, &rh->quiesced_regions);
++
++ spin_unlock_irq(&rh->region_lock);
++ return 1;
++}
++
++int dm_rh_recovery_prepare(struct dm_rh_client *rh_in)
++{
++ int r = 0;
++ struct region_hash *rh = (struct region_hash *) rh_in;
++
++ /* Extra reference to avoid race with rh_stop_recovery */
++ atomic_inc(&rh->recovery_in_flight);
++
++ while (!down_trylock(&rh->recovery_count)) {
++ atomic_inc(&rh->recovery_in_flight);
++
++ if (__rh_recovery_prepare(rh) <= 0) {
++ atomic_dec(&rh->recovery_in_flight);
++ up(&rh->recovery_count);
++ r = -ENOENT;
++ break;
++ }
++ }
++
++ /* Drop the extra reference. */
++ if (atomic_dec_and_test(&rh->recovery_in_flight))
++ r = -ESRCH;
++
++ return r;
++}
++EXPORT_SYMBOL_GPL(dm_rh_recovery_prepare);
++
++/*
++ * Returns any quiesced regions.
++ */
++struct dm_region *dm_rh_recovery_start(struct dm_rh_client *rh_in)
++{
++ struct region_hash *rh = (struct region_hash *) rh_in;
++ struct region *reg = NULL;
++
++ spin_lock_irq(&rh->region_lock);
++ if (!list_empty(&rh->quiesced_regions)) {
++ reg = list_entry(rh->quiesced_regions.next,
++ struct region, list);
++ list_del_init(®->list); /* Remove from the quiesced list. */
++ }
++
++ spin_unlock_irq(&rh->region_lock);
++ return (struct dm_region *) reg;
++}
++EXPORT_SYMBOL_GPL(dm_rh_recovery_start);
++
++/*
++ * Put region on list of recovered ones.
++ */
++void dm_rh_recovery_end(struct dm_rh_client *rh_in, struct dm_region *reg_in,
++ int error)
++{
++ struct region_hash *rh = (struct region_hash *) rh_in;
++ struct region *reg = (struct region *) reg_in;
++
++ spin_lock_irq(&rh->region_lock);
++ if (error) {
++ reg->state = DM_RH_NOSYNC;
++ list_add(®->list, &rh->failed_recovered_regions);
++ } else
++ list_add(®->list, &rh->recovered_regions);
++
++ atomic_dec(&rh->recovery_in_flight);
++ spin_unlock_irq(&rh->region_lock);
++
++ rh->wake(rh->wake_context);
++ BUG_ON(atomic_read(&rh->recovery_in_flight) < 0);
++}
++EXPORT_SYMBOL_GPL(dm_rh_recovery_end);
++
++/* Return recovery in flight count. */
++int dm_rh_recovery_in_flight(struct dm_rh_client *rh_in)
++{
++ return atomic_read(&((struct region_hash *) rh_in)->recovery_in_flight);
++}
++EXPORT_SYMBOL_GPL(dm_rh_recovery_in_flight);
++
++int dm_rh_flush(struct dm_rh_client *rh_in)
++{
++ struct region_hash *rh = (struct region_hash *) rh_in;
++
++ return rh->log->type->flush(rh->log);
++}
++EXPORT_SYMBOL_GPL(dm_rh_flush);
++
++void dm_rh_delay_by_region(struct dm_rh_client *rh_in,
++ struct bio *bio, region_t region)
++{
++ struct region_hash *rh = (struct region_hash *) rh_in;
++ struct region *reg;
++
++ /* FIXME: locking. */
++ read_lock(&rh->hash_lock);
++ reg = __rh_find(rh, region);
++ bio_list_add(®->delayed_bios, bio);
++ read_unlock(&rh->hash_lock);
++}
++EXPORT_SYMBOL_GPL(dm_rh_delay_by_region);
++
++void dm_rh_delay(struct dm_rh_client *rh_in, struct bio *bio)
++{
++ return dm_rh_delay_by_region(rh_in, bio,
++ dm_rh_bio_to_region(rh_in, bio));
++}
++EXPORT_SYMBOL_GPL(dm_rh_delay);
++
++void dm_rh_dispatch_bios(struct dm_rh_client *rh_in,
++ region_t region, int error)
++{
++ struct region_hash *rh = (struct region_hash *) rh_in;
++ struct region *reg;
++ struct bio_list delayed_bios;
++
++ /* FIXME: locking. */
++ read_lock(&rh->hash_lock);
++ reg = __rh_find(rh, region);
++ BUG_ON(!reg);
++ delayed_bios = reg->delayed_bios;
++ bio_list_init(®->delayed_bios);
++ read_unlock(&rh->hash_lock);
++
++ if (delayed_bios.head)
++ rh->dispatch(rh->dispatch_context, &delayed_bios, error);
++
++ up(&rh->recovery_count);
++}
++EXPORT_SYMBOL_GPL(dm_rh_dispatch_bios);
++
++void dm_rh_stop_recovery(struct dm_rh_client *rh_in)
++{
++ int i;
++ struct region_hash *rh = (struct region_hash *) rh_in;
++
++ rh->wake(rh->wake_context);
++
++ /* wait for any recovering regions */
++ for (i = 0; i < rh->max_recovery; i++)
++ down(&rh->recovery_count);
++}
++EXPORT_SYMBOL_GPL(dm_rh_stop_recovery);
++
++void dm_rh_start_recovery(struct dm_rh_client *rh_in)
++{
++ int i;
++ struct region_hash *rh = (struct region_hash *) rh_in;
++
++ for (i = 0; i < rh->max_recovery; i++)
++ up(&rh->recovery_count);
++
++ rh->wake(rh->wake_context);
++}
++EXPORT_SYMBOL_GPL(dm_rh_start_recovery);
++
++MODULE_DESCRIPTION(DM_NAME " region hash");
++MODULE_AUTHOR("Joe Thornber/Heinz Mauelshagen <hjm@redhat.com>");
++MODULE_LICENSE("GPL");
+--- a/drivers/md/Kconfig
++++ b/drivers/md/Kconfig
+@@ -269,6 +269,14 @@ config DM_DELAY
+
+ If unsure, say N.
+
++config DM_RAID45
++ tristate "RAID 4/5 target (EXPERIMENTAL)"
++ depends on BLK_DEV_DM && EXPERIMENTAL
++ ---help---
++ A target that supports RAID4 and RAID5 mappings.
++
++ If unsure, say N.
++
+ config DM_UEVENT
+ bool "DM uevents (EXPERIMENTAL)"
+ depends on BLK_DEV_DM && EXPERIMENTAL
+--- a/drivers/md/Makefile
++++ b/drivers/md/Makefile
+@@ -34,7 +34,9 @@ obj-$(CONFIG_DM_CRYPT) += dm-crypt.o
+ obj-$(CONFIG_DM_DELAY) += dm-delay.o
+ obj-$(CONFIG_DM_MULTIPATH) += dm-multipath.o dm-round-robin.o
+ obj-$(CONFIG_DM_SNAPSHOT) += dm-snapshot.o
+-obj-$(CONFIG_DM_MIRROR) += dm-mirror.o dm-log.o
++obj-$(CONFIG_DM_MIRROR) += dm-mirror.o dm-regions.o dm-log.o
++obj-$(CONFIG_DM_RAID45) += dm-raid45.o dm-log.o dm-memcache.o \
++ dm-regions.o dm-message.o
+ obj-$(CONFIG_DM_ZERO) += dm-zero.o
+
+ quiet_cmd_unroll = UNROLL $@
+--- /dev/null
++++ b/include/linux/dm-regions.h
+@@ -0,0 +1,115 @@
++/*
++ * Copyright (C) 2003 Sistina Software Limited.
++ * Copyright (C) 2004-2008 Red Hat, Inc. All rights reserved.
++ *
++ * Device-Mapper dirty region hash interface.
++ *
++ * This file is released under the GPL.
++ */
++
++#ifndef DM_REGION_HASH_H
++#define DM_REGION_HASH_H
++
++#include <linux/dm-dirty-log.h>
++
++/*-----------------------------------------------------------------
++ * Region hash
++ *----------------------------------------------------------------*/
++struct dm_rh_client;
++struct dm_region;
++
++/*
++ * States a region can have.
++ */
++enum dm_rh_region_states {
++ DM_RH_CLEAN = 0x01, /* No writes in flight. */
++ DM_RH_DIRTY = 0x02, /* Writes in flight. */
++ DM_RH_NOSYNC = 0x04, /* Out of sync. */
++ DM_RH_RECOVERING = 0x08, /* Under resynchronization. */
++};
++
++/*
++ * Region hash create/destroy.
++ */
++struct bio_list;
++struct dm_rh_client *dm_rh_client_create(
++ unsigned max_recovery,
++ void (*dispatch)(void *dispatch_context,
++ struct bio_list *bios, int error),
++ void *dispatch_context,
++ void (*wake)(void *wake_context), void *wake_context,
++ struct dm_dirty_log *log, uint32_t region_size,
++ region_t nr_regions);
++void dm_rh_client_destroy(struct dm_rh_client *rh);
++
++/*
++ * Conversion fns:
++ *
++ * bio -> region
++ * sector -> region
++ * region -> sector
++ */
++region_t dm_rh_bio_to_region(struct dm_rh_client *rh, struct bio *bio);
++region_t dm_rh_sector_to_region(struct dm_rh_client *rh, sector_t sector);
++sector_t dm_rh_region_to_sector(struct dm_rh_client *rh, region_t region);
++
++/*
++ * Functions to set a caller context in a region.
++ */
++void *dm_rh_reg_get_context(struct dm_region *reg);
++void dm_rh_reg_set_context(struct dm_region *reg, void *context);
++
++/*
++ * Get region size and key (ie. number of the region).
++ */
++sector_t dm_rh_get_region_size(struct dm_rh_client *rh);
++sector_t dm_rh_get_region_key(struct dm_region *reg);
++
++/*
++ * Get/set/update region state (and dirty log).
++ *
++ * dm_rh_update_states
++ * @errors_handled != 0 influences
++ * that the state of the region will be kept NOSYNC
++ */
++int dm_rh_get_state(struct dm_rh_client *rh, region_t region, int may_block);
++void dm_rh_set_state(struct dm_rh_client *rh, region_t region,
++ enum dm_rh_region_states state, int may_block);
++void dm_rh_update_states(struct dm_rh_client *rh, int errors_handled);
++
++/* Flush the region hash and dirty log. */
++int dm_rh_flush(struct dm_rh_client *rh);
++
++/* Inc/dec pending count on regions. */
++void dm_rh_inc(struct dm_rh_client *rh, region_t region);
++void dm_rh_inc_pending(struct dm_rh_client *rh, struct bio_list *bios);
++int dm_rh_dec(struct dm_rh_client *rh, region_t region);
++
++/* Delay bios on regions. */
++void dm_rh_delay(struct dm_rh_client *rh, struct bio *bio);
++void dm_rh_delay_by_region(struct dm_rh_client *rh,
++ struct bio *bio, region_t region);
++
++/*
++ * Normally, the region hash will automatically call the dispatch function.
++ * dm_rh_dispatch_bios() is for intentional dispatching of bios.
++ */
++void dm_rh_dispatch_bios(struct dm_rh_client *rh, region_t region, int error);
++
++/*
++ * Region recovery control.
++ */
++/* Prepare some regions for recovery by starting to quiesce them. */
++int dm_rh_recovery_prepare(struct dm_rh_client *rh);
++/* Try fetching a quiesced region for recovery. */
++struct dm_region *dm_rh_recovery_start(struct dm_rh_client *rh);
++/* Report recovery end on a region. */
++void dm_rh_recovery_end(struct dm_rh_client *rh, struct dm_region *reg,
++ int error);
++/* Check for amount of recoveries in flight. */
++int dm_rh_recovery_in_flight(struct dm_rh_client *rh);
++/* Start/stop recovery. */
++void dm_rh_stop_recovery(struct dm_rh_client *rh);
++void dm_rh_start_recovery(struct dm_rh_client *rh);
++
++#endif /* #ifdef DM_REGION_HASH_H */
projects / people / teissler / ipfire-2.x.git / blobdiff