[thirdparty/linux.git] / block / t10-pi.c

// SPDX-License-Identifier: GPL-2.0
/*
 * t10_pi.c - Functions for generating and verifying T10 Protection
 *	      Information.
 */

#include <linux/t10-pi.h>
#include <linux/blk-integrity.h>
#include <linux/crc-t10dif.h>
#include <linux/crc64.h>
#include <linux/module.h>
#include <net/checksum.h>
#include <asm/unaligned.h>

typedef __be16 (csum_fn) (void *, unsigned int);

static __be16 t10_pi_crc_fn(void *data, unsigned int len)
{
	return cpu_to_be16(crc_t10dif(data, len));
}

static __be16 t10_pi_ip_fn(void *data, unsigned int len)
{
	return (__force __be16)ip_compute_csum(data, len);
}

/*
 * Type 1 and Type 2 protection use the same format: 16 bit guard tag,
 * 16 bit app tag, 32 bit reference tag. Type 3 does not define the ref
 * tag.
 */
static blk_status_t t10_pi_generate(struct blk_integrity_iter *iter,
		csum_fn *fn, enum t10_dif_type type)
{
	unsigned int i;

	for (i = 0 ; i < iter->data_size ; i += iter->interval) {
		struct t10_pi_tuple *pi = iter->prot_buf;

		pi->guard_tag = fn(iter->data_buf, iter->interval);
		pi->app_tag = 0;

		if (type == T10_PI_TYPE1_PROTECTION)
			pi->ref_tag = cpu_to_be32(lower_32_bits(iter->seed));
		else
			pi->ref_tag = 0;

		iter->data_buf += iter->interval;
		iter->prot_buf += iter->tuple_size;
		iter->seed++;
	}

	return BLK_STS_OK;
}

static blk_status_t t10_pi_verify(struct blk_integrity_iter *iter,
		csum_fn *fn, enum t10_dif_type type)
{
	unsigned int i;

	BUG_ON(type == T10_PI_TYPE0_PROTECTION);

	for (i = 0 ; i < iter->data_size ; i += iter->interval) {
		struct t10_pi_tuple *pi = iter->prot_buf;
		__be16 csum;

		if (type == T10_PI_TYPE1_PROTECTION ||
		    type == T10_PI_TYPE2_PROTECTION) {
			if (pi->app_tag == T10_PI_APP_ESCAPE)
				goto next;

			if (be32_to_cpu(pi->ref_tag) !=
			    lower_32_bits(iter->seed)) {
				pr_err("%s: ref tag error at location %llu " \
				       "(rcvd %u)\n", iter->disk_name,
				       (unsigned long long)
				       iter->seed, be32_to_cpu(pi->ref_tag));
				return BLK_STS_PROTECTION;
			}
		} else if (type == T10_PI_TYPE3_PROTECTION) {
			if (pi->app_tag == T10_PI_APP_ESCAPE &&
			    pi->ref_tag == T10_PI_REF_ESCAPE)
				goto next;
		}

		csum = fn(iter->data_buf, iter->interval);

		if (pi->guard_tag != csum) {
			pr_err("%s: guard tag error at sector %llu " \
			       "(rcvd %04x, want %04x)\n", iter->disk_name,
			       (unsigned long long)iter->seed,
			       be16_to_cpu(pi->guard_tag), be16_to_cpu(csum));
			return BLK_STS_PROTECTION;
		}

next:
		iter->data_buf += iter->interval;
		iter->prot_buf += iter->tuple_size;
		iter->seed++;
	}

	return BLK_STS_OK;
}

static blk_status_t t10_pi_type1_generate_crc(struct blk_integrity_iter *iter)
{
	return t10_pi_generate(iter, t10_pi_crc_fn, T10_PI_TYPE1_PROTECTION);
}

static blk_status_t t10_pi_type1_generate_ip(struct blk_integrity_iter *iter)
{
	return t10_pi_generate(iter, t10_pi_ip_fn, T10_PI_TYPE1_PROTECTION);
}

static blk_status_t t10_pi_type1_verify_crc(struct blk_integrity_iter *iter)
{
	return t10_pi_verify(iter, t10_pi_crc_fn, T10_PI_TYPE1_PROTECTION);
}

static blk_status_t t10_pi_type1_verify_ip(struct blk_integrity_iter *iter)
{
	return t10_pi_verify(iter, t10_pi_ip_fn, T10_PI_TYPE1_PROTECTION);
}

/**
 * t10_pi_type1_prepare - prepare PI prior submitting request to device
 * @rq:              request with PI that should be prepared
 *
 * For Type 1/Type 2, the virtual start sector is the one that was
 * originally submitted by the block layer for the ref_tag usage. Due to
 * partitioning, MD/DM cloning, etc. the actual physical start sector is
 * likely to be different. Remap protection information to match the
 * physical LBA.
 */
static void t10_pi_type1_prepare(struct request *rq)
{
	const int tuple_sz = rq->q->integrity.tuple_size;
	u32 ref_tag = t10_pi_ref_tag(rq);
	struct bio *bio;

	__rq_for_each_bio(bio, rq) {
		struct bio_integrity_payload *bip = bio_integrity(bio);
		u32 virt = bip_get_seed(bip) & 0xffffffff;
		struct bio_vec iv;
		struct bvec_iter iter;

		/* Already remapped? */
		if (bip->bip_flags & BIP_MAPPED_INTEGRITY)
			break;

		bip_for_each_vec(iv, bip, iter) {
			unsigned int j;
			void *p;

			p = bvec_kmap_local(&iv);
			for (j = 0; j < iv.bv_len; j += tuple_sz) {
				struct t10_pi_tuple *pi = p;

				if (be32_to_cpu(pi->ref_tag) == virt)
					pi->ref_tag = cpu_to_be32(ref_tag);
				virt++;
				ref_tag++;
				p += tuple_sz;
			}
			kunmap_local(p);
		}

		bip->bip_flags |= BIP_MAPPED_INTEGRITY;
	}
}

/**
 * t10_pi_type1_complete - prepare PI prior returning request to the blk layer
 * @rq:              request with PI that should be prepared
 * @nr_bytes:        total bytes to prepare
 *
 * For Type 1/Type 2, the virtual start sector is the one that was
 * originally submitted by the block layer for the ref_tag usage. Due to
 * partitioning, MD/DM cloning, etc. the actual physical start sector is
 * likely to be different. Since the physical start sector was submitted
 * to the device, we should remap it back to virtual values expected by the
 * block layer.
 */
static void t10_pi_type1_complete(struct request *rq, unsigned int nr_bytes)
{
	unsigned intervals = nr_bytes >> rq->q->integrity.interval_exp;
	const int tuple_sz = rq->q->integrity.tuple_size;
	u32 ref_tag = t10_pi_ref_tag(rq);
	struct bio *bio;

	__rq_for_each_bio(bio, rq) {
		struct bio_integrity_payload *bip = bio_integrity(bio);
		u32 virt = bip_get_seed(bip) & 0xffffffff;
		struct bio_vec iv;
		struct bvec_iter iter;

		bip_for_each_vec(iv, bip, iter) {
			unsigned int j;
			void *p;

			p = bvec_kmap_local(&iv);
			for (j = 0; j < iv.bv_len && intervals; j += tuple_sz) {
				struct t10_pi_tuple *pi = p;

				if (be32_to_cpu(pi->ref_tag) == ref_tag)
					pi->ref_tag = cpu_to_be32(virt);
				virt++;
				ref_tag++;
				intervals--;
				p += tuple_sz;
			}
			kunmap_local(p);
		}
	}
}

static blk_status_t t10_pi_type3_generate_crc(struct blk_integrity_iter *iter)
{
	return t10_pi_generate(iter, t10_pi_crc_fn, T10_PI_TYPE3_PROTECTION);
}

static blk_status_t t10_pi_type3_generate_ip(struct blk_integrity_iter *iter)
{
	return t10_pi_generate(iter, t10_pi_ip_fn, T10_PI_TYPE3_PROTECTION);
}

static blk_status_t t10_pi_type3_verify_crc(struct blk_integrity_iter *iter)
{
	return t10_pi_verify(iter, t10_pi_crc_fn, T10_PI_TYPE3_PROTECTION);
}

static blk_status_t t10_pi_type3_verify_ip(struct blk_integrity_iter *iter)
{
	return t10_pi_verify(iter, t10_pi_ip_fn, T10_PI_TYPE3_PROTECTION);
}

/* Type 3 does not have a reference tag so no remapping is required. */
static void t10_pi_type3_prepare(struct request *rq)
{
}

/* Type 3 does not have a reference tag so no remapping is required. */
static void t10_pi_type3_complete(struct request *rq, unsigned int nr_bytes)
{
}

const struct blk_integrity_profile t10_pi_type1_crc = {
	.name			= "T10-DIF-TYPE1-CRC",
	.generate_fn		= t10_pi_type1_generate_crc,
	.verify_fn		= t10_pi_type1_verify_crc,
	.prepare_fn		= t10_pi_type1_prepare,
	.complete_fn		= t10_pi_type1_complete,
};
EXPORT_SYMBOL(t10_pi_type1_crc);

const struct blk_integrity_profile t10_pi_type1_ip = {
	.name			= "T10-DIF-TYPE1-IP",
	.generate_fn		= t10_pi_type1_generate_ip,
	.verify_fn		= t10_pi_type1_verify_ip,
	.prepare_fn		= t10_pi_type1_prepare,
	.complete_fn		= t10_pi_type1_complete,
};
EXPORT_SYMBOL(t10_pi_type1_ip);

const struct blk_integrity_profile t10_pi_type3_crc = {
	.name			= "T10-DIF-TYPE3-CRC",
	.generate_fn		= t10_pi_type3_generate_crc,
	.verify_fn		= t10_pi_type3_verify_crc,
	.prepare_fn		= t10_pi_type3_prepare,
	.complete_fn		= t10_pi_type3_complete,
};
EXPORT_SYMBOL(t10_pi_type3_crc);

const struct blk_integrity_profile t10_pi_type3_ip = {
	.name			= "T10-DIF-TYPE3-IP",
	.generate_fn		= t10_pi_type3_generate_ip,
	.verify_fn		= t10_pi_type3_verify_ip,
	.prepare_fn		= t10_pi_type3_prepare,
	.complete_fn		= t10_pi_type3_complete,
};
EXPORT_SYMBOL(t10_pi_type3_ip);

static __be64 ext_pi_crc64(void *data, unsigned int len)
{
	return cpu_to_be64(crc64_rocksoft(data, len));
}

static blk_status_t ext_pi_crc64_generate(struct blk_integrity_iter *iter,
					enum t10_dif_type type)
{
	unsigned int i;

	for (i = 0 ; i < iter->data_size ; i += iter->interval) {
		struct crc64_pi_tuple *pi = iter->prot_buf;

		pi->guard_tag = ext_pi_crc64(iter->data_buf, iter->interval);
		pi->app_tag = 0;

		if (type == T10_PI_TYPE1_PROTECTION)
			put_unaligned_be48(iter->seed, pi->ref_tag);
		else
			put_unaligned_be48(0ULL, pi->ref_tag);

		iter->data_buf += iter->interval;
		iter->prot_buf += iter->tuple_size;
		iter->seed++;
	}

	return BLK_STS_OK;
}

static bool ext_pi_ref_escape(u8 *ref_tag)
{
	static u8 ref_escape[6] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff };

	return memcmp(ref_tag, ref_escape, sizeof(ref_escape)) == 0;
}

static blk_status_t ext_pi_crc64_verify(struct blk_integrity_iter *iter,
				      enum t10_dif_type type)
{
	unsigned int i;

	for (i = 0; i < iter->data_size; i += iter->interval) {
		struct crc64_pi_tuple *pi = iter->prot_buf;
		u64 ref, seed;
		__be64 csum;

		if (type == T10_PI_TYPE1_PROTECTION) {
			if (pi->app_tag == T10_PI_APP_ESCAPE)
				goto next;

			ref = get_unaligned_be48(pi->ref_tag);
			seed = lower_48_bits(iter->seed);
			if (ref != seed) {
				pr_err("%s: ref tag error at location %llu (rcvd %llu)\n",
					iter->disk_name, seed, ref);
				return BLK_STS_PROTECTION;
			}
		} else if (type == T10_PI_TYPE3_PROTECTION) {
			if (pi->app_tag == T10_PI_APP_ESCAPE &&
			    ext_pi_ref_escape(pi->ref_tag))
				goto next;
		}

		csum = ext_pi_crc64(iter->data_buf, iter->interval);
		if (pi->guard_tag != csum) {
			pr_err("%s: guard tag error at sector %llu " \
			       "(rcvd %016llx, want %016llx)\n",
				iter->disk_name, (unsigned long long)iter->seed,
				be64_to_cpu(pi->guard_tag), be64_to_cpu(csum));
			return BLK_STS_PROTECTION;
		}

next:
		iter->data_buf += iter->interval;
		iter->prot_buf += iter->tuple_size;
		iter->seed++;
	}

	return BLK_STS_OK;
}

static blk_status_t ext_pi_type1_verify_crc64(struct blk_integrity_iter *iter)
{
	return ext_pi_crc64_verify(iter, T10_PI_TYPE1_PROTECTION);
}

static blk_status_t ext_pi_type1_generate_crc64(struct blk_integrity_iter *iter)
{
	return ext_pi_crc64_generate(iter, T10_PI_TYPE1_PROTECTION);
}

static void ext_pi_type1_prepare(struct request *rq)
{
	const int tuple_sz = rq->q->integrity.tuple_size;
	u64 ref_tag = ext_pi_ref_tag(rq);
	struct bio *bio;

	__rq_for_each_bio(bio, rq) {
		struct bio_integrity_payload *bip = bio_integrity(bio);
		u64 virt = lower_48_bits(bip_get_seed(bip));
		struct bio_vec iv;
		struct bvec_iter iter;

		/* Already remapped? */
		if (bip->bip_flags & BIP_MAPPED_INTEGRITY)
			break;

		bip_for_each_vec(iv, bip, iter) {
			unsigned int j;
			void *p;

			p = bvec_kmap_local(&iv);
			for (j = 0; j < iv.bv_len; j += tuple_sz) {
				struct crc64_pi_tuple *pi = p;
				u64 ref = get_unaligned_be48(pi->ref_tag);

				if (ref == virt)
					put_unaligned_be48(ref_tag, pi->ref_tag);
				virt++;
				ref_tag++;
				p += tuple_sz;
			}
			kunmap_local(p);
		}

		bip->bip_flags |= BIP_MAPPED_INTEGRITY;
	}
}

static void ext_pi_type1_complete(struct request *rq, unsigned int nr_bytes)
{
	unsigned intervals = nr_bytes >> rq->q->integrity.interval_exp;
	const int tuple_sz = rq->q->integrity.tuple_size;
	u64 ref_tag = ext_pi_ref_tag(rq);
	struct bio *bio;

	__rq_for_each_bio(bio, rq) {
		struct bio_integrity_payload *bip = bio_integrity(bio);
		u64 virt = lower_48_bits(bip_get_seed(bip));
		struct bio_vec iv;
		struct bvec_iter iter;

		bip_for_each_vec(iv, bip, iter) {
			unsigned int j;
			void *p;

			p = bvec_kmap_local(&iv);
			for (j = 0; j < iv.bv_len && intervals; j += tuple_sz) {
				struct crc64_pi_tuple *pi = p;
				u64 ref = get_unaligned_be48(pi->ref_tag);

				if (ref == ref_tag)
					put_unaligned_be48(virt, pi->ref_tag);
				virt++;
				ref_tag++;
				intervals--;
				p += tuple_sz;
			}
			kunmap_local(p);
		}
	}
}

static blk_status_t ext_pi_type3_verify_crc64(struct blk_integrity_iter *iter)
{
	return ext_pi_crc64_verify(iter, T10_PI_TYPE3_PROTECTION);
}

static blk_status_t ext_pi_type3_generate_crc64(struct blk_integrity_iter *iter)
{
	return ext_pi_crc64_generate(iter, T10_PI_TYPE3_PROTECTION);
}

const struct blk_integrity_profile ext_pi_type1_crc64 = {
	.name			= "EXT-DIF-TYPE1-CRC64",
	.generate_fn		= ext_pi_type1_generate_crc64,
	.verify_fn		= ext_pi_type1_verify_crc64,
	.prepare_fn		= ext_pi_type1_prepare,
	.complete_fn		= ext_pi_type1_complete,
};
EXPORT_SYMBOL_GPL(ext_pi_type1_crc64);

const struct blk_integrity_profile ext_pi_type3_crc64 = {
	.name			= "EXT-DIF-TYPE3-CRC64",
	.generate_fn		= ext_pi_type3_generate_crc64,
	.verify_fn		= ext_pi_type3_verify_crc64,
	.prepare_fn		= t10_pi_type3_prepare,
	.complete_fn		= t10_pi_type3_complete,
};
EXPORT_SYMBOL_GPL(ext_pi_type3_crc64);

MODULE_LICENSE("GPL");
MODULE_LICENSE("GPL");
Commit	Line	Data
8c16567d	1	// SPDX-License-Identifier: GPL-2.0
2341c2f8 MP	2	/*
	3	* t10_pi.c - Functions for generating and verifying T10 Protection
	4	* Information.
2341c2f8 MP	5	*/
	6
	7	#include <linux/t10-pi.h>
fe45e630	8	#include <linux/blk-integrity.h>
2341c2f8	9	#include <linux/crc-t10dif.h>
a7d4383f	10	#include <linux/crc64.h>
a754bd5f	11	#include <linux/module.h>
2341c2f8	12	#include <net/checksum.h>
a7d4383f	13	#include <asm/unaligned.h>
2341c2f8 MP	14
	15	typedef __be16 (csum_fn) (void *, unsigned int);
	16
2341c2f8 MP	17	static __be16 t10_pi_crc_fn(void *data, unsigned int len)
	18	{
	19	return cpu_to_be16(crc_t10dif(data, len));
	20	}
	21
	22	static __be16 t10_pi_ip_fn(void *data, unsigned int len)
	23	{
	24	return (__force __be16)ip_compute_csum(data, len);
	25	}
	26
	27	/*
	28	* Type 1 and Type 2 protection use the same format: 16 bit guard tag,
	29	* 16 bit app tag, 32 bit reference tag. Type 3 does not define the ref
	30	* tag.
	31	*/
4e4cbee9	32	static blk_status_t t10_pi_generate(struct blk_integrity_iter *iter,
5eaed68d	33	csum_fn *fn, enum t10_dif_type type)
2341c2f8 MP	34	{
	35	unsigned int i;
	36
	37	for (i = 0 ; i < iter->data_size ; i += iter->interval) {
	38	struct t10_pi_tuple *pi = iter->prot_buf;
	39
	40	pi->guard_tag = fn(iter->data_buf, iter->interval);
	41	pi->app_tag = 0;
	42
5eaed68d	43	if (type == T10_PI_TYPE1_PROTECTION)
2341c2f8 MP	44	pi->ref_tag = cpu_to_be32(lower_32_bits(iter->seed));
	45	else
	46	pi->ref_tag = 0;
	47
	48	iter->data_buf += iter->interval;
c340b990	49	iter->prot_buf += iter->tuple_size;
2341c2f8 MP	50	iter->seed++;
	51	}
	52
4e4cbee9	53	return BLK_STS_OK;
2341c2f8 MP	54	}
2341c2f8 MP	55
4e4cbee9	56	static blk_status_t t10_pi_verify(struct blk_integrity_iter *iter,
5eaed68d	57	csum_fn *fn, enum t10_dif_type type)
2341c2f8 MP	58	{
	59	unsigned int i;
	60
be21683e MG	61	BUG_ON(type == T10_PI_TYPE0_PROTECTION);
be21683e MG	62
2341c2f8 MP	63	for (i = 0 ; i < iter->data_size ; i += iter->interval) {
	64	struct t10_pi_tuple *pi = iter->prot_buf;
	65	__be16 csum;
	66
be21683e MG	67	if (type == T10_PI_TYPE1_PROTECTION \|\|
be21683e MG	68	type == T10_PI_TYPE2_PROTECTION) {
128b6f9f	69	if (pi->app_tag == T10_PI_APP_ESCAPE)
2341c2f8 MP	70	goto next;
	71
	72	if (be32_to_cpu(pi->ref_tag) !=
	73	lower_32_bits(iter->seed)) {
	74	pr_err("%s: ref tag error at location %llu " \
	75	"(rcvd %u)\n", iter->disk_name,
	76	(unsigned long long)
	77	iter->seed, be32_to_cpu(pi->ref_tag));
a462b950	78	return BLK_STS_PROTECTION;
2341c2f8	79	}
be21683e	80	} else if (type == T10_PI_TYPE3_PROTECTION) {
128b6f9f DM	81	if (pi->app_tag == T10_PI_APP_ESCAPE &&
128b6f9f DM	82	pi->ref_tag == T10_PI_REF_ESCAPE)
2341c2f8	83	goto next;
2341c2f8 MP	84	}
	85
	86	csum = fn(iter->data_buf, iter->interval);
	87
	88	if (pi->guard_tag != csum) {
	89	pr_err("%s: guard tag error at sector %llu " \
	90	"(rcvd %04x, want %04x)\n", iter->disk_name,
	91	(unsigned long long)iter->seed,
	92	be16_to_cpu(pi->guard_tag), be16_to_cpu(csum));
4e4cbee9	93	return BLK_STS_PROTECTION;
2341c2f8 MP	94	}
	95
	96	next:
	97	iter->data_buf += iter->interval;
c340b990	98	iter->prot_buf += iter->tuple_size;
2341c2f8 MP	99	iter->seed++;
	100	}
	101
4e4cbee9	102	return BLK_STS_OK;
2341c2f8 MP	103	}
2341c2f8 MP	104
4e4cbee9	105	static blk_status_t t10_pi_type1_generate_crc(struct blk_integrity_iter *iter)
2341c2f8	106	{
5eaed68d	107	return t10_pi_generate(iter, t10_pi_crc_fn, T10_PI_TYPE1_PROTECTION);
2341c2f8 MP	108	}
2341c2f8 MP	109
4e4cbee9	110	static blk_status_t t10_pi_type1_generate_ip(struct blk_integrity_iter *iter)
2341c2f8	111	{
5eaed68d	112	return t10_pi_generate(iter, t10_pi_ip_fn, T10_PI_TYPE1_PROTECTION);
2341c2f8 MP	113	}
2341c2f8 MP	114
4e4cbee9	115	static blk_status_t t10_pi_type1_verify_crc(struct blk_integrity_iter *iter)
2341c2f8	116	{
5eaed68d	117	return t10_pi_verify(iter, t10_pi_crc_fn, T10_PI_TYPE1_PROTECTION);
2341c2f8 MP	118	}
2341c2f8 MP	119
4e4cbee9	120	static blk_status_t t10_pi_type1_verify_ip(struct blk_integrity_iter *iter)
2341c2f8	121	{
5eaed68d	122	return t10_pi_verify(iter, t10_pi_ip_fn, T10_PI_TYPE1_PROTECTION);
2341c2f8 MP	123	}
2341c2f8 MP	124
10c41ddd	125	/**
54d4e6ab	126	* t10_pi_type1_prepare - prepare PI prior submitting request to device
10c41ddd	127	* @rq: request with PI that should be prepared
10c41ddd MG	128	*
	129	* For Type 1/Type 2, the virtual start sector is the one that was
	130	* originally submitted by the block layer for the ref_tag usage. Due to
	131	* partitioning, MD/DM cloning, etc. the actual physical start sector is
	132	* likely to be different. Remap protection information to match the
	133	* physical LBA.
10c41ddd	134	*/
54d4e6ab	135	static void t10_pi_type1_prepare(struct request *rq)
10c41ddd MG	136	{
	137	const int tuple_sz = rq->q->integrity.tuple_size;
	138	u32 ref_tag = t10_pi_ref_tag(rq);
	139	struct bio *bio;
	140
10c41ddd MG	141	__rq_for_each_bio(bio, rq) {
	142	struct bio_integrity_payload *bip = bio_integrity(bio);
	143	u32 virt = bip_get_seed(bip) & 0xffffffff;
	144	struct bio_vec iv;
	145	struct bvec_iter iter;
	146
	147	/* Already remapped? */
	148	if (bip->bip_flags & BIP_MAPPED_INTEGRITY)
	149	break;
	150
	151	bip_for_each_vec(iv, bip, iter) {
10c41ddd	152	unsigned int j;
8aec120a	153	void *p;
10c41ddd	154
8aec120a	155	p = bvec_kmap_local(&iv);
10c41ddd MG	156	for (j = 0; j < iv.bv_len; j += tuple_sz) {
	157	struct t10_pi_tuple *pi = p;
	158
	159	if (be32_to_cpu(pi->ref_tag) == virt)
	160	pi->ref_tag = cpu_to_be32(ref_tag);
	161	virt++;
	162	ref_tag++;
	163	p += tuple_sz;
	164	}
8aec120a	165	kunmap_local(p);
10c41ddd MG	166	}
	167
	168	bip->bip_flags \|= BIP_MAPPED_INTEGRITY;
	169	}
	170	}
10c41ddd MG	171
10c41ddd MG	172	/**
54d4e6ab	173	* t10_pi_type1_complete - prepare PI prior returning request to the blk layer
10c41ddd	174	* @rq: request with PI that should be prepared
54d4e6ab	175	* @nr_bytes: total bytes to prepare
10c41ddd MG	176	*
	177	* For Type 1/Type 2, the virtual start sector is the one that was
	178	* originally submitted by the block layer for the ref_tag usage. Due to
	179	* partitioning, MD/DM cloning, etc. the actual physical start sector is
	180	* likely to be different. Since the physical start sector was submitted
	181	* to the device, we should remap it back to virtual values expected by the
	182	* block layer.
10c41ddd	183	*/
54d4e6ab	184	static void t10_pi_type1_complete(struct request *rq, unsigned int nr_bytes)
10c41ddd	185	{
54d4e6ab	186	unsigned intervals = nr_bytes >> rq->q->integrity.interval_exp;
10c41ddd MG	187	const int tuple_sz = rq->q->integrity.tuple_size;
	188	u32 ref_tag = t10_pi_ref_tag(rq);
	189	struct bio *bio;
	190
10c41ddd MG	191	__rq_for_each_bio(bio, rq) {
	192	struct bio_integrity_payload *bip = bio_integrity(bio);
	193	u32 virt = bip_get_seed(bip) & 0xffffffff;
	194	struct bio_vec iv;
	195	struct bvec_iter iter;
	196
	197	bip_for_each_vec(iv, bip, iter) {
10c41ddd	198	unsigned int j;
8aec120a	199	void *p;
10c41ddd	200
8aec120a	201	p = bvec_kmap_local(&iv);
10c41ddd MG	202	for (j = 0; j < iv.bv_len && intervals; j += tuple_sz) {
	203	struct t10_pi_tuple *pi = p;
	204
	205	if (be32_to_cpu(pi->ref_tag) == ref_tag)
	206	pi->ref_tag = cpu_to_be32(virt);
	207	virt++;
	208	ref_tag++;
	209	intervals--;
	210	p += tuple_sz;
	211	}
8aec120a	212	kunmap_local(p);
10c41ddd MG	213	}
	214	}
	215	}
54d4e6ab MG	216
	217	static blk_status_t t10_pi_type3_generate_crc(struct blk_integrity_iter *iter)
	218	{
	219	return t10_pi_generate(iter, t10_pi_crc_fn, T10_PI_TYPE3_PROTECTION);
	220	}
	221
	222	static blk_status_t t10_pi_type3_generate_ip(struct blk_integrity_iter *iter)
	223	{
	224	return t10_pi_generate(iter, t10_pi_ip_fn, T10_PI_TYPE3_PROTECTION);
	225	}
	226
	227	static blk_status_t t10_pi_type3_verify_crc(struct blk_integrity_iter *iter)
	228	{
	229	return t10_pi_verify(iter, t10_pi_crc_fn, T10_PI_TYPE3_PROTECTION);
	230	}
	231
	232	static blk_status_t t10_pi_type3_verify_ip(struct blk_integrity_iter *iter)
	233	{
	234	return t10_pi_verify(iter, t10_pi_ip_fn, T10_PI_TYPE3_PROTECTION);
	235	}
	236
98e54402	237	/* Type 3 does not have a reference tag so no remapping is required. */
54d4e6ab MG	238	static void t10_pi_type3_prepare(struct request *rq)
	239	{
	240	}
	241
98e54402	242	/* Type 3 does not have a reference tag so no remapping is required. */
54d4e6ab MG	243	static void t10_pi_type3_complete(struct request *rq, unsigned int nr_bytes)
	244	{
	245	}
	246
	247	const struct blk_integrity_profile t10_pi_type1_crc = {
	248	.name = "T10-DIF-TYPE1-CRC",
	249	.generate_fn = t10_pi_type1_generate_crc,
	250	.verify_fn = t10_pi_type1_verify_crc,
	251	.prepare_fn = t10_pi_type1_prepare,
	252	.complete_fn = t10_pi_type1_complete,
	253	};
	254	EXPORT_SYMBOL(t10_pi_type1_crc);
	255
	256	const struct blk_integrity_profile t10_pi_type1_ip = {
	257	.name = "T10-DIF-TYPE1-IP",
	258	.generate_fn = t10_pi_type1_generate_ip,
	259	.verify_fn = t10_pi_type1_verify_ip,
	260	.prepare_fn = t10_pi_type1_prepare,
	261	.complete_fn = t10_pi_type1_complete,
	262	};
	263	EXPORT_SYMBOL(t10_pi_type1_ip);
	264
	265	const struct blk_integrity_profile t10_pi_type3_crc = {
	266	.name = "T10-DIF-TYPE3-CRC",
	267	.generate_fn = t10_pi_type3_generate_crc,
	268	.verify_fn = t10_pi_type3_verify_crc,
	269	.prepare_fn = t10_pi_type3_prepare,
	270	.complete_fn = t10_pi_type3_complete,
	271	};
	272	EXPORT_SYMBOL(t10_pi_type3_crc);
	273
	274	const struct blk_integrity_profile t10_pi_type3_ip = {
	275	.name = "T10-DIF-TYPE3-IP",
	276	.generate_fn = t10_pi_type3_generate_ip,
	277	.verify_fn = t10_pi_type3_verify_ip,
	278	.prepare_fn = t10_pi_type3_prepare,
	279	.complete_fn = t10_pi_type3_complete,
	280	};
	281	EXPORT_SYMBOL(t10_pi_type3_ip);
a754bd5f	282
a7d4383f KB	283	static __be64 ext_pi_crc64(void *data, unsigned int len)
	284	{
	285	return cpu_to_be64(crc64_rocksoft(data, len));
	286	}
	287
	288	static blk_status_t ext_pi_crc64_generate(struct blk_integrity_iter *iter,
	289	enum t10_dif_type type)
	290	{
	291	unsigned int i;
	292
	293	for (i = 0 ; i < iter->data_size ; i += iter->interval) {
	294	struct crc64_pi_tuple *pi = iter->prot_buf;
	295
	296	pi->guard_tag = ext_pi_crc64(iter->data_buf, iter->interval);
	297	pi->app_tag = 0;
	298
	299	if (type == T10_PI_TYPE1_PROTECTION)
	300	put_unaligned_be48(iter->seed, pi->ref_tag);
	301	else
	302	put_unaligned_be48(0ULL, pi->ref_tag);
	303
	304	iter->data_buf += iter->interval;
	305	iter->prot_buf += iter->tuple_size;
	306	iter->seed++;
	307	}
	308
	309	return BLK_STS_OK;
	310	}
	311
	312	static bool ext_pi_ref_escape(u8 *ref_tag)
	313	{
	314	static u8 ref_escape[6] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff };
	315
	316	return memcmp(ref_tag, ref_escape, sizeof(ref_escape)) == 0;
	317	}
	318
	319	static blk_status_t ext_pi_crc64_verify(struct blk_integrity_iter *iter,
	320	enum t10_dif_type type)
	321	{
	322	unsigned int i;
	323
	324	for (i = 0; i < iter->data_size; i += iter->interval) {
	325	struct crc64_pi_tuple *pi = iter->prot_buf;
	326	u64 ref, seed;
	327	__be64 csum;
	328
	329	if (type == T10_PI_TYPE1_PROTECTION) {
	330	if (pi->app_tag == T10_PI_APP_ESCAPE)
	331	goto next;
	332
	333	ref = get_unaligned_be48(pi->ref_tag);
	334	seed = lower_48_bits(iter->seed);
	335	if (ref != seed) {
	336	pr_err("%s: ref tag error at location %llu (rcvd %llu)\n",
	337	iter->disk_name, seed, ref);
	338	return BLK_STS_PROTECTION;
	339	}
	340	} else if (type == T10_PI_TYPE3_PROTECTION) {
	341	if (pi->app_tag == T10_PI_APP_ESCAPE &&
	342	ext_pi_ref_escape(pi->ref_tag))
	343	goto next;
	344	}
	345
	346	csum = ext_pi_crc64(iter->data_buf, iter->interval);
347	if (pi->guard_tag != csum) {
348	pr_err("%s: guard tag error at sector %llu " \
349	"(rcvd %016llx, want %016llx)\n",
350	iter->disk_name, (unsigned long long)iter->seed,
351	be64_to_cpu(pi->guard_tag), be64_to_cpu(csum));
352	return BLK_STS_PROTECTION;
353	}
354
355	next:
356	iter->data_buf += iter->interval;
357	iter->prot_buf += iter->tuple_size;
358	iter->seed++;
359	}
360
361	return BLK_STS_OK;
362	}
363
364	static blk_status_t ext_pi_type1_verify_crc64(struct blk_integrity_iter *iter)
365	{
366	return ext_pi_crc64_verify(iter, T10_PI_TYPE1_PROTECTION);
367	}
368
369	static blk_status_t ext_pi_type1_generate_crc64(struct blk_integrity_iter *iter)
370	{
371	return ext_pi_crc64_generate(iter, T10_PI_TYPE1_PROTECTION);
372	}
373
374	static void ext_pi_type1_prepare(struct request *rq)
375	{
376	const int tuple_sz = rq->q->integrity.tuple_size;
377	u64 ref_tag = ext_pi_ref_tag(rq);
378	struct bio *bio;
379
380	__rq_for_each_bio(bio, rq) {
381	struct bio_integrity_payload *bip = bio_integrity(bio);
382	u64 virt = lower_48_bits(bip_get_seed(bip));
383	struct bio_vec iv;
384	struct bvec_iter iter;
385
386	/* Already remapped? */
387	if (bip->bip_flags & BIP_MAPPED_INTEGRITY)
388	break;
389
390	bip_for_each_vec(iv, bip, iter) {
391	unsigned int j;
392	void *p;
393
394	p = bvec_kmap_local(&iv);
395	for (j = 0; j < iv.bv_len; j += tuple_sz) {
396	struct crc64_pi_tuple *pi = p;
397	u64 ref = get_unaligned_be48(pi->ref_tag);
398
399	if (ref == virt)
400	put_unaligned_be48(ref_tag, pi->ref_tag);
401	virt++;
402	ref_tag++;
403	p += tuple_sz;
404	}
405	kunmap_local(p);
406	}
407
408	bip->bip_flags \|= BIP_MAPPED_INTEGRITY;
409	}
410	}
411
412	static void ext_pi_type1_complete(struct request *rq, unsigned int nr_bytes)
413	{
414	unsigned intervals = nr_bytes >> rq->q->integrity.interval_exp;
415	const int tuple_sz = rq->q->integrity.tuple_size;
416	u64 ref_tag = ext_pi_ref_tag(rq);
417	struct bio *bio;
418
419	__rq_for_each_bio(bio, rq) {
420	struct bio_integrity_payload *bip = bio_integrity(bio);
421	u64 virt = lower_48_bits(bip_get_seed(bip));
422	struct bio_vec iv;
423	struct bvec_iter iter;
424
425	bip_for_each_vec(iv, bip, iter) {
426	unsigned int j;
427	void *p;
428
429	p = bvec_kmap_local(&iv);
430	for (j = 0; j < iv.bv_len && intervals; j += tuple_sz) {
431	struct crc64_pi_tuple *pi = p;
432	u64 ref = get_unaligned_be48(pi->ref_tag);
433
434	if (ref == ref_tag)
435	put_unaligned_be48(virt, pi->ref_tag);
436	virt++;
437	ref_tag++;
438	intervals--;
439	p += tuple_sz;
440	}
441	kunmap_local(p);
442	}
443	}
444	}
445
446	static blk_status_t ext_pi_type3_verify_crc64(struct blk_integrity_iter *iter)
447	{
448	return ext_pi_crc64_verify(iter, T10_PI_TYPE3_PROTECTION);
449	}
450
451	static blk_status_t ext_pi_type3_generate_crc64(struct blk_integrity_iter *iter)
452	{
453	return ext_pi_crc64_generate(iter, T10_PI_TYPE3_PROTECTION);
454	}
455
456	const struct blk_integrity_profile ext_pi_type1_crc64 = {
457	.name = "EXT-DIF-TYPE1-CRC64",
458	.generate_fn = ext_pi_type1_generate_crc64,
459	.verify_fn = ext_pi_type1_verify_crc64,
460	.prepare_fn = ext_pi_type1_prepare,
461	.complete_fn = ext_pi_type1_complete,
462	};
463	EXPORT_SYMBOL_GPL(ext_pi_type1_crc64);
464
465	const struct blk_integrity_profile ext_pi_type3_crc64 = {
466	.name = "EXT-DIF-TYPE3-CRC64",
467	.generate_fn = ext_pi_type3_generate_crc64,
468	.verify_fn = ext_pi_type3_verify_crc64,
469	.prepare_fn = t10_pi_type3_prepare,
470	.complete_fn = t10_pi_type3_complete,
471	};
472	EXPORT_SYMBOL_GPL(ext_pi_type3_crc64);
473
474	MODULE_LICENSE("GPL");
a754bd5f	475	MODULE_LICENSE("GPL");