[thirdparty/kernel/stable.git] / crypto / crypto_engine.c

// SPDX-License-Identifier: GPL-2.0-or-later
/*
 * Handle async block request by crypto hardware engine.
 *
 * Copyright (C) 2016 Linaro, Inc.
 *
 * Author: Baolin Wang <baolin.wang@linaro.org>
 */

#include <linux/err.h>
#include <linux/delay.h>
#include <crypto/engine.h>
#include <uapi/linux/sched/types.h>
#include "internal.h"

#define CRYPTO_ENGINE_MAX_QLEN 10

/**
 * crypto_finalize_request - finalize one request if the request is done
 * @engine: the hardware engine
 * @req: the request need to be finalized
 * @err: error number
 */
static void crypto_finalize_request(struct crypto_engine *engine,
			     struct crypto_async_request *req, int err)
{
	unsigned long flags;
	bool finalize_cur_req = false;
	int ret;
	struct crypto_engine_ctx *enginectx;

	spin_lock_irqsave(&engine->queue_lock, flags);
	if (engine->cur_req == req)
		finalize_cur_req = true;
	spin_unlock_irqrestore(&engine->queue_lock, flags);

	if (finalize_cur_req) {
		enginectx = crypto_tfm_ctx(req->tfm);
		if (engine->cur_req_prepared &&
		    enginectx->op.unprepare_request) {
			ret = enginectx->op.unprepare_request(engine, req);
			if (ret)
				dev_err(engine->dev, "failed to unprepare request\n");
		}
		spin_lock_irqsave(&engine->queue_lock, flags);
		engine->cur_req = NULL;
		engine->cur_req_prepared = false;
		spin_unlock_irqrestore(&engine->queue_lock, flags);
	}

	req->complete(req, err);

	kthread_queue_work(engine->kworker, &engine->pump_requests);
}

/**
 * crypto_pump_requests - dequeue one request from engine queue to process
 * @engine: the hardware engine
 * @in_kthread: true if we are in the context of the request pump thread
 *
 * This function checks if there is any request in the engine queue that
 * needs processing and if so call out to the driver to initialize hardware
 * and handle each request.
 */
static void crypto_pump_requests(struct crypto_engine *engine,
				 bool in_kthread)
{
	struct crypto_async_request *async_req, *backlog;
	unsigned long flags;
	bool was_busy = false;
	int ret;
	struct crypto_engine_ctx *enginectx;

	spin_lock_irqsave(&engine->queue_lock, flags);

	/* Make sure we are not already running a request */
	if (engine->cur_req)
		goto out;

	/* If another context is idling then defer */
	if (engine->idling) {
		kthread_queue_work(engine->kworker, &engine->pump_requests);
		goto out;
	}

	/* Check if the engine queue is idle */
	if (!crypto_queue_len(&engine->queue) || !engine->running) {
		if (!engine->busy)
			goto out;

		/* Only do teardown in the thread */
		if (!in_kthread) {
			kthread_queue_work(engine->kworker,
					   &engine->pump_requests);
			goto out;
		}

		engine->busy = false;
		engine->idling = true;
		spin_unlock_irqrestore(&engine->queue_lock, flags);

		if (engine->unprepare_crypt_hardware &&
		    engine->unprepare_crypt_hardware(engine))
			dev_err(engine->dev, "failed to unprepare crypt hardware\n");

		spin_lock_irqsave(&engine->queue_lock, flags);
		engine->idling = false;
		goto out;
	}

	/* Get the fist request from the engine queue to handle */
	backlog = crypto_get_backlog(&engine->queue);
	async_req = crypto_dequeue_request(&engine->queue);
	if (!async_req)
		goto out;

	engine->cur_req = async_req;
	if (backlog)
		backlog->complete(backlog, -EINPROGRESS);

	if (engine->busy)
		was_busy = true;
	else
		engine->busy = true;

	spin_unlock_irqrestore(&engine->queue_lock, flags);

	/* Until here we get the request need to be encrypted successfully */
	if (!was_busy && engine->prepare_crypt_hardware) {
		ret = engine->prepare_crypt_hardware(engine);
		if (ret) {
			dev_err(engine->dev, "failed to prepare crypt hardware\n");
			goto req_err;
		}
	}

	enginectx = crypto_tfm_ctx(async_req->tfm);

	if (enginectx->op.prepare_request) {
		ret = enginectx->op.prepare_request(engine, async_req);
		if (ret) {
			dev_err(engine->dev, "failed to prepare request: %d\n",
				ret);
			goto req_err;
		}
		engine->cur_req_prepared = true;
	}
	if (!enginectx->op.do_one_request) {
		dev_err(engine->dev, "failed to do request\n");
		ret = -EINVAL;
		goto req_err;
	}
	ret = enginectx->op.do_one_request(engine, async_req);
	if (ret) {
		dev_err(engine->dev, "Failed to do one request from queue: %d\n", ret);
		goto req_err;
	}
	return;

req_err:
	crypto_finalize_request(engine, async_req, ret);
	return;

out:
	spin_unlock_irqrestore(&engine->queue_lock, flags);
}

static void crypto_pump_work(struct kthread_work *work)
{
	struct crypto_engine *engine =
		container_of(work, struct crypto_engine, pump_requests);

	crypto_pump_requests(engine, true);
}

/**
 * crypto_transfer_request - transfer the new request into the engine queue
 * @engine: the hardware engine
 * @req: the request need to be listed into the engine queue
 */
static int crypto_transfer_request(struct crypto_engine *engine,
				   struct crypto_async_request *req,
				   bool need_pump)
{
	unsigned long flags;
	int ret;

	spin_lock_irqsave(&engine->queue_lock, flags);

	if (!engine->running) {
		spin_unlock_irqrestore(&engine->queue_lock, flags);
		return -ESHUTDOWN;
	}

	ret = crypto_enqueue_request(&engine->queue, req);

	if (!engine->busy && need_pump)
		kthread_queue_work(engine->kworker, &engine->pump_requests);

	spin_unlock_irqrestore(&engine->queue_lock, flags);
	return ret;
}

/**
 * crypto_transfer_request_to_engine - transfer one request to list
 * into the engine queue
 * @engine: the hardware engine
 * @req: the request need to be listed into the engine queue
 */
static int crypto_transfer_request_to_engine(struct crypto_engine *engine,
					     struct crypto_async_request *req)
{
	return crypto_transfer_request(engine, req, true);
}

/**
 * crypto_transfer_ablkcipher_request_to_engine - transfer one ablkcipher_request
 * to list into the engine queue
 * @engine: the hardware engine
 * @req: the request need to be listed into the engine queue
 * TODO: Remove this function when skcipher conversion is finished
 */
int crypto_transfer_ablkcipher_request_to_engine(struct crypto_engine *engine,
						 struct ablkcipher_request *req)
{
	return crypto_transfer_request_to_engine(engine, &req->base);
}
EXPORT_SYMBOL_GPL(crypto_transfer_ablkcipher_request_to_engine);

/**
 * crypto_transfer_aead_request_to_engine - transfer one aead_request
 * to list into the engine queue
 * @engine: the hardware engine
 * @req: the request need to be listed into the engine queue
 */
int crypto_transfer_aead_request_to_engine(struct crypto_engine *engine,
					   struct aead_request *req)
{
	return crypto_transfer_request_to_engine(engine, &req->base);
}
EXPORT_SYMBOL_GPL(crypto_transfer_aead_request_to_engine);

/**
 * crypto_transfer_akcipher_request_to_engine - transfer one akcipher_request
 * to list into the engine queue
 * @engine: the hardware engine
 * @req: the request need to be listed into the engine queue
 */
int crypto_transfer_akcipher_request_to_engine(struct crypto_engine *engine,
					       struct akcipher_request *req)
{
	return crypto_transfer_request_to_engine(engine, &req->base);
}
EXPORT_SYMBOL_GPL(crypto_transfer_akcipher_request_to_engine);

/**
 * crypto_transfer_hash_request_to_engine - transfer one ahash_request
 * to list into the engine queue
 * @engine: the hardware engine
 * @req: the request need to be listed into the engine queue
 */
int crypto_transfer_hash_request_to_engine(struct crypto_engine *engine,
					   struct ahash_request *req)
{
	return crypto_transfer_request_to_engine(engine, &req->base);
}
EXPORT_SYMBOL_GPL(crypto_transfer_hash_request_to_engine);

/**
 * crypto_transfer_skcipher_request_to_engine - transfer one skcipher_request
 * to list into the engine queue
 * @engine: the hardware engine
 * @req: the request need to be listed into the engine queue
 */
int crypto_transfer_skcipher_request_to_engine(struct crypto_engine *engine,
					       struct skcipher_request *req)
{
	return crypto_transfer_request_to_engine(engine, &req->base);
}
EXPORT_SYMBOL_GPL(crypto_transfer_skcipher_request_to_engine);

/**
 * crypto_finalize_ablkcipher_request - finalize one ablkcipher_request if
 * the request is done
 * @engine: the hardware engine
 * @req: the request need to be finalized
 * @err: error number
 * TODO: Remove this function when skcipher conversion is finished
 */
void crypto_finalize_ablkcipher_request(struct crypto_engine *engine,
					struct ablkcipher_request *req, int err)
{
	return crypto_finalize_request(engine, &req->base, err);
}
EXPORT_SYMBOL_GPL(crypto_finalize_ablkcipher_request);

/**
 * crypto_finalize_aead_request - finalize one aead_request if
 * the request is done
 * @engine: the hardware engine
 * @req: the request need to be finalized
 * @err: error number
 */
void crypto_finalize_aead_request(struct crypto_engine *engine,
				  struct aead_request *req, int err)
{
	return crypto_finalize_request(engine, &req->base, err);
}
EXPORT_SYMBOL_GPL(crypto_finalize_aead_request);

/**
 * crypto_finalize_akcipher_request - finalize one akcipher_request if
 * the request is done
 * @engine: the hardware engine
 * @req: the request need to be finalized
 * @err: error number
 */
void crypto_finalize_akcipher_request(struct crypto_engine *engine,
				      struct akcipher_request *req, int err)
{
	return crypto_finalize_request(engine, &req->base, err);
}
EXPORT_SYMBOL_GPL(crypto_finalize_akcipher_request);

/**
 * crypto_finalize_hash_request - finalize one ahash_request if
 * the request is done
 * @engine: the hardware engine
 * @req: the request need to be finalized
 * @err: error number
 */
void crypto_finalize_hash_request(struct crypto_engine *engine,
				  struct ahash_request *req, int err)
{
	return crypto_finalize_request(engine, &req->base, err);
}
EXPORT_SYMBOL_GPL(crypto_finalize_hash_request);

/**
 * crypto_finalize_skcipher_request - finalize one skcipher_request if
 * the request is done
 * @engine: the hardware engine
 * @req: the request need to be finalized
 * @err: error number
 */
void crypto_finalize_skcipher_request(struct crypto_engine *engine,
				      struct skcipher_request *req, int err)
{
	return crypto_finalize_request(engine, &req->base, err);
}
EXPORT_SYMBOL_GPL(crypto_finalize_skcipher_request);

/**
 * crypto_engine_start - start the hardware engine
 * @engine: the hardware engine need to be started
 *
 * Return 0 on success, else on fail.
 */
int crypto_engine_start(struct crypto_engine *engine)
{
	unsigned long flags;

	spin_lock_irqsave(&engine->queue_lock, flags);

	if (engine->running || engine->busy) {
		spin_unlock_irqrestore(&engine->queue_lock, flags);
		return -EBUSY;
	}

	engine->running = true;
	spin_unlock_irqrestore(&engine->queue_lock, flags);

	kthread_queue_work(engine->kworker, &engine->pump_requests);

	return 0;
}
EXPORT_SYMBOL_GPL(crypto_engine_start);

/**
 * crypto_engine_stop - stop the hardware engine
 * @engine: the hardware engine need to be stopped
 *
 * Return 0 on success, else on fail.
 */
int crypto_engine_stop(struct crypto_engine *engine)
{
	unsigned long flags;
	unsigned int limit = 500;
	int ret = 0;

	spin_lock_irqsave(&engine->queue_lock, flags);

	/*
	 * If the engine queue is not empty or the engine is on busy state,
	 * we need to wait for a while to pump the requests of engine queue.
	 */
	while ((crypto_queue_len(&engine->queue) || engine->busy) && limit--) {
		spin_unlock_irqrestore(&engine->queue_lock, flags);
		msleep(20);
		spin_lock_irqsave(&engine->queue_lock, flags);
	}

	if (crypto_queue_len(&engine->queue) || engine->busy)
		ret = -EBUSY;
	else
		engine->running = false;

	spin_unlock_irqrestore(&engine->queue_lock, flags);

	if (ret)
		dev_warn(engine->dev, "could not stop engine\n");

	return ret;
}
EXPORT_SYMBOL_GPL(crypto_engine_stop);

/**
 * crypto_engine_alloc_init - allocate crypto hardware engine structure and
 * initialize it.
 * @dev: the device attached with one hardware engine
 * @rt: whether this queue is set to run as a realtime task
 *
 * This must be called from context that can sleep.
 * Return: the crypto engine structure on success, else NULL.
 */
struct crypto_engine *crypto_engine_alloc_init(struct device *dev, bool rt)
{
	struct sched_param param = { .sched_priority = MAX_RT_PRIO - 1 };
	struct crypto_engine *engine;

	if (!dev)
		return NULL;

	engine = devm_kzalloc(dev, sizeof(*engine), GFP_KERNEL);
	if (!engine)
		return NULL;

	engine->dev = dev;
	engine->rt = rt;
	engine->running = false;
	engine->busy = false;
	engine->idling = false;
	engine->cur_req_prepared = false;
	engine->priv_data = dev;
	snprintf(engine->name, sizeof(engine->name),
		 "%s-engine", dev_name(dev));

	crypto_init_queue(&engine->queue, CRYPTO_ENGINE_MAX_QLEN);
	spin_lock_init(&engine->queue_lock);

	engine->kworker = kthread_create_worker(0, "%s", engine->name);
	if (IS_ERR(engine->kworker)) {
		dev_err(dev, "failed to create crypto request pump task\n");
		return NULL;
	}
	kthread_init_work(&engine->pump_requests, crypto_pump_work);

	if (engine->rt) {
		dev_info(dev, "will run requests pump with realtime priority\n");
		sched_setscheduler(engine->kworker->task, SCHED_FIFO, &param);
	}

	return engine;
}
EXPORT_SYMBOL_GPL(crypto_engine_alloc_init);

/**
 * crypto_engine_exit - free the resources of hardware engine when exit
 * @engine: the hardware engine need to be freed
 *
 * Return 0 for success.
 */
int crypto_engine_exit(struct crypto_engine *engine)
{
	int ret;

	ret = crypto_engine_stop(engine);
	if (ret)
		return ret;

	kthread_destroy_worker(engine->kworker);

	return 0;
}
EXPORT_SYMBOL_GPL(crypto_engine_exit);

MODULE_LICENSE("GPL");
MODULE_DESCRIPTION("Crypto hardware engine framework");
Commit	Line	Data
2874c5fd	1	// SPDX-License-Identifier: GPL-2.0-or-later
735d37b5 BW	2	/*
	3	* Handle async block request by crypto hardware engine.
	4	*
	5	* Copyright (C) 2016 Linaro, Inc.
	6	*
	7	* Author: Baolin Wang <baolin.wang@linaro.org>
735d37b5 BW	8	*/
	9
	10	#include <linux/err.h>
	11	#include <linux/delay.h>
2589ad84	12	#include <crypto/engine.h>
ae7e81c0	13	#include <uapi/linux/sched/types.h>
735d37b5 BW	14	#include "internal.h"
	15
	16	#define CRYPTO_ENGINE_MAX_QLEN 10
	17
218d1cc1 CL	18	/**
	19	* crypto_finalize_request - finalize one request if the request is done
	20	* @engine: the hardware engine
	21	* @req: the request need to be finalized
	22	* @err: error number
	23	*/
	24	static void crypto_finalize_request(struct crypto_engine *engine,
	25	struct crypto_async_request *req, int err)
	26	{
	27	unsigned long flags;
	28	bool finalize_cur_req = false;
	29	int ret;
	30	struct crypto_engine_ctx *enginectx;
	31
	32	spin_lock_irqsave(&engine->queue_lock, flags);
	33	if (engine->cur_req == req)
	34	finalize_cur_req = true;
	35	spin_unlock_irqrestore(&engine->queue_lock, flags);
	36
	37	if (finalize_cur_req) {
	38	enginectx = crypto_tfm_ctx(req->tfm);
	39	if (engine->cur_req_prepared &&
	40	enginectx->op.unprepare_request) {
	41	ret = enginectx->op.unprepare_request(engine, req);
	42	if (ret)
	43	dev_err(engine->dev, "failed to unprepare request\n");
	44	}
	45	spin_lock_irqsave(&engine->queue_lock, flags);
	46	engine->cur_req = NULL;
	47	engine->cur_req_prepared = false;
	48	spin_unlock_irqrestore(&engine->queue_lock, flags);
	49	}
	50
	51	req->complete(req, err);
	52
	53	kthread_queue_work(engine->kworker, &engine->pump_requests);
	54	}
	55
735d37b5 BW	56	/**
	57	* crypto_pump_requests - dequeue one request from engine queue to process
	58	* @engine: the hardware engine
	59	* @in_kthread: true if we are in the context of the request pump thread
	60	*
	61	* This function checks if there is any request in the engine queue that
	62	* needs processing and if so call out to the driver to initialize hardware
	63	* and handle each request.
	64	*/
	65	static void crypto_pump_requests(struct crypto_engine *engine,
	66	bool in_kthread)
	67	{
	68	struct crypto_async_request async_req, backlog;
735d37b5 BW	69	unsigned long flags;
735d37b5 BW	70	bool was_busy = false;
218d1cc1 CL	71	int ret;
218d1cc1 CL	72	struct crypto_engine_ctx *enginectx;
735d37b5 BW	73
	74	spin_lock_irqsave(&engine->queue_lock, flags);
	75
	76	/* Make sure we are not already running a request */
	77	if (engine->cur_req)
	78	goto out;
	79
	80	/* If another context is idling then defer */
	81	if (engine->idling) {
c4ca2b0b	82	kthread_queue_work(engine->kworker, &engine->pump_requests);
735d37b5 BW	83	goto out;
	84	}
	85
	86	/* Check if the engine queue is idle */
	87	if (!crypto_queue_len(&engine->queue) \|\| !engine->running) {
	88	if (!engine->busy)
	89	goto out;
	90
	91	/* Only do teardown in the thread */
	92	if (!in_kthread) {
c4ca2b0b	93	kthread_queue_work(engine->kworker,
735d37b5 BW	94	&engine->pump_requests);
	95	goto out;
	96	}
	97
	98	engine->busy = false;
	99	engine->idling = true;
	100	spin_unlock_irqrestore(&engine->queue_lock, flags);
	101
	102	if (engine->unprepare_crypt_hardware &&
	103	engine->unprepare_crypt_hardware(engine))
88d58ef8	104	dev_err(engine->dev, "failed to unprepare crypt hardware\n");
735d37b5 BW	105
	106	spin_lock_irqsave(&engine->queue_lock, flags);
	107	engine->idling = false;
	108	goto out;
	109	}
	110
	111	/* Get the fist request from the engine queue to handle */
	112	backlog = crypto_get_backlog(&engine->queue);
	113	async_req = crypto_dequeue_request(&engine->queue);
	114	if (!async_req)
	115	goto out;
	116
4cba7cf0	117	engine->cur_req = async_req;
735d37b5 BW	118	if (backlog)
	119	backlog->complete(backlog, -EINPROGRESS);
	120
	121	if (engine->busy)
	122	was_busy = true;
	123	else
	124	engine->busy = true;
	125
	126	spin_unlock_irqrestore(&engine->queue_lock, flags);
	127
	128	/* Until here we get the request need to be encrypted successfully */
	129	if (!was_busy && engine->prepare_crypt_hardware) {
	130	ret = engine->prepare_crypt_hardware(engine);
	131	if (ret) {
88d58ef8	132	dev_err(engine->dev, "failed to prepare crypt hardware\n");
735d37b5 BW	133	goto req_err;
	134	}
	135	}
	136
218d1cc1 CL	137	enginectx = crypto_tfm_ctx(async_req->tfm);
	138
	139	if (enginectx->op.prepare_request) {
	140	ret = enginectx->op.prepare_request(engine, async_req);
4cba7cf0	141	if (ret) {
218d1cc1 CL	142	dev_err(engine->dev, "failed to prepare request: %d\n",
218d1cc1 CL	143	ret);
4cba7cf0 CL	144	goto req_err;
4cba7cf0 CL	145	}
218d1cc1 CL	146	engine->cur_req_prepared = true;
	147	}
	148	if (!enginectx->op.do_one_request) {
	149	dev_err(engine->dev, "failed to do request\n");
	150	ret = -EINVAL;
	151	goto req_err;
735d37b5	152	}
218d1cc1 CL	153	ret = enginectx->op.do_one_request(engine, async_req);
	154	if (ret) {
	155	dev_err(engine->dev, "Failed to do one request from queue: %d\n", ret);
	156	goto req_err;
	157	}
	158	return;
735d37b5 BW	159
735d37b5 BW	160	req_err:
218d1cc1	161	crypto_finalize_request(engine, async_req, ret);
735d37b5 BW	162	return;
	163
	164	out:
	165	spin_unlock_irqrestore(&engine->queue_lock, flags);
	166	}
	167
	168	static void crypto_pump_work(struct kthread_work *work)
	169	{
	170	struct crypto_engine *engine =
	171	container_of(work, struct crypto_engine, pump_requests);
	172
	173	crypto_pump_requests(engine, true);
	174	}
	175
	176	/**
218d1cc1	177	* crypto_transfer_request - transfer the new request into the engine queue
735d37b5 BW	178	* @engine: the hardware engine
	179	* @req: the request need to be listed into the engine queue
	180	*/
218d1cc1 CL	181	static int crypto_transfer_request(struct crypto_engine *engine,
218d1cc1 CL	182	struct crypto_async_request *req,
4cba7cf0	183	bool need_pump)
735d37b5 BW	184	{
	185	unsigned long flags;
	186	int ret;
	187
	188	spin_lock_irqsave(&engine->queue_lock, flags);
	189
	190	if (!engine->running) {
	191	spin_unlock_irqrestore(&engine->queue_lock, flags);
	192	return -ESHUTDOWN;
	193	}
	194
218d1cc1	195	ret = crypto_enqueue_request(&engine->queue, req);
735d37b5 BW	196
735d37b5 BW	197	if (!engine->busy && need_pump)
c4ca2b0b	198	kthread_queue_work(engine->kworker, &engine->pump_requests);
735d37b5 BW	199
	200	spin_unlock_irqrestore(&engine->queue_lock, flags);
	201	return ret;
	202	}
4cba7cf0 CL	203
4cba7cf0 CL	204	/**
218d1cc1	205	* crypto_transfer_request_to_engine - transfer one request to list
4cba7cf0 CL	206	* into the engine queue
	207	* @engine: the hardware engine
	208	* @req: the request need to be listed into the engine queue
	209	*/
218d1cc1 CL	210	static int crypto_transfer_request_to_engine(struct crypto_engine *engine,
218d1cc1 CL	211	struct crypto_async_request *req)
4cba7cf0	212	{
218d1cc1	213	return crypto_transfer_request(engine, req, true);
4cba7cf0	214	}
4cba7cf0 CL	215
4cba7cf0 CL	216	/**
218d1cc1 CL	217	* crypto_transfer_ablkcipher_request_to_engine - transfer one ablkcipher_request
218d1cc1 CL	218	* to list into the engine queue
4cba7cf0 CL	219	* @engine: the hardware engine
4cba7cf0 CL	220	* @req: the request need to be listed into the engine queue
218d1cc1	221	* TODO: Remove this function when skcipher conversion is finished
4cba7cf0	222	*/
218d1cc1 CL	223	int crypto_transfer_ablkcipher_request_to_engine(struct crypto_engine *engine,
218d1cc1 CL	224	struct ablkcipher_request *req)
4cba7cf0	225	{
218d1cc1 CL	226	return crypto_transfer_request_to_engine(engine, &req->base);
	227	}
	228	EXPORT_SYMBOL_GPL(crypto_transfer_ablkcipher_request_to_engine);
4cba7cf0	229
218d1cc1 CL	230	/**
	231	* crypto_transfer_aead_request_to_engine - transfer one aead_request
	232	* to list into the engine queue
	233	* @engine: the hardware engine
	234	* @req: the request need to be listed into the engine queue
	235	*/
	236	int crypto_transfer_aead_request_to_engine(struct crypto_engine *engine,
	237	struct aead_request *req)
	238	{
	239	return crypto_transfer_request_to_engine(engine, &req->base);
	240	}
	241	EXPORT_SYMBOL_GPL(crypto_transfer_aead_request_to_engine);
4cba7cf0	242
218d1cc1 CL	243	/**
	244	* crypto_transfer_akcipher_request_to_engine - transfer one akcipher_request
	245	* to list into the engine queue
	246	* @engine: the hardware engine
	247	* @req: the request need to be listed into the engine queue
	248	*/
	249	int crypto_transfer_akcipher_request_to_engine(struct crypto_engine *engine,
	250	struct akcipher_request *req)
	251	{
	252	return crypto_transfer_request_to_engine(engine, &req->base);
4cba7cf0	253	}
218d1cc1	254	EXPORT_SYMBOL_GPL(crypto_transfer_akcipher_request_to_engine);
735d37b5 BW	255
735d37b5 BW	256	/**
218d1cc1 CL	257	* crypto_transfer_hash_request_to_engine - transfer one ahash_request
218d1cc1 CL	258	* to list into the engine queue
735d37b5 BW	259	* @engine: the hardware engine
	260	* @req: the request need to be listed into the engine queue
	261	*/
4cba7cf0 CL	262	int crypto_transfer_hash_request_to_engine(struct crypto_engine *engine,
4cba7cf0 CL	263	struct ahash_request *req)
735d37b5	264	{
218d1cc1	265	return crypto_transfer_request_to_engine(engine, &req->base);
735d37b5	266	}
4cba7cf0	267	EXPORT_SYMBOL_GPL(crypto_transfer_hash_request_to_engine);
735d37b5 BW	268
735d37b5 BW	269	/**
218d1cc1 CL	270	* crypto_transfer_skcipher_request_to_engine - transfer one skcipher_request
	271	* to list into the engine queue
	272	* @engine: the hardware engine
	273	* @req: the request need to be listed into the engine queue
	274	*/
	275	int crypto_transfer_skcipher_request_to_engine(struct crypto_engine *engine,
	276	struct skcipher_request *req)
	277	{
	278	return crypto_transfer_request_to_engine(engine, &req->base);
	279	}
	280	EXPORT_SYMBOL_GPL(crypto_transfer_skcipher_request_to_engine);
	281
	282	/**
	283	* crypto_finalize_ablkcipher_request - finalize one ablkcipher_request if
	284	* the request is done
735d37b5 BW	285	* @engine: the hardware engine
	286	* @req: the request need to be finalized
	287	* @err: error number
218d1cc1	288	* TODO: Remove this function when skcipher conversion is finished
735d37b5	289	*/
218d1cc1 CL	290	void crypto_finalize_ablkcipher_request(struct crypto_engine *engine,
218d1cc1 CL	291	struct ablkcipher_request *req, int err)
735d37b5	292	{
218d1cc1 CL	293	return crypto_finalize_request(engine, &req->base, err);
	294	}
	295	EXPORT_SYMBOL_GPL(crypto_finalize_ablkcipher_request);
4cba7cf0	296
218d1cc1 CL	297	/**
	298	* crypto_finalize_aead_request - finalize one aead_request if
	299	* the request is done
	300	* @engine: the hardware engine
	301	* @req: the request need to be finalized
	302	* @err: error number
	303	*/
	304	void crypto_finalize_aead_request(struct crypto_engine *engine,
	305	struct aead_request *req, int err)
	306	{
	307	return crypto_finalize_request(engine, &req->base, err);
	308	}
	309	EXPORT_SYMBOL_GPL(crypto_finalize_aead_request);
735d37b5	310
218d1cc1 CL	311	/**
	312	* crypto_finalize_akcipher_request - finalize one akcipher_request if
	313	* the request is done
	314	* @engine: the hardware engine
	315	* @req: the request need to be finalized
	316	* @err: error number
	317	*/
	318	void crypto_finalize_akcipher_request(struct crypto_engine *engine,
	319	struct akcipher_request *req, int err)
	320	{
	321	return crypto_finalize_request(engine, &req->base, err);
4cba7cf0	322	}
218d1cc1	323	EXPORT_SYMBOL_GPL(crypto_finalize_akcipher_request);
4cba7cf0 CL	324
4cba7cf0 CL	325	/**
218d1cc1 CL	326	* crypto_finalize_hash_request - finalize one ahash_request if
218d1cc1 CL	327	* the request is done
4cba7cf0 CL	328	* @engine: the hardware engine
	329	* @req: the request need to be finalized
	330	* @err: error number
	331	*/
	332	void crypto_finalize_hash_request(struct crypto_engine *engine,
	333	struct ahash_request *req, int err)
	334	{
218d1cc1	335	return crypto_finalize_request(engine, &req->base, err);
735d37b5	336	}
4cba7cf0	337	EXPORT_SYMBOL_GPL(crypto_finalize_hash_request);
735d37b5	338
218d1cc1 CL	339	/**
	340	* crypto_finalize_skcipher_request - finalize one skcipher_request if
	341	* the request is done
	342	* @engine: the hardware engine
	343	* @req: the request need to be finalized
	344	* @err: error number
	345	*/
	346	void crypto_finalize_skcipher_request(struct crypto_engine *engine,
	347	struct skcipher_request *req, int err)
	348	{
	349	return crypto_finalize_request(engine, &req->base, err);
	350	}
	351	EXPORT_SYMBOL_GPL(crypto_finalize_skcipher_request);
	352
735d37b5 BW	353	/**
	354	* crypto_engine_start - start the hardware engine
	355	* @engine: the hardware engine need to be started
	356	*
	357	* Return 0 on success, else on fail.
	358	*/
	359	int crypto_engine_start(struct crypto_engine *engine)
	360	{
	361	unsigned long flags;
	362
	363	spin_lock_irqsave(&engine->queue_lock, flags);
	364
	365	if (engine->running \|\| engine->busy) {
	366	spin_unlock_irqrestore(&engine->queue_lock, flags);
	367	return -EBUSY;
	368	}
	369
	370	engine->running = true;
	371	spin_unlock_irqrestore(&engine->queue_lock, flags);
	372
c4ca2b0b	373	kthread_queue_work(engine->kworker, &engine->pump_requests);
735d37b5 BW	374
	375	return 0;
	376	}
	377	EXPORT_SYMBOL_GPL(crypto_engine_start);
	378
	379	/**
	380	* crypto_engine_stop - stop the hardware engine
	381	* @engine: the hardware engine need to be stopped
	382	*
	383	* Return 0 on success, else on fail.
	384	*/
	385	int crypto_engine_stop(struct crypto_engine *engine)
	386	{
	387	unsigned long flags;
4cba7cf0	388	unsigned int limit = 500;
735d37b5 BW	389	int ret = 0;
	390
	391	spin_lock_irqsave(&engine->queue_lock, flags);
	392
	393	/*
	394	* If the engine queue is not empty or the engine is on busy state,
	395	* we need to wait for a while to pump the requests of engine queue.
	396	*/
	397	while ((crypto_queue_len(&engine->queue) \|\| engine->busy) && limit--) {
	398	spin_unlock_irqrestore(&engine->queue_lock, flags);
	399	msleep(20);
	400	spin_lock_irqsave(&engine->queue_lock, flags);
	401	}
	402
	403	if (crypto_queue_len(&engine->queue) \|\| engine->busy)
	404	ret = -EBUSY;
	405	else
	406	engine->running = false;
	407
	408	spin_unlock_irqrestore(&engine->queue_lock, flags);
	409
	410	if (ret)
88d58ef8	411	dev_warn(engine->dev, "could not stop engine\n");
735d37b5 BW	412
	413	return ret;
	414	}
	415	EXPORT_SYMBOL_GPL(crypto_engine_stop);
	416
	417	/**
	418	* crypto_engine_alloc_init - allocate crypto hardware engine structure and
	419	* initialize it.
	420	* @dev: the device attached with one hardware engine
	421	* @rt: whether this queue is set to run as a realtime task
	422	*
	423	* This must be called from context that can sleep.
	424	* Return: the crypto engine structure on success, else NULL.
	425	*/
	426	struct crypto_engine crypto_engine_alloc_init(struct device dev, bool rt)
	427	{
	428	struct sched_param param = { .sched_priority = MAX_RT_PRIO - 1 };
	429	struct crypto_engine *engine;
	430
	431	if (!dev)
	432	return NULL;
	433
	434	engine = devm_kzalloc(dev, sizeof(*engine), GFP_KERNEL);
	435	if (!engine)
	436	return NULL;
	437
88d58ef8	438	engine->dev = dev;
735d37b5 BW	439	engine->rt = rt;
	440	engine->running = false;
	441	engine->busy = false;
	442	engine->idling = false;
	443	engine->cur_req_prepared = false;
	444	engine->priv_data = dev;
	445	snprintf(engine->name, sizeof(engine->name),
	446	"%s-engine", dev_name(dev));
	447
	448	crypto_init_queue(&engine->queue, CRYPTO_ENGINE_MAX_QLEN);
	449	spin_lock_init(&engine->queue_lock);
	450
c4ca2b0b PM	451	engine->kworker = kthread_create_worker(0, "%s", engine->name);
c4ca2b0b PM	452	if (IS_ERR(engine->kworker)) {
735d37b5 BW	453	dev_err(dev, "failed to create crypto request pump task\n");
	454	return NULL;
	455	}
3989144f	456	kthread_init_work(&engine->pump_requests, crypto_pump_work);
735d37b5 BW	457
	458	if (engine->rt) {
	459	dev_info(dev, "will run requests pump with realtime priority\n");
c4ca2b0b	460	sched_setscheduler(engine->kworker->task, SCHED_FIFO, &param);
735d37b5 BW	461	}
	462
	463	return engine;
	464	}
	465	EXPORT_SYMBOL_GPL(crypto_engine_alloc_init);
	466
	467	/**
	468	* crypto_engine_exit - free the resources of hardware engine when exit
	469	* @engine: the hardware engine need to be freed
	470	*
	471	* Return 0 for success.
	472	*/
	473	int crypto_engine_exit(struct crypto_engine *engine)
	474	{
	475	int ret;
	476
	477	ret = crypto_engine_stop(engine);
	478	if (ret)
	479	return ret;
	480
c4ca2b0b	481	kthread_destroy_worker(engine->kworker);
735d37b5 BW	482
	483	return 0;
	484	}
	485	EXPORT_SYMBOL_GPL(crypto_engine_exit);
	486
	487	MODULE_LICENSE("GPL");
	488	MODULE_DESCRIPTION("Crypto hardware engine framework");