crypto: hisilicon/sec - move backlog management to qp and store sqe in qp for callback

When multiple tfm use a same qp, the backlog data should be managed
centrally by the qp, rather than in the qp_ctx of each req.

Additionally, since SEC_BD_TYPE1 and SEC_BD_TYPE2 cannot use the
tag of the sqe to carry the virtual address of the req, the sent
sqe is stored in the qp. This allows the callback function to get
the req address. To handle the differences between hardware types,
the callback functions are split into two separate implementations.

Fixes: f0ae287c5045 ("crypto: hisilicon/sec2 - implement full backlog mode for sec")
Signed-off-by: Chenghai Huang <huangchenghai2@huawei.com>
Signed-off-by: Weili Qian <qianweili@huawei.com>
Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>

diff --git a/drivers/crypto/hisilicon/qm.c b/drivers/crypto/hisilicon/qm.c
index d47bf06a90f7d3b723f2b94c3a02679c7dd215a2..71071ac559d4bbf0d00f304550efc2dfdb7d15f2 100644 (file)
--- a/drivers/crypto/hisilicon/qm.c
+++ b/drivers/crypto/hisilicon/qm.c
@@ -2219,6 +2219,7 @@ static void qp_stop_fail_cb(struct hisi_qp *qp)
        for (i = 0; i < qp_used; i++) {
                pos = (i + cur_head) % sq_depth;
                qp->req_cb(qp, qp->sqe + (u32)(qm->sqe_size * pos));
+               qm_cq_head_update(qp);
                atomic_dec(&qp->qp_status.used);
        }
 }
@@ -2383,6 +2384,7 @@ int hisi_qp_send(struct hisi_qp *qp, const void *msg)
                return -EBUSY;
 
        memcpy(sqe, msg, qp->qm->sqe_size);
+       qp->msg[sq_tail] = msg;
 
        qm_db(qp->qm, qp->qp_id, QM_DOORBELL_CMD_SQ, sq_tail_next, 0);
        atomic_inc(&qp->qp_status.used);
@@ -2919,12 +2921,13 @@ EXPORT_SYMBOL_GPL(hisi_qm_wait_task_finish);
 static void hisi_qp_memory_uninit(struct hisi_qm *qm, int num)
 {
        struct device *dev = &qm->pdev->dev;
-       struct qm_dma *qdma;
+       struct hisi_qp *qp;
        int i;
 
        for (i = num - 1; i >= 0; i--) {
-               qdma = &qm->qp_array[i].qdma;
-               dma_free_coherent(dev, qdma->size, qdma->va, qdma->dma);
+               qp = &qm->qp_array[i];
+               dma_free_coherent(dev, qp->qdma.size, qp->qdma.va, qp->qdma.dma);
+               kfree(qp->msg);
                kfree(qm->poll_data[i].qp_finish_id);
        }
 
@@ -2946,10 +2949,14 @@ static int hisi_qp_memory_init(struct hisi_qm *qm, size_t dma_size, int id,
                return -ENOMEM;
 
        qp = &qm->qp_array[id];
+       qp->msg = kmalloc_array(sq_depth, sizeof(void *), GFP_KERNEL);
+       if (!qp->msg)
+               goto err_free_qp_finish_id;
+
        qp->qdma.va = dma_alloc_coherent(dev, dma_size, &qp->qdma.dma,
                                         GFP_KERNEL);
        if (!qp->qdma.va)
-               goto err_free_qp_finish_id;
+               goto err_free_qp_msg;
 
        qp->sqe = qp->qdma.va;
        qp->sqe_dma = qp->qdma.dma;
@@ -2961,8 +2968,13 @@ static int hisi_qp_memory_init(struct hisi_qm *qm, size_t dma_size, int id,
        qp->qm = qm;
        qp->qp_id = id;
 
+       spin_lock_init(&qp->backlog.lock);
+       INIT_LIST_HEAD(&qp->backlog.list);
+
        return 0;
 
+err_free_qp_msg:
+       kfree(qp->msg);
 err_free_qp_finish_id:
        kfree(qm->poll_data[id].qp_finish_id);
        return ret;

diff --git a/drivers/crypto/hisilicon/sec2/sec.h b/drivers/crypto/hisilicon/sec2/sec.h
index 81d0beda93b2bbbd3903bac7b257602a427de343..0710977861f32343f34b980db7f5ad7057c4360d 100644 (file)
--- a/drivers/crypto/hisilicon/sec2/sec.h
+++ b/drivers/crypto/hisilicon/sec2/sec.h
@@ -82,11 +82,6 @@ struct sec_aead_req {
        __u8 out_mac_buf[SEC_MAX_MAC_LEN];
 };
 
-struct sec_instance_backlog {
-       struct list_head list;
-       spinlock_t lock;
-};
-
 /* SEC request of Crypto */
 struct sec_req {
        union {
@@ -112,7 +107,6 @@ struct sec_req {
        bool use_pbuf;
 
        struct list_head list;
-       struct sec_instance_backlog *backlog;
        struct sec_request_buf buf;
 };
 
@@ -172,7 +166,6 @@ struct sec_qp_ctx {
        spinlock_t id_lock;
        struct hisi_acc_sgl_pool *c_in_pool;
        struct hisi_acc_sgl_pool *c_out_pool;
-       struct sec_instance_backlog backlog;
        u16 send_head;
 };
 

diff --git a/drivers/crypto/hisilicon/sec2/sec_crypto.c b/drivers/crypto/hisilicon/sec2/sec_crypto.c
index 31590d01139a37195f9cf653644bf394e07e2c83..4e41235116e15dc7b0ff6b5c46609c2181857501 100644 (file)
--- a/drivers/crypto/hisilicon/sec2/sec_crypto.c
+++ b/drivers/crypto/hisilicon/sec2/sec_crypto.c
@@ -54,7 +54,6 @@
 #define SEC_AUTH_CIPHER_V3     0x40
 #define SEC_FLAG_OFFSET                7
 #define SEC_FLAG_MASK          0x0780
-#define SEC_TYPE_MASK          0x0F
 #define SEC_DONE_MASK          0x0001
 #define SEC_ICV_MASK           0x000E
 
@@ -148,7 +147,7 @@ static void sec_free_req_id(struct sec_req *req)
        spin_unlock_bh(&qp_ctx->id_lock);
 }
 
-static u8 pre_parse_finished_bd(struct bd_status *status, void *resp)
+static void pre_parse_finished_bd(struct bd_status *status, void *resp)
 {
        struct sec_sqe *bd = resp;
 
@@ -158,11 +157,9 @@ static u8 pre_parse_finished_bd(struct bd_status *status, void *resp)
                                        SEC_FLAG_MASK) >> SEC_FLAG_OFFSET;
        status->tag = le16_to_cpu(bd->type2.tag);
        status->err_type = bd->type2.error_type;
-
-       return bd->type_cipher_auth & SEC_TYPE_MASK;
 }
 
-static u8 pre_parse_finished_bd3(struct bd_status *status, void *resp)
+static void pre_parse_finished_bd3(struct bd_status *status, void *resp)
 {
        struct sec_sqe3 *bd3 = resp;
 
@@ -172,8 +169,6 @@ static u8 pre_parse_finished_bd3(struct bd_status *status, void *resp)
                                        SEC_FLAG_MASK) >> SEC_FLAG_OFFSET;
        status->tag = le64_to_cpu(bd3->tag);
        status->err_type = bd3->error_type;
-
-       return le32_to_cpu(bd3->bd_param) & SEC_TYPE_MASK;
 }
 
 static int sec_cb_status_check(struct sec_req *req,
@@ -244,7 +239,7 @@ static void sec_alg_send_backlog_soft(struct sec_ctx *ctx, struct sec_qp_ctx *qp
        struct sec_req *req, *tmp;
        int ret;
 
-       list_for_each_entry_safe(req, tmp, &qp_ctx->backlog.list, list) {
+       list_for_each_entry_safe(req, tmp, &qp_ctx->qp->backlog.list, list) {
                list_del(&req->list);
                ctx->req_op->buf_unmap(ctx, req);
                if (req->req_id >= 0)
@@ -265,11 +260,12 @@ static void sec_alg_send_backlog_soft(struct sec_ctx *ctx, struct sec_qp_ctx *qp
 
 static void sec_alg_send_backlog(struct sec_ctx *ctx, struct sec_qp_ctx *qp_ctx)
 {
+       struct hisi_qp *qp = qp_ctx->qp;
        struct sec_req *req, *tmp;
        int ret;
 
-       spin_lock_bh(&qp_ctx->backlog.lock);
-       list_for_each_entry_safe(req, tmp, &qp_ctx->backlog.list, list) {
+       spin_lock_bh(&qp->backlog.lock);
+       list_for_each_entry_safe(req, tmp, &qp->backlog.list, list) {
                ret = qp_send_message(req);
                switch (ret) {
                case -EINPROGRESS:
@@ -287,42 +283,46 @@ static void sec_alg_send_backlog(struct sec_ctx *ctx, struct sec_qp_ctx *qp_ctx)
        }
 
 unlock:
-       spin_unlock_bh(&qp_ctx->backlog.lock);
+       spin_unlock_bh(&qp->backlog.lock);
 }
 
 static void sec_req_cb(struct hisi_qp *qp, void *resp)
 {
-       struct sec_qp_ctx *qp_ctx = qp->qp_ctx;
-       struct sec_dfx *dfx = &qp_ctx->ctx->sec->debug.dfx;
-       u8 type_supported = qp_ctx->ctx->type_supported;
+       const struct sec_sqe *sqe = qp->msg[qp->qp_status.cq_head];
+       struct sec_req *req = container_of(sqe, struct sec_req, sec_sqe);
+       struct sec_ctx *ctx = req->ctx;
+       struct sec_dfx *dfx = &ctx->sec->debug.dfx;
        struct bd_status status;
-       struct sec_ctx *ctx;
-       struct sec_req *req;
        int err;
-       u8 type;
 
-       if (type_supported == SEC_BD_TYPE2) {
-               type = pre_parse_finished_bd(&status, resp);
-               req = qp_ctx->req_list[status.tag];
-       } else {
-               type = pre_parse_finished_bd3(&status, resp);
-               req = (void *)(uintptr_t)status.tag;
-       }
+       pre_parse_finished_bd(&status, resp);
 
-       if (unlikely(type != type_supported)) {
-               atomic64_inc(&dfx->err_bd_cnt);
-               pr_err("err bd type [%u]\n", type);
-               return;
-       }
+       req->err_type = status.err_type;
+       err = sec_cb_status_check(req, &status);
+       if (err)
+               atomic64_inc(&dfx->done_flag_cnt);
 
-       if (unlikely(!req)) {
-               atomic64_inc(&dfx->invalid_req_cnt);
-               atomic_inc(&qp->qp_status.used);
-               return;
-       }
+       atomic64_inc(&dfx->recv_cnt);
 
+       ctx->req_op->buf_unmap(ctx, req);
+       ctx->req_op->callback(ctx, req, err);
+}
+
+static void sec_req_cb3(struct hisi_qp *qp, void *resp)
+{
+       struct bd_status status;
+       struct sec_ctx *ctx;
+       struct sec_dfx *dfx;
+       struct sec_req *req;
+       int err;
+
+       pre_parse_finished_bd3(&status, resp);
+
+       req = (void *)(uintptr_t)status.tag;
        req->err_type = status.err_type;
        ctx = req->ctx;
+       dfx = &ctx->sec->debug.dfx;
+
        err = sec_cb_status_check(req, &status);
        if (err)
                atomic64_inc(&dfx->done_flag_cnt);
@@ -330,7 +330,6 @@ static void sec_req_cb(struct hisi_qp *qp, void *resp)
        atomic64_inc(&dfx->recv_cnt);
 
        ctx->req_op->buf_unmap(ctx, req);
-
        ctx->req_op->callback(ctx, req, err);
 }
 
@@ -348,8 +347,10 @@ static int sec_alg_send_message_retry(struct sec_req *req)
 
 static int sec_alg_try_enqueue(struct sec_req *req)
 {
+       struct hisi_qp *qp = req->qp_ctx->qp;
+
        /* Check if any request is already backlogged */
-       if (!list_empty(&req->backlog->list))
+       if (!list_empty(&qp->backlog.list))
                return -EBUSY;
 
        /* Try to enqueue to HW ring */
@@ -359,17 +360,18 @@ static int sec_alg_try_enqueue(struct sec_req *req)
 
 static int sec_alg_send_message_maybacklog(struct sec_req *req)
 {
+       struct hisi_qp *qp = req->qp_ctx->qp;
        int ret;
 
        ret = sec_alg_try_enqueue(req);
        if (ret != -EBUSY)
                return ret;
 
-       spin_lock_bh(&req->backlog->lock);
+       spin_lock_bh(&qp->backlog.lock);
        ret = sec_alg_try_enqueue(req);
        if (ret == -EBUSY)
-               list_add_tail(&req->list, &req->backlog->list);
-       spin_unlock_bh(&req->backlog->lock);
+               list_add_tail(&req->list, &qp->backlog.list);
+       spin_unlock_bh(&qp->backlog.lock);
 
        return ret;
 }
@@ -629,13 +631,14 @@ static int sec_create_qp_ctx(struct sec_ctx *ctx, int qp_ctx_id)
        qp_ctx->qp = qp;
        qp_ctx->ctx = ctx;
 
-       qp->req_cb = sec_req_cb;
+       if (ctx->type_supported == SEC_BD_TYPE3)
+               qp->req_cb = sec_req_cb3;
+       else
+               qp->req_cb = sec_req_cb;
 
        spin_lock_init(&qp_ctx->req_lock);
        idr_init(&qp_ctx->req_idr);
-       spin_lock_init(&qp_ctx->backlog.lock);
        spin_lock_init(&qp_ctx->id_lock);
-       INIT_LIST_HEAD(&qp_ctx->backlog.list);
        qp_ctx->send_head = 0;
 
        ret = sec_alloc_qp_ctx_resource(ctx, qp_ctx);
@@ -1952,7 +1955,6 @@ static int sec_request_init(struct sec_ctx *ctx, struct sec_req *req)
        } while (req->req_id < 0 && ++i < ctx->sec->ctx_q_num);
 
        req->qp_ctx = qp_ctx;
-       req->backlog = &qp_ctx->backlog;
 
        return 0;
 }

diff --git a/include/linux/hisi_acc_qm.h b/include/linux/hisi_acc_qm.h
index ca1ec437a3ca3bfc829ef94326a4583df96544b8..9485896d5dc08d3b567b766d2d60ab60847be942 100644 (file)
--- a/include/linux/hisi_acc_qm.h
+++ b/include/linux/hisi_acc_qm.h
@@ -447,6 +447,11 @@ struct hisi_qp_ops {
        int (*fill_sqe)(void *sqe, void *q_parm, void *d_parm);
 };
 
+struct instance_backlog {
+       struct list_head list;
+       spinlock_t lock;
+};
+
 struct hisi_qp {
        u32 qp_id;
        u16 sq_depth;
@@ -471,6 +476,9 @@ struct hisi_qp {
        bool is_in_kernel;
        u16 pasid;
        struct uacce_queue *uacce_q;
+
+       struct instance_backlog backlog;
+       const void **msg;
 };
 
 static inline int vfs_num_set(const char *val, const struct kernel_param *kp)