return sp->max_read_write_size;
}
-static int smb_direct_post_send_data(struct smbdirect_socket *sc,
- struct smbdirect_send_batch *send_ctx,
- struct iov_iter *iter,
- u32 remaining_data_length);
-
-static void smb_direct_send_immediate_work(struct work_struct *work)
-{
- struct smbdirect_socket *sc =
- container_of(work, struct smbdirect_socket, idle.immediate_work);
-
- if (sc->status != SMBDIRECT_SOCKET_CONNECTED)
- return;
-
- smb_direct_post_send_data(sc, NULL, NULL, 0);
-}
-
static struct smb_direct_transport *alloc_transport(struct rdma_cm_id *cm_id)
{
struct smb_direct_transport *t;
return ret;
}
-static int smb_direct_post_send(struct smbdirect_socket *sc,
- struct ib_send_wr *wr)
-{
- int ret;
-
- atomic_inc(&sc->send_io.pending.count);
- ret = ib_post_send(sc->ib.qp, wr, NULL);
- if (ret) {
- pr_err("failed to post send: %d\n", ret);
- smbdirect_socket_schedule_cleanup(sc, ret);
- }
- return ret;
-}
-
-static void smb_direct_send_ctx_init(struct smbdirect_send_batch *send_ctx,
- bool need_invalidate_rkey,
- unsigned int remote_key)
-{
- INIT_LIST_HEAD(&send_ctx->msg_list);
- send_ctx->wr_cnt = 0;
- send_ctx->need_invalidate_rkey = need_invalidate_rkey;
- send_ctx->remote_key = remote_key;
- send_ctx->credit = 0;
-}
-
-static int smb_direct_flush_send_list(struct smbdirect_socket *sc,
- struct smbdirect_send_batch *send_ctx,
- bool is_last)
-{
- struct smbdirect_send_io *first, *last;
- int ret = 0;
-
- if (list_empty(&send_ctx->msg_list))
- goto release_credit;
-
- first = list_first_entry(&send_ctx->msg_list,
- struct smbdirect_send_io,
- sibling_list);
- last = list_last_entry(&send_ctx->msg_list,
- struct smbdirect_send_io,
- sibling_list);
-
- if (send_ctx->need_invalidate_rkey) {
- first->wr.opcode = IB_WR_SEND_WITH_INV;
- first->wr.ex.invalidate_rkey = send_ctx->remote_key;
- send_ctx->need_invalidate_rkey = false;
- send_ctx->remote_key = 0;
- }
-
- last->wr.send_flags = IB_SEND_SIGNALED;
- last->wr.wr_cqe = &last->cqe;
-
- /*
- * Remove last from send_ctx->msg_list
- * and splice the rest of send_ctx->msg_list
- * to last->sibling_list.
- *
- * send_ctx->msg_list is a valid empty list
- * at the end.
- */
- list_del_init(&last->sibling_list);
- list_splice_tail_init(&send_ctx->msg_list, &last->sibling_list);
- send_ctx->wr_cnt = 0;
-
- ret = smb_direct_post_send(sc, &first->wr);
- if (ret) {
- struct smbdirect_send_io *sibling, *next;
-
- list_for_each_entry_safe(sibling, next, &last->sibling_list, sibling_list) {
- list_del_init(&sibling->sibling_list);
- smbdirect_connection_free_send_io(sibling);
- }
- smbdirect_connection_free_send_io(last);
- }
-
-release_credit:
- if (is_last && !ret && send_ctx->credit) {
- atomic_add(send_ctx->credit, &sc->send_io.bcredits.count);
- send_ctx->credit = 0;
- wake_up(&sc->send_io.bcredits.wait_queue);
- }
-
- return ret;
-}
-
-
-static int wait_for_send_bcredit(struct smbdirect_socket *sc,
- struct smbdirect_send_batch *send_ctx)
-{
- int ret;
-
- if (send_ctx->credit)
- return 0;
-
- ret = smbdirect_socket_wait_for_credits(sc,
- SMBDIRECT_SOCKET_CONNECTED,
- -ENOTCONN,
- &sc->send_io.bcredits.wait_queue,
- &sc->send_io.bcredits.count,
- 1);
- if (ret)
- return ret;
-
- send_ctx->credit = 1;
- return 0;
-}
-
-static int wait_for_send_lcredit(struct smbdirect_socket *sc,
- struct smbdirect_send_batch *send_ctx)
-{
- if (send_ctx && (atomic_read(&sc->send_io.lcredits.count) <= 1)) {
- int ret;
-
- ret = smb_direct_flush_send_list(sc, send_ctx, false);
- if (ret)
- return ret;
- }
-
- return smbdirect_socket_wait_for_credits(sc,
- SMBDIRECT_SOCKET_CONNECTED,
- -ENOTCONN,
- &sc->send_io.lcredits.wait_queue,
- &sc->send_io.lcredits.count,
- 1);
-}
-
-static int wait_for_send_credits(struct smbdirect_socket *sc,
- struct smbdirect_send_batch *send_ctx)
-{
- int ret;
-
- if (send_ctx &&
- (send_ctx->wr_cnt >= 16 || atomic_read(&sc->send_io.credits.count) <= 1)) {
- ret = smb_direct_flush_send_list(sc, send_ctx, false);
- if (ret)
- return ret;
- }
-
- return smbdirect_socket_wait_for_credits(sc,
- SMBDIRECT_SOCKET_CONNECTED,
- -ENOTCONN,
- &sc->send_io.credits.wait_queue,
- &sc->send_io.credits.count,
- 1);
-}
-
-static int post_sendmsg(struct smbdirect_socket *sc,
- struct smbdirect_send_batch *send_ctx,
- struct smbdirect_send_io *msg)
-{
- int i;
-
- for (i = 0; i < msg->num_sge; i++)
- ib_dma_sync_single_for_device(sc->ib.dev,
- msg->sge[i].addr, msg->sge[i].length,
- DMA_TO_DEVICE);
-
- msg->cqe.done = smbdirect_connection_send_io_done;
- msg->wr.opcode = IB_WR_SEND;
- msg->wr.sg_list = &msg->sge[0];
- msg->wr.num_sge = msg->num_sge;
- msg->wr.next = NULL;
-
- if (send_ctx) {
- msg->wr.wr_cqe = NULL;
- msg->wr.send_flags = 0;
- if (!list_empty(&send_ctx->msg_list)) {
- struct smbdirect_send_io *last;
-
- last = list_last_entry(&send_ctx->msg_list,
- struct smbdirect_send_io,
- sibling_list);
- last->wr.next = &msg->wr;
- }
- list_add_tail(&msg->sibling_list, &send_ctx->msg_list);
- send_ctx->wr_cnt++;
- return 0;
- }
-
- msg->wr.wr_cqe = &msg->cqe;
- msg->wr.send_flags = IB_SEND_SIGNALED;
- return smb_direct_post_send(sc, &msg->wr);
-}
-
-static int smb_direct_post_send_data(struct smbdirect_socket *sc,
- struct smbdirect_send_batch *send_ctx,
- struct iov_iter *iter,
- u32 remaining_data_length)
-{
- const struct smbdirect_socket_parameters *sp = &sc->parameters;
- int ret;
- struct smbdirect_send_io *msg;
- struct smbdirect_data_transfer *packet;
- size_t header_length;
- u32 data_length = 0;
- struct smbdirect_send_batch _send_ctx;
- u16 new_credits;
-
- if (iter) {
- header_length = sizeof(struct smbdirect_data_transfer);
- if (WARN_ON_ONCE(remaining_data_length == 0 ||
- iov_iter_count(iter) > remaining_data_length))
- return -EINVAL;
- } else {
- /* If this is a packet without payload, don't send padding */
- header_length = offsetof(struct smbdirect_data_transfer, padding);
- if (WARN_ON_ONCE(remaining_data_length))
- return -EINVAL;
- }
-
- if (!send_ctx) {
- smb_direct_send_ctx_init(&_send_ctx, false, 0);
- send_ctx = &_send_ctx;
- }
-
- ret = wait_for_send_bcredit(sc, send_ctx);
- if (ret)
- goto bcredit_failed;
-
- ret = wait_for_send_lcredit(sc, send_ctx);
- if (ret)
- goto lcredit_failed;
-
- ret = wait_for_send_credits(sc, send_ctx);
- if (ret)
- goto credit_failed;
-
- new_credits = smbdirect_connection_grant_recv_credits(sc);
- if (new_credits == 0 &&
- atomic_read(&sc->send_io.credits.count) == 0 &&
- atomic_read(&sc->recv_io.credits.count) == 0) {
- queue_work(sc->workqueue, &sc->recv_io.posted.refill_work);
- ret = wait_event_interruptible(sc->send_io.credits.wait_queue,
- atomic_read(&sc->send_io.credits.count) >= 1 ||
- atomic_read(&sc->recv_io.credits.available) >= 1 ||
- sc->status != SMBDIRECT_SOCKET_CONNECTED);
- if (sc->status != SMBDIRECT_SOCKET_CONNECTED)
- ret = -ENOTCONN;
- if (ret < 0)
- goto credit_failed;
-
- new_credits = smbdirect_connection_grant_recv_credits(sc);
- }
-
- msg = smbdirect_connection_alloc_send_io(sc);
- if (IS_ERR(msg)) {
- ret = PTR_ERR(msg);
- goto alloc_failed;
- }
-
- /* Map the packet to DMA */
- msg->sge[0].addr = ib_dma_map_single(sc->ib.dev,
- msg->packet,
- header_length,
- DMA_TO_DEVICE);
- ret = ib_dma_mapping_error(sc->ib.dev, msg->sge[0].addr);
- if (ret)
- goto err;
-
- msg->sge[0].length = header_length;
- msg->sge[0].lkey = sc->ib.pd->local_dma_lkey;
- msg->num_sge = 1;
-
- if (iter) {
- struct smbdirect_map_sges extract = {
- .num_sge = msg->num_sge,
- .max_sge = ARRAY_SIZE(msg->sge),
- .sge = msg->sge,
- .device = sc->ib.dev,
- .local_dma_lkey = sc->ib.pd->local_dma_lkey,
- .direction = DMA_TO_DEVICE,
- };
- size_t payload_len = umin(iov_iter_count(iter),
- sp->max_send_size - sizeof(*packet));
-
- ret = smbdirect_map_sges_from_iter(iter, payload_len, &extract);
- if (ret < 0)
- goto err;
- data_length = ret;
- remaining_data_length -= data_length;
- msg->num_sge = extract.num_sge;
- }
-
- /* Fill in the packet header */
- packet = (struct smbdirect_data_transfer *)msg->packet;
- packet->credits_requested = cpu_to_le16(sp->send_credit_target);
- new_credits = smbdirect_connection_grant_recv_credits(sc);
- packet->credits_granted = cpu_to_le16(new_credits);
-
- packet->flags = 0;
- if (smbdirect_connection_request_keep_alive(sc))
- packet->flags |= cpu_to_le16(SMBDIRECT_FLAG_RESPONSE_REQUESTED);
-
- packet->reserved = 0;
- if (!data_length)
- packet->data_offset = 0;
- else
- packet->data_offset = cpu_to_le32(24);
- packet->data_length = cpu_to_le32(data_length);
- packet->remaining_data_length = cpu_to_le32(remaining_data_length);
- packet->padding = 0;
-
- ksmbd_debug(RDMA,
- "credits_req=%u credits_granted=%u flags=0x%x ofs=%u len=%u remaining=%u\n",
- le16_to_cpu(packet->credits_requested),
- le16_to_cpu(packet->credits_granted),
- le16_to_cpu(packet->flags),
- le32_to_cpu(packet->data_offset),
- le32_to_cpu(packet->data_length),
- le32_to_cpu(packet->remaining_data_length));
-
- ret = post_sendmsg(sc, send_ctx, msg);
- if (ret)
- goto err;
-
- /*
- * From here msg is moved to send_ctx
- * and we should not free it explicitly.
- */
-
- if (send_ctx == &_send_ctx) {
- ret = smb_direct_flush_send_list(sc, send_ctx, true);
- if (ret)
- goto flush_failed;
- }
-
- return data_length;
-err:
- smbdirect_connection_free_send_io(msg);
-flush_failed:
-alloc_failed:
- atomic_inc(&sc->send_io.credits.count);
-credit_failed:
- atomic_inc(&sc->send_io.lcredits.count);
-lcredit_failed:
- atomic_add(send_ctx->credit, &sc->send_io.bcredits.count);
- send_ctx->credit = 0;
-bcredit_failed:
- return ret;
-}
-
-static int smb_direct_send_iter(struct smbdirect_socket *sc,
- struct iov_iter *iter,
- bool need_invalidate,
- unsigned int remote_key)
-{
- struct smbdirect_socket_parameters *sp = &sc->parameters;
- int ret;
- struct smbdirect_send_batch send_ctx;
- int error = 0;
- __be32 hdr;
-
- if (sc->status != SMBDIRECT_SOCKET_CONNECTED)
- return -ENOTCONN;
-
- /*
- * For now we expect the iter to have the full
- * message, including a 4 byte length header.
- */
- if (iov_iter_count(iter) <= 4)
- return -EINVAL;
- if (!copy_from_iter_full(&hdr, sizeof(hdr), iter))
- return -EFAULT;
- if (iov_iter_count(iter) != be32_to_cpu(hdr))
- return -EINVAL;
-
- /*
- * The size must fit into the negotiated
- * fragmented send size.
- */
- if (iov_iter_count(iter) > sp->max_fragmented_send_size)
- return -EMSGSIZE;
-
- ksmbd_debug(RDMA, "Sending smb (RDMA): smb_len=%zu\n",
- iov_iter_count(iter));
-
- smb_direct_send_ctx_init(&send_ctx, need_invalidate, remote_key);
- while (iov_iter_count(iter)) {
- ret = smb_direct_post_send_data(sc,
- &send_ctx,
- iter,
- iov_iter_count(iter));
- if (unlikely(ret < 0)) {
- error = ret;
- break;
- }
- }
-
- ret = smb_direct_flush_send_list(sc, &send_ctx, true);
- if (unlikely(!ret && error))
- ret = error;
-
- /*
- * As an optimization, we don't wait for individual I/O to finish
- * before sending the next one.
- * Send them all and wait for pending send count to get to 0
- * that means all the I/Os have been out and we are good to return
- */
-
- wait_event(sc->send_io.pending.zero_wait_queue,
- atomic_read(&sc->send_io.pending.count) == 0 ||
- sc->status != SMBDIRECT_SOCKET_CONNECTED);
- if (sc->status != SMBDIRECT_SOCKET_CONNECTED && ret == 0)
- ret = -ENOTCONN;
-
- return ret;
-}
-
static int smb_direct_writev(struct ksmbd_transport *t,
struct kvec *iov, int niovs, int buflen,
bool need_invalidate, unsigned int remote_key)
iov_iter_kvec(&iter, ITER_SOURCE, iov, niovs, buflen);
- return smb_direct_send_iter(sc, &iter, need_invalidate, remote_key);
+ return smbdirect_connection_send_iter(sc, &iter, 0,
+ need_invalidate, remote_key);
}
static int smb_direct_rdma_write(struct ksmbd_transport *t,
sendmsg->sge[0].length = sizeof(*resp);
sendmsg->sge[0].lkey = sc->ib.pd->local_dma_lkey;
- ret = post_sendmsg(sc, NULL, sendmsg);
+ ret = smbdirect_connection_post_send_io(sc, NULL, sendmsg);
if (ret) {
smbdirect_connection_free_send_io(sendmsg);
return ret;
return ret;
INIT_WORK(&sc->recv_io.posted.refill_work, smbdirect_connection_recv_io_refill_work);
- INIT_WORK(&sc->idle.immediate_work, smb_direct_send_immediate_work);
+ INIT_WORK(&sc->idle.immediate_work, smbdirect_connection_send_immediate_work);
return 0;
}
projects / thirdparty / kernel / linux.git / commitdiff
