return smbd_ib_post_send(sc, &request->wr);
}
-static void smbd_send_batch_init(struct smbdirect_send_batch *batch,
- bool need_invalidate_rkey,
- unsigned int remote_key)
-{
- INIT_LIST_HEAD(&batch->msg_list);
- batch->wr_cnt = 0;
- batch->need_invalidate_rkey = need_invalidate_rkey;
- batch->remote_key = remote_key;
- batch->credit = 0;
-}
-
-static int smbd_send_batch_flush(struct smbdirect_socket *sc,
- struct smbdirect_send_batch *batch,
- bool is_last)
-{
- struct smbdirect_send_io *first, *last;
- int ret = 0;
-
- if (list_empty(&batch->msg_list))
- goto release_credit;
-
- first = list_first_entry(&batch->msg_list,
- struct smbdirect_send_io,
- sibling_list);
- last = list_last_entry(&batch->msg_list,
- struct smbdirect_send_io,
- sibling_list);
-
- if (batch->need_invalidate_rkey) {
- first->wr.opcode = IB_WR_SEND_WITH_INV;
- first->wr.ex.invalidate_rkey = batch->remote_key;
- batch->need_invalidate_rkey = false;
- batch->remote_key = 0;
- }
-
- last->wr.send_flags = IB_SEND_SIGNALED;
- last->wr.wr_cqe = &last->cqe;
-
- /*
- * Remove last from batch->msg_list
- * and splice the rest of batch->msg_list
- * to last->sibling_list.
- *
- * batch->msg_list is a valid empty list
- * at the end.
- */
- list_del_init(&last->sibling_list);
- list_splice_tail_init(&batch->msg_list, &last->sibling_list);
- batch->wr_cnt = 0;
-
- ret = smbd_ib_post_send(sc, &first->wr);
- if (ret) {
- struct smbdirect_send_io *sibling, *next;
-
- list_for_each_entry_safe(sibling, next, &last->sibling_list, sibling_list) {
- list_del_init(&sibling->sibling_list);
- smbdirect_connection_free_send_io(sibling);
- }
- smbdirect_connection_free_send_io(last);
- }
-
-release_credit:
- if (is_last && !ret && batch->credit) {
- atomic_add(batch->credit, &sc->send_io.bcredits.count);
- batch->credit = 0;
- wake_up(&sc->send_io.bcredits.wait_queue);
- }
-
- return ret;
-}
-
-static int wait_for_credits(struct smbdirect_socket *sc,
- wait_queue_head_t *waitq, atomic_t *total_credits,
- int needed)
-{
- int ret;
-
- do {
- if (atomic_sub_return(needed, total_credits) >= 0)
- return 0;
-
- atomic_add(needed, total_credits);
- ret = wait_event_interruptible(*waitq,
- atomic_read(total_credits) >= needed ||
- sc->status != SMBDIRECT_SOCKET_CONNECTED);
-
- if (sc->status != SMBDIRECT_SOCKET_CONNECTED)
- return -ENOTCONN;
- else if (ret < 0)
- return ret;
- } while (true);
-}
-
-static int wait_for_send_bcredit(struct smbdirect_socket *sc,
- struct smbdirect_send_batch *batch)
-{
- int ret;
-
- if (batch->credit)
- return 0;
-
- ret = wait_for_credits(sc,
- &sc->send_io.bcredits.wait_queue,
- &sc->send_io.bcredits.count,
- 1);
- if (ret)
- return ret;
-
- batch->credit = 1;
- return 0;
-}
-
-static int wait_for_send_lcredit(struct smbdirect_socket *sc,
- struct smbdirect_send_batch *batch)
-{
- if (batch && (atomic_read(&sc->send_io.lcredits.count) <= 1)) {
- int ret;
-
- ret = smbd_send_batch_flush(sc, batch, false);
- if (ret)
- return ret;
- }
-
- return wait_for_credits(sc,
- &sc->send_io.lcredits.wait_queue,
- &sc->send_io.lcredits.count,
- 1);
-}
-
-static int wait_for_send_credits(struct smbdirect_socket *sc,
- struct smbdirect_send_batch *batch)
-{
- if (batch &&
- (batch->wr_cnt >= 16 || atomic_read(&sc->send_io.credits.count) <= 1)) {
- int ret;
-
- ret = smbd_send_batch_flush(sc, batch, false);
- if (ret)
- return ret;
- }
-
- return wait_for_credits(sc,
- &sc->send_io.credits.wait_queue,
- &sc->send_io.credits.count,
- 1);
-}
-
-static int smbd_post_send_iter(struct smbdirect_socket *sc,
- struct smbdirect_send_batch *batch,
- struct iov_iter *iter,
- u32 remaining_data_length)
-{
- struct smbdirect_socket_parameters *sp = &sc->parameters;
- int rc;
- int header_length;
- int data_length;
- struct smbdirect_send_io *request;
- struct smbdirect_data_transfer *packet;
- u16 new_credits = 0;
- struct smbdirect_send_batch _batch;
-
- if (iter) {
- header_length = sizeof(struct smbdirect_data_transfer);
- if (WARN_ON_ONCE(remaining_data_length == 0 ||
- iov_iter_count(iter) > remaining_data_length))
- return -EINVAL;
- } else {
- /* If this is a packet without payload, don't send padding */
- header_length = offsetof(struct smbdirect_data_transfer, padding);
- if (WARN_ON_ONCE(remaining_data_length))
- return -EINVAL;
- }
-
- if (!batch) {
- smbd_send_batch_init(&_batch, false, 0);
- batch = &_batch;
- }
-
- rc = wait_for_send_bcredit(sc, batch);
- if (rc) {
- log_outgoing(ERR, "disconnected not sending on wait_bcredit\n");
- rc = -EAGAIN;
- goto err_wait_bcredit;
- }
-
- rc = wait_for_send_lcredit(sc, batch);
- if (rc) {
- log_outgoing(ERR, "disconnected not sending on wait_lcredit\n");
- rc = -EAGAIN;
- goto err_wait_lcredit;
- }
-
- rc = wait_for_send_credits(sc, batch);
- if (rc) {
- log_outgoing(ERR, "disconnected not sending on wait_credit\n");
- rc = -EAGAIN;
- goto err_wait_credit;
- }
-
- new_credits = smbdirect_connection_grant_recv_credits(sc);
- if (new_credits == 0 &&
- atomic_read(&sc->send_io.credits.count) == 0 &&
- atomic_read(&sc->recv_io.credits.count) == 0) {
- queue_work(sc->workqueue, &sc->recv_io.posted.refill_work);
- rc = wait_event_interruptible(sc->send_io.credits.wait_queue,
- atomic_read(&sc->send_io.credits.count) >= 1 ||
- atomic_read(&sc->recv_io.credits.available) >= 1 ||
- sc->status != SMBDIRECT_SOCKET_CONNECTED);
- if (sc->status != SMBDIRECT_SOCKET_CONNECTED)
- rc = -ENOTCONN;
- if (rc < 0) {
- log_outgoing(ERR, "disconnected not sending on last credit\n");
- rc = -EAGAIN;
- goto err_wait_credit;
- }
-
- new_credits = smbdirect_connection_grant_recv_credits(sc);
- }
-
- request = smbdirect_connection_alloc_send_io(sc);
- if (IS_ERR(request)) {
- rc = PTR_ERR(request);
- goto err_alloc;
- }
-
- memset(request->sge, 0, sizeof(request->sge));
-
- packet = smbdirect_send_io_payload(request);
- request->sge[0].addr = ib_dma_map_single(sc->ib.dev,
- (void *)packet,
- header_length,
- DMA_TO_DEVICE);
- if (ib_dma_mapping_error(sc->ib.dev, request->sge[0].addr)) {
- rc = -EIO;
- goto err_dma;
- }
-
- request->sge[0].length = header_length;
- request->sge[0].lkey = sc->ib.pd->local_dma_lkey;
- request->num_sge = 1;
-
- /* Fill in the data payload to find out how much data we can add */
- if (iter) {
- struct smbdirect_map_sges extract = {
- .num_sge = request->num_sge,
- .max_sge = ARRAY_SIZE(request->sge),
- .sge = request->sge,
- .device = sc->ib.dev,
- .local_dma_lkey = sc->ib.pd->local_dma_lkey,
- .direction = DMA_TO_DEVICE,
- };
- size_t payload_len = umin(iov_iter_count(iter),
- sp->max_send_size - sizeof(*packet));
-
- rc = smbdirect_map_sges_from_iter(iter, payload_len, &extract);
- if (rc < 0)
- goto err_dma;
- data_length = rc;
- request->num_sge = extract.num_sge;
- remaining_data_length -= data_length;
- } else {
- data_length = 0;
- }
-
- /* Fill in the packet header */
- packet->credits_requested = cpu_to_le16(sp->send_credit_target);
- packet->credits_granted = cpu_to_le16(new_credits);
-
- packet->flags = 0;
- if (smbdirect_connection_request_keep_alive(sc))
- packet->flags |= cpu_to_le16(SMBDIRECT_FLAG_RESPONSE_REQUESTED);
-
- packet->reserved = 0;
- if (!data_length)
- packet->data_offset = 0;
- else
- packet->data_offset = cpu_to_le32(24);
- packet->data_length = cpu_to_le32(data_length);
- packet->remaining_data_length = cpu_to_le32(remaining_data_length);
- packet->padding = 0;
-
- log_outgoing(INFO, "credits_requested=%d credits_granted=%d data_offset=%d data_length=%d remaining_data_length=%d\n",
- le16_to_cpu(packet->credits_requested),
- le16_to_cpu(packet->credits_granted),
- le32_to_cpu(packet->data_offset),
- le32_to_cpu(packet->data_length),
- le32_to_cpu(packet->remaining_data_length));
-
- rc = smbd_post_send(sc, batch, request);
- if (!rc) {
- /*
- * From here request is moved to batch
- * and we should not free it explicitly.
- */
-
- if (batch != &_batch)
- return data_length;
-
- rc = smbd_send_batch_flush(sc, batch, true);
- if (!rc)
- return data_length;
-
- goto err_flush;
- }
-
-err_dma:
- smbdirect_connection_free_send_io(request);
-
-err_flush:
-err_alloc:
- atomic_inc(&sc->send_io.credits.count);
- wake_up(&sc->send_io.credits.wait_queue);
-
-err_wait_credit:
- atomic_inc(&sc->send_io.lcredits.count);
- wake_up(&sc->send_io.lcredits.wait_queue);
-
-err_wait_lcredit:
- atomic_add(batch->credit, &sc->send_io.bcredits.count);
- batch->credit = 0;
- wake_up(&sc->send_io.bcredits.wait_queue);
-
-err_wait_bcredit:
- return rc;
-}
-
-/*
- * Send an empty message
- * Empty message is used to extend credits to peer to for keep live
- * while there is no upper layer payload to send at the time
- */
-static void smbd_post_send_empty(struct smbdirect_socket *sc)
-{
- int ret;
-
- sc->statistics.send_empty++;
- ret = smbd_post_send_iter(sc, NULL, NULL, 0);
- if (ret < 0) {
- log_rdma_send(ERR, "smbd_post_send_iter failed ret=%d\n", ret);
- smbdirect_socket_schedule_cleanup(sc, ret);
- }
-}
-
static int smbd_post_send_full_iter(struct smbdirect_socket *sc,
struct smbdirect_send_batch *batch,
struct iov_iter *iter,
int bytes = 0;
/*
- * smbd_post_send_iter() respects the
+ * smbdirect_connection_send_single_iter() respects the
* negotiated max_send_size, so we need to
* loop until the full iter is posted
*/
while (iov_iter_count(iter) > 0) {
int rc;
- rc = smbd_post_send_iter(sc, batch, iter, remaining_data_length);
+ rc = smbdirect_connection_send_single_iter(sc,
+ batch,
+ iter,
+ 0, /* flags */
+ remaining_data_length);
if (rc < 0)
return rc;
remaining_data_length -= rc;
return rc;
}
-static void send_immediate_empty_message(struct work_struct *work)
-{
- struct smbdirect_socket *sc =
- container_of(work, struct smbdirect_socket, idle.immediate_work);
-
- if (sc->status != SMBDIRECT_SOCKET_CONNECTED)
- return;
-
- log_keep_alive(INFO, "send an empty message\n");
- smbd_post_send_empty(sc);
-}
-
/*
* Destroy the transport and related RDMA and memory resources
* Need to go through all the pending counters and make sure on one is using
list_for_each_entry(recv_io, &sc->recv_io.free.list, list)
recv_io->cqe.done = recv_done;
- INIT_WORK(&sc->idle.immediate_work, send_immediate_empty_message);
+ INIT_WORK(&sc->idle.immediate_work, smbdirect_connection_send_immediate_work);
/*
* start with the negotiate timeout and SMBDIRECT_KEEPALIVE_PENDING
* so that the timer will cause a disconnect.
struct smbdirect_socket_parameters *sp = &sc->parameters;
struct smb_rqst *rqst;
struct iov_iter iter;
- struct smbdirect_send_batch batch;
+ struct smbdirect_send_batch_storage bstorage;
+ struct smbdirect_send_batch *batch;
unsigned int remaining_data_length, klen;
int rc, i, rqst_idx;
int error = 0;
num_rqst, remaining_data_length);
rqst_idx = 0;
- smbd_send_batch_init(&batch, false, 0);
+ batch = smbdirect_init_send_batch_storage(&bstorage, false, 0);
do {
rqst = &rqst_array[rqst_idx];
klen += rqst->rq_iov[i].iov_len;
iov_iter_kvec(&iter, ITER_SOURCE, rqst->rq_iov, rqst->rq_nvec, klen);
- rc = smbd_post_send_full_iter(sc, &batch, &iter, remaining_data_length);
+ rc = smbd_post_send_full_iter(sc, batch, &iter, remaining_data_length);
if (rc < 0) {
error = rc;
break;
if (iov_iter_count(&rqst->rq_iter) > 0) {
/* And then the data pages if there are any */
- rc = smbd_post_send_full_iter(sc, &batch, &rqst->rq_iter,
+ rc = smbd_post_send_full_iter(sc, batch, &rqst->rq_iter,
remaining_data_length);
if (rc < 0) {
error = rc;
} while (++rqst_idx < num_rqst);
- rc = smbd_send_batch_flush(sc, &batch, true);
+ rc = smbdirect_connection_send_batch_flush(sc, batch, true);
if (unlikely(!rc && error))
rc = error;
* that means all the I/Os have been out and we are good to return
*/
- wait_event(sc->send_io.pending.zero_wait_queue,
- atomic_read(&sc->send_io.pending.count) == 0 ||
- sc->status != SMBDIRECT_SOCKET_CONNECTED);
+ error = rc;
+ rc = smbdirect_connection_send_wait_zero_pending(sc);
+ if (unlikely(rc && !error))
+ error = -EAGAIN;
- if (sc->status != SMBDIRECT_SOCKET_CONNECTED && rc == 0)
- rc = -EAGAIN;
+ if (unlikely(error))
+ return error;
- return rc;
+ return 0;
}
/*
projects / thirdparty / kernel / linux.git / commitdiff
