return rc;
}
-static int tls_split_open_record(struct sock *sk, struct tls_rec *from,
- struct tls_rec **to, struct sk_msg *msg_opl,
- struct sk_msg *msg_oen, u32 split_point,
- u32 tx_overhead_size, u32 *orig_end)
-{
- u32 i, j, bytes = 0, apply = msg_opl->apply_bytes;
- struct scatterlist *sge, *osge, *nsge;
- u32 orig_size = msg_opl->sg.size;
- struct scatterlist tmp = { };
- struct sk_msg *msg_npl;
- struct tls_rec *new;
- int ret;
-
- new = tls_get_rec(sk);
- if (!new)
- return -ENOMEM;
- ret = sk_msg_alloc(sk, &new->msg_encrypted, msg_opl->sg.size +
- tx_overhead_size, 0);
- if (ret < 0) {
- tls_free_rec(sk, new);
- return ret;
- }
-
- *orig_end = msg_opl->sg.end;
- i = msg_opl->sg.start;
- sge = sk_msg_elem(msg_opl, i);
- while (apply && sge->length) {
- if (sge->length > apply) {
- u32 len = sge->length - apply;
-
- get_page(sg_page(sge));
- sg_set_page(&tmp, sg_page(sge), len,
- sge->offset + apply);
- sge->length = apply;
- bytes += apply;
- apply = 0;
- } else {
- apply -= sge->length;
- bytes += sge->length;
- }
-
- sk_msg_iter_var_next(i);
- if (i == msg_opl->sg.end)
- break;
- sge = sk_msg_elem(msg_opl, i);
- }
-
- msg_opl->sg.end = i;
- msg_opl->sg.curr = i;
- msg_opl->sg.copybreak = 0;
- msg_opl->apply_bytes = 0;
- msg_opl->sg.size = bytes;
-
- msg_npl = &new->msg_plaintext;
- msg_npl->apply_bytes = apply;
- msg_npl->sg.size = orig_size - bytes;
-
- j = msg_npl->sg.start;
- nsge = sk_msg_elem(msg_npl, j);
- if (tmp.length) {
- memcpy(nsge, &tmp, sizeof(*nsge));
- sk_msg_iter_var_next(j);
- nsge = sk_msg_elem(msg_npl, j);
- }
-
- osge = sk_msg_elem(msg_opl, i);
- while (osge->length) {
- memcpy(nsge, osge, sizeof(*nsge));
- sg_unmark_end(nsge);
- sk_msg_iter_var_next(i);
- sk_msg_iter_var_next(j);
- if (i == *orig_end)
- break;
- osge = sk_msg_elem(msg_opl, i);
- nsge = sk_msg_elem(msg_npl, j);
- }
-
- msg_npl->sg.end = j;
- msg_npl->sg.curr = j;
- msg_npl->sg.copybreak = 0;
-
- *to = new;
- return 0;
-}
-
-static void tls_merge_open_record(struct sock *sk, struct tls_rec *to,
- struct tls_rec *from, u32 orig_end)
-{
- struct sk_msg *msg_npl = &from->msg_plaintext;
- struct sk_msg *msg_opl = &to->msg_plaintext;
- struct scatterlist *osge, *nsge;
- u32 i, j;
-
- i = msg_opl->sg.end;
- sk_msg_iter_var_prev(i);
- j = msg_npl->sg.start;
-
- osge = sk_msg_elem(msg_opl, i);
- nsge = sk_msg_elem(msg_npl, j);
-
- if (sg_page(osge) == sg_page(nsge) &&
- osge->offset + osge->length == nsge->offset) {
- osge->length += nsge->length;
- put_page(sg_page(nsge));
- }
-
- msg_opl->sg.end = orig_end;
- msg_opl->sg.curr = orig_end;
- msg_opl->sg.copybreak = 0;
- msg_opl->apply_bytes = msg_opl->sg.size + msg_npl->sg.size;
- msg_opl->sg.size += msg_npl->sg.size;
-
- sk_msg_free(sk, &to->msg_encrypted);
- sk_msg_xfer_full(&to->msg_encrypted, &from->msg_encrypted);
-
- kfree(from);
-}
-
static int tls_push_record(struct sock *sk, int flags,
unsigned char record_type)
{
struct tls_context *tls_ctx = tls_get_ctx(sk);
struct tls_prot_info *prot = &tls_ctx->prot_info;
struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx);
- struct tls_rec *rec = ctx->open_rec, *tmp = NULL;
- u32 i, split_point, orig_end;
+ struct tls_rec *rec = ctx->open_rec;
struct sk_msg *msg_pl, *msg_en;
struct aead_request *req;
- bool split;
int rc;
+ u32 i;
if (!rec)
return 0;
msg_pl = &rec->msg_plaintext;
msg_en = &rec->msg_encrypted;
- split_point = msg_pl->apply_bytes;
- split = split_point && split_point < msg_pl->sg.size;
- if (unlikely((!split &&
- msg_pl->sg.size +
- prot->overhead_size > msg_en->sg.size) ||
- (split &&
- split_point +
- prot->overhead_size > msg_en->sg.size))) {
- split = true;
- split_point = msg_en->sg.size;
- }
- if (split) {
- rc = tls_split_open_record(sk, rec, &tmp, msg_pl, msg_en,
- split_point, prot->overhead_size,
- &orig_end);
- if (rc < 0)
- return rc;
- /* This can happen if above tls_split_open_record allocates
- * a single large encryption buffer instead of two smaller
- * ones. In this case adjust pointers and continue without
- * split.
- */
- if (!msg_pl->sg.size) {
- tls_merge_open_record(sk, rec, tmp, orig_end);
- msg_pl = &rec->msg_plaintext;
- msg_en = &rec->msg_encrypted;
- split = false;
- }
- sk_msg_trim(sk, msg_en, msg_pl->sg.size +
- prot->overhead_size);
- }
-
rec->tx_flags = flags;
req = &rec->aead_req;
rc = tls_do_encryption(sk, tls_ctx, ctx, req,
msg_pl->sg.size + prot->tail_size, i);
if (rc < 0) {
- if (rc != -EINPROGRESS) {
+ if (rc != -EINPROGRESS)
tls_err_abort(sk, -EBADMSG);
- if (split) {
- tls_ctx->pending_open_record_frags = true;
- tls_merge_open_record(sk, rec, tmp, orig_end);
- }
- }
ctx->async_capable = 1;
return rc;
- } else if (split) {
- msg_pl = &tmp->msg_plaintext;
- msg_en = &tmp->msg_encrypted;
- sk_msg_trim(sk, msg_en, msg_pl->sg.size + prot->overhead_size);
- tls_ctx->pending_open_record_frags = true;
- ctx->open_rec = tmp;
}
return tls_tx_records(sk, flags);
}
static int bpf_exec_tx_verdict(struct sk_msg *msg, struct sock *sk,
- bool full_record, u8 record_type,
- ssize_t *copied, int flags)
+ u8 record_type, ssize_t *copied, int flags)
{
- struct tls_context *tls_ctx = tls_get_ctx(sk);
- struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx);
- struct sk_msg msg_redir = { };
- struct sk_psock *psock;
- struct sock *sk_redir;
- struct tls_rec *rec;
- bool enospc, policy, redir_ingress;
- int err = 0, send;
- u32 delta = 0;
-
- policy = !(flags & MSG_SENDPAGE_NOPOLICY);
- psock = sk_psock_get(sk);
- if (!psock || !policy) {
- err = tls_push_record(sk, flags, record_type);
- if (err && err != -EINPROGRESS && sk->sk_err == EBADMSG) {
- *copied -= sk_msg_free(sk, msg);
- tls_free_open_rec(sk);
- err = -sk->sk_err;
- }
- if (psock)
- sk_psock_put(sk, psock);
- return err;
- }
-more_data:
- enospc = sk_msg_full(msg);
- if (psock->eval == __SK_NONE) {
- delta = msg->sg.size;
- psock->eval = sk_psock_msg_verdict(sk, psock, msg);
- delta -= msg->sg.size;
-
- if ((s32)delta > 0) {
- /* It indicates that we executed bpf_msg_pop_data(),
- * causing the plaintext data size to decrease.
- * Therefore the encrypted data size also needs to
- * correspondingly decrease. We only need to subtract
- * delta to calculate the new ciphertext length since
- * ktls does not support block encryption.
- */
- struct sk_msg *enc = &ctx->open_rec->msg_encrypted;
+ int err;
- sk_msg_trim(sk, enc, enc->sg.size - delta);
- }
- }
- if (msg->cork_bytes && msg->cork_bytes > msg->sg.size &&
- !enospc && !full_record) {
- err = -ENOSPC;
- goto out_err;
- }
- msg->cork_bytes = 0;
- send = msg->sg.size;
- if (msg->apply_bytes && msg->apply_bytes < send)
- send = msg->apply_bytes;
-
- switch (psock->eval) {
- case __SK_PASS:
- err = tls_push_record(sk, flags, record_type);
- if (err && err != -EINPROGRESS && sk->sk_err == EBADMSG) {
- *copied -= sk_msg_free(sk, msg);
- tls_free_open_rec(sk);
- err = -sk->sk_err;
- goto out_err;
- }
- break;
- case __SK_REDIRECT:
- redir_ingress = psock->redir_ingress;
- sk_redir = psock->sk_redir;
- memcpy(&msg_redir, msg, sizeof(*msg));
- if (msg->apply_bytes < send)
- msg->apply_bytes = 0;
- else
- msg->apply_bytes -= send;
- sk_msg_return_zero(sk, msg, send);
- msg->sg.size -= send;
- release_sock(sk);
- err = tcp_bpf_sendmsg_redir(sk_redir, redir_ingress,
- &msg_redir, send, flags);
- lock_sock(sk);
- if (err < 0) {
- /* Regardless of whether the data represented by
- * msg_redir is sent successfully, we have already
- * uncharged it via sk_msg_return_zero(). The
- * msg->sg.size represents the remaining unprocessed
- * data, which needs to be uncharged here.
- */
- sk_mem_uncharge(sk, msg->sg.size);
- *copied -= sk_msg_free_nocharge(sk, &msg_redir);
- msg->sg.size = 0;
- }
- if (msg->sg.size == 0)
- tls_free_open_rec(sk);
- break;
- case __SK_DROP:
- default:
- sk_msg_free_partial(sk, msg, send);
- if (msg->apply_bytes < send)
- msg->apply_bytes = 0;
- else
- msg->apply_bytes -= send;
- if (msg->sg.size == 0)
- tls_free_open_rec(sk);
- *copied -= (send + delta);
- err = -EACCES;
+ err = tls_push_record(sk, flags, record_type);
+ if (err && err != -EINPROGRESS && sk->sk_err == EBADMSG) {
+ *copied -= sk_msg_free(sk, msg);
+ tls_free_open_rec(sk);
+ err = -sk->sk_err;
}
-
- if (likely(!err)) {
- bool reset_eval = !ctx->open_rec;
-
- rec = ctx->open_rec;
- if (rec) {
- msg = &rec->msg_plaintext;
- if (!msg->apply_bytes)
- reset_eval = true;
- }
- if (reset_eval) {
- psock->eval = __SK_NONE;
- if (psock->sk_redir) {
- sock_put(psock->sk_redir);
- psock->sk_redir = NULL;
- }
- }
- if (rec)
- goto more_data;
- }
- out_err:
- sk_psock_put(sk, psock);
return err;
}
if (!copied)
return 0;
- return bpf_exec_tx_verdict(msg_pl, sk, true, TLS_RECORD_TYPE_DATA,
+ return bpf_exec_tx_verdict(msg_pl, sk, TLS_RECORD_TYPE_DATA,
&copied, flags);
}
copied += try_to_copy;
sk_msg_sg_copy_set(msg_pl, first);
- ret = bpf_exec_tx_verdict(msg_pl, sk, full_record,
+ ret = bpf_exec_tx_verdict(msg_pl, sk,
record_type, &copied,
msg->msg_flags);
if (ret) {
num_async++;
else if (ret == -ENOMEM)
goto wait_for_memory;
- else if (ctx->open_rec && ret == -ENOSPC) {
- if (msg_pl->cork_bytes) {
- ret = 0;
- goto send_end;
- }
- goto rollback_iter;
- } else if (ret != -EAGAIN)
+ else if (ret != -EAGAIN)
goto send_end;
}
}
continue;
-rollback_iter:
- copied -= try_to_copy;
- sk_msg_sg_copy_clear(msg_pl, first);
- iov_iter_revert(&msg->msg_iter,
- msg_pl->sg.size - orig_size);
fallback_to_reg_send:
sk_msg_trim(sk, msg_pl, orig_size);
}
copied += try_to_copy;
copied:
if (full_record || eor) {
- ret = bpf_exec_tx_verdict(msg_pl, sk, full_record,
+ ret = bpf_exec_tx_verdict(msg_pl, sk,
record_type, &copied,
msg->msg_flags);
if (ret) {
num_async++;
else if (ret == -ENOMEM)
goto wait_for_memory;
- else if (ret != -EAGAIN) {
- if (ret == -ENOSPC)
- ret = 0;
+ else if (ret != -EAGAIN)
goto send_end;
- }
}
/* Transmit if any encryptions have completed */
if (msg_pl->sg.size == 0)
goto unlock;
- /* Check the BPF advisor and perform transmission. */
- ret = bpf_exec_tx_verdict(msg_pl, sk, false, TLS_RECORD_TYPE_DATA,
+ /* Perform transmission. */
+ ret = bpf_exec_tx_verdict(msg_pl, sk, TLS_RECORD_TYPE_DATA,
&copied, 0);
switch (ret) {
case 0:
* consumed via sock_error().
*/
static int
-tls_rx_rec_wait(struct sock *sk, struct sk_psock *psock, bool nonblock,
- bool released, bool has_copied)
+tls_rx_rec_wait(struct sock *sk, bool nonblock, bool released, bool has_copied)
{
struct tls_context *tls_ctx = tls_get_ctx(sk);
struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx);
timeo = sock_rcvtimeo(sk, nonblock);
while (!tls_strp_msg_ready(ctx)) {
- if (!sk_psock_queue_empty(psock))
- return 0;
-
if (sk->sk_err) {
if (has_copied)
return -READ_ONCE(sk->sk_err);
add_wait_queue(sk_sleep(sk), &wait);
sk_set_bit(SOCKWQ_ASYNC_WAITDATA, sk);
ret = sk_wait_event(sk, &timeo,
- tls_strp_msg_ready(ctx) ||
- !sk_psock_queue_empty(psock),
- &wait);
+ tls_strp_msg_ready(ctx), &wait);
sk_clear_bit(SOCKWQ_ASYNC_WAITDATA, sk);
remove_wait_queue(sk_sleep(sk), &wait);
}
if (unlikely(!tls_strp_msg_load(&ctx->strp, released)))
- return tls_rx_rec_wait(sk, psock, nonblock, false, has_copied);
+ return tls_rx_rec_wait(sk, nonblock, false, has_copied);
return 1;
}
struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx);
struct tls_prot_info *prot = &tls_ctx->prot_info;
ssize_t decrypted = 0, async_copy_bytes = 0;
- struct sk_psock *psock;
unsigned char control = 0;
size_t flushed_at = 0;
struct strp_msg *rxm;
bool is_peek = flags & MSG_PEEK;
bool rx_more = false;
bool released = true;
- bool bpf_strp_enabled;
bool zc_capable;
if (unlikely(flags & MSG_ERRQUEUE))
err = tls_rx_reader_lock(sk, ctx, flags & MSG_DONTWAIT);
if (err < 0)
return err;
- psock = sk_psock_get(sk);
- bpf_strp_enabled = sk_psock_strp_enabled(psock);
/* If crypto failed the connection is broken */
err = ctx->async_wait.err;
target = sock_rcvlowat(sk, flags & MSG_WAITALL, len);
len = len - copied;
- zc_capable = !bpf_strp_enabled && !is_kvec && !is_peek &&
- ctx->zc_capable;
+ zc_capable = !is_kvec && !is_peek && ctx->zc_capable;
decrypted = 0;
while (len && (decrypted + copied < target || tls_strp_msg_ready(ctx))) {
struct tls_decrypt_arg darg;
int to_decrypt, chunk;
- err = tls_rx_rec_wait(sk, psock, flags & MSG_DONTWAIT,
+ err = tls_rx_rec_wait(sk, flags & MSG_DONTWAIT,
released, !!(decrypted + copied));
- if (err <= 0) {
- if (psock) {
- chunk = sk_msg_recvmsg(sk, psock, msg, len,
- flags);
- if (chunk > 0) {
- decrypted += chunk;
- len -= chunk;
- continue;
- }
- }
+ if (err <= 0)
goto recv_end;
- }
memset(&darg.inargs, 0, sizeof(darg.inargs));
darg.zc = true;
/* Do not use async mode if record is non-data */
- if (tlm->control == TLS_RECORD_TYPE_DATA && !bpf_strp_enabled)
+ if (tlm->control == TLS_RECORD_TYPE_DATA)
darg.async = ctx->async_capable;
else
darg.async = false;
continue;
}
- if (bpf_strp_enabled) {
- released = true;
- err = sk_psock_tls_strp_read(psock, skb);
- if (err != __SK_PASS) {
- rxm->offset = rxm->offset + rxm->full_len;
- rxm->full_len = 0;
- if (err == __SK_DROP)
- consume_skb(skb);
- continue;
- }
- }
-
if (partially_consumed)
chunk = len;
end:
tls_rx_reader_unlock(sk, ctx);
- if (psock)
- sk_psock_put(sk, psock);
return copied ? : err;
}
} else {
struct tls_decrypt_arg darg;
- err = tls_rx_rec_wait(sk, NULL, flags & SPLICE_F_NONBLOCK,
+ err = tls_rx_rec_wait(sk, flags & SPLICE_F_NONBLOCK,
true, false);
if (err <= 0)
goto splice_read_end;
} else {
struct tls_decrypt_arg darg;
- err = tls_rx_rec_wait(sk, NULL, true, released, !!copied);
+ err = tls_rx_rec_wait(sk, true, released, !!copied);
if (err <= 0)
goto read_sock_end;
{
struct tls_context *tls_ctx = tls_get_ctx(sk);
struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx);
- bool ingress_empty = true;
- struct sk_psock *psock;
- rcu_read_lock();
- psock = sk_psock(sk);
- if (psock)
- ingress_empty = list_empty(&psock->ingress_msg);
- rcu_read_unlock();
-
- return !ingress_empty || tls_strp_msg_ready(ctx) ||
+ return tls_strp_msg_ready(ctx) ||
!skb_queue_empty(&ctx->rx_list);
}
{
struct tls_context *tls_ctx = tls_get_ctx(sk);
struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx);
- struct sk_psock *psock;
gfp_t alloc_save;
trace_sk_data_ready(sk);
sk->sk_allocation = GFP_ATOMIC;
tls_strp_data_ready(&ctx->strp);
sk->sk_allocation = alloc_save;
-
- psock = sk_psock_get(sk);
- if (psock) {
- if (!list_empty(&psock->ingress_msg))
- ctx->saved_data_ready(sk);
- sk_psock_put(sk, psock);
- }
}
void tls_sw_cancel_work_tx(struct tls_context *tls_ctx)
projects / thirdparty / kernel / linux.git / commitdiff
