1 // SPDX-License-Identifier: GPL-2.0-only
2 /* Copyright (C) 2020 Chelsio Communications. All rights reserved. */
4 #ifdef CONFIG_CHELSIO_TLS_DEVICE
5 #include <linux/highmem.h>
9 static int chcr_init_tcb_fields(struct chcr_ktls_info
*tx_info
);
11 * chcr_ktls_save_keys: calculate and save crypto keys.
12 * @tx_info - driver specific tls info.
13 * @crypto_info - tls crypto information.
14 * @direction - TX/RX direction.
15 * return - SUCCESS/FAILURE.
17 static int chcr_ktls_save_keys(struct chcr_ktls_info
*tx_info
,
18 struct tls_crypto_info
*crypto_info
,
19 enum tls_offload_ctx_dir direction
)
21 int ck_size
, key_ctx_size
, mac_key_size
, keylen
, ghash_size
, ret
;
22 unsigned char ghash_h
[TLS_CIPHER_AES_GCM_256_TAG_SIZE
];
23 struct tls12_crypto_info_aes_gcm_128
*info_128_gcm
;
24 struct ktls_key_ctx
*kctx
= &tx_info
->key_ctx
;
25 struct crypto_cipher
*cipher
;
26 unsigned char *key
, *salt
;
28 switch (crypto_info
->cipher_type
) {
29 case TLS_CIPHER_AES_GCM_128
:
31 (struct tls12_crypto_info_aes_gcm_128
*)crypto_info
;
32 keylen
= TLS_CIPHER_AES_GCM_128_KEY_SIZE
;
33 ck_size
= CHCR_KEYCTX_CIPHER_KEY_SIZE_128
;
34 tx_info
->salt_size
= TLS_CIPHER_AES_GCM_128_SALT_SIZE
;
35 mac_key_size
= CHCR_KEYCTX_MAC_KEY_SIZE_128
;
36 tx_info
->iv_size
= TLS_CIPHER_AES_GCM_128_IV_SIZE
;
37 tx_info
->iv
= be64_to_cpu(*(__be64
*)info_128_gcm
->iv
);
39 ghash_size
= TLS_CIPHER_AES_GCM_128_TAG_SIZE
;
40 key
= info_128_gcm
->key
;
41 salt
= info_128_gcm
->salt
;
42 tx_info
->record_no
= *(u64
*)info_128_gcm
->rec_seq
;
44 /* The SCMD fields used when encrypting a full TLS
45 * record. Its a one time calculation till the
48 tx_info
->scmd0_seqno_numivs
=
49 SCMD_SEQ_NO_CTRL_V(CHCR_SCMD_SEQ_NO_CTRL_64BIT
) |
50 SCMD_CIPH_AUTH_SEQ_CTRL_F
|
51 SCMD_PROTO_VERSION_V(CHCR_SCMD_PROTO_VERSION_TLS
) |
52 SCMD_CIPH_MODE_V(CHCR_SCMD_CIPHER_MODE_AES_GCM
) |
53 SCMD_AUTH_MODE_V(CHCR_SCMD_AUTH_MODE_GHASH
) |
54 SCMD_IV_SIZE_V(TLS_CIPHER_AES_GCM_128_IV_SIZE
>> 1) |
57 /* keys will be sent inline. */
58 tx_info
->scmd0_ivgen_hdrlen
= SCMD_KEY_CTX_INLINE_F
;
60 /* The SCMD fields used when encrypting a partial TLS
61 * record (no trailer and possibly a truncated payload).
63 tx_info
->scmd0_short_seqno_numivs
=
64 SCMD_CIPH_AUTH_SEQ_CTRL_F
|
65 SCMD_PROTO_VERSION_V(CHCR_SCMD_PROTO_VERSION_GENERIC
) |
66 SCMD_CIPH_MODE_V(CHCR_SCMD_CIPHER_MODE_AES_CTR
) |
67 SCMD_IV_SIZE_V(AES_BLOCK_LEN
>> 1);
69 tx_info
->scmd0_short_ivgen_hdrlen
=
70 tx_info
->scmd0_ivgen_hdrlen
| SCMD_AADIVDROP_F
;
75 pr_err("GCM: cipher type 0x%x not supported\n",
76 crypto_info
->cipher_type
);
81 key_ctx_size
= CHCR_KTLS_KEY_CTX_LEN
+
82 roundup(keylen
, 16) + ghash_size
;
83 /* Calculate the H = CIPH(K, 0 repeated 16 times).
84 * It will go in key context
86 cipher
= crypto_alloc_cipher("aes", 0, 0);
92 ret
= crypto_cipher_setkey(cipher
, key
, keylen
);
96 memset(ghash_h
, 0, ghash_size
);
97 crypto_cipher_encrypt_one(cipher
, ghash_h
, ghash_h
);
99 /* fill the Key context */
100 if (direction
== TLS_OFFLOAD_CTX_DIR_TX
) {
101 kctx
->ctx_hdr
= FILL_KEY_CTX_HDR(ck_size
,
109 memcpy(kctx
->salt
, salt
, tx_info
->salt_size
);
110 memcpy(kctx
->key
, key
, keylen
);
111 memcpy(kctx
->key
+ keylen
, ghash_h
, ghash_size
);
112 tx_info
->key_ctx_len
= key_ctx_size
;
115 crypto_free_cipher(cipher
);
120 static int chcr_ktls_update_connection_state(struct chcr_ktls_info
*tx_info
,
123 /* This function can be called from both rx (interrupt context) and tx
126 spin_lock_bh(&tx_info
->lock
);
127 switch (tx_info
->connection_state
) {
128 case KTLS_CONN_CLOSED
:
129 tx_info
->connection_state
= new_state
;
132 case KTLS_CONN_ACT_OPEN_REQ
:
133 /* only go forward if state is greater than current state. */
134 if (new_state
<= tx_info
->connection_state
)
136 /* update to the next state and also initialize TCB */
137 tx_info
->connection_state
= new_state
;
139 case KTLS_CONN_ACT_OPEN_RPL
:
140 /* if we are stuck in this state, means tcb init might not
141 * received by HW, try sending it again.
143 if (!chcr_init_tcb_fields(tx_info
))
144 tx_info
->connection_state
= KTLS_CONN_SET_TCB_REQ
;
147 case KTLS_CONN_SET_TCB_REQ
:
148 /* only go forward if state is greater than current state. */
149 if (new_state
<= tx_info
->connection_state
)
151 /* update to the next state and check if l2t_state is valid */
152 tx_info
->connection_state
= new_state
;
154 case KTLS_CONN_SET_TCB_RPL
:
155 /* Check if l2t state is valid, then move to ready state. */
156 if (cxgb4_check_l2t_valid(tx_info
->l2te
)) {
157 tx_info
->connection_state
= KTLS_CONN_TX_READY
;
158 atomic64_inc(&tx_info
->adap
->chcr_stats
.ktls_tx_ctx
);
162 case KTLS_CONN_TX_READY
:
163 /* nothing to be done here */
167 pr_err("unknown KTLS connection state\n");
170 spin_unlock_bh(&tx_info
->lock
);
172 return tx_info
->connection_state
;
175 * chcr_ktls_act_open_req: creates TCB entry for ipv4 connection.
177 * @tx_info - driver specific tls info.
178 * @atid - connection active tid.
179 * return - send success/failure.
181 static int chcr_ktls_act_open_req(struct sock
*sk
,
182 struct chcr_ktls_info
*tx_info
,
185 struct inet_sock
*inet
= inet_sk(sk
);
186 struct cpl_t6_act_open_req
*cpl6
;
187 struct cpl_act_open_req
*cpl
;
194 skb
= alloc_skb(len
, GFP_KERNEL
);
197 /* mark it a control pkt */
198 set_wr_txq(skb
, CPL_PRIORITY_CONTROL
, tx_info
->port_id
);
200 cpl6
= __skb_put_zero(skb
, len
);
201 cpl
= (struct cpl_act_open_req
*)cpl6
;
203 qid_atid
= TID_QID_V(tx_info
->rx_qid
) |
205 OPCODE_TID(cpl
) = htonl(MK_OPCODE_TID(CPL_ACT_OPEN_REQ
, qid_atid
));
206 cpl
->local_port
= inet
->inet_sport
;
207 cpl
->peer_port
= inet
->inet_dport
;
208 cpl
->local_ip
= inet
->inet_rcv_saddr
;
209 cpl
->peer_ip
= inet
->inet_daddr
;
211 /* fill first 64 bit option field. */
212 options
= TCAM_BYPASS_F
| ULP_MODE_V(ULP_MODE_NONE
) | NON_OFFLOAD_F
|
213 SMAC_SEL_V(tx_info
->smt_idx
) | TX_CHAN_V(tx_info
->tx_chan
);
214 cpl
->opt0
= cpu_to_be64(options
);
216 /* next 64 bit option field. */
218 TX_QUEUE_V(tx_info
->adap
->params
.tp
.tx_modq
[tx_info
->tx_chan
]);
219 cpl
->opt2
= htonl(options
);
221 return cxgb4_l2t_send(tx_info
->netdev
, skb
, tx_info
->l2te
);
225 * chcr_ktls_act_open_req6: creates TCB entry for ipv6 connection.
227 * @tx_info - driver specific tls info.
228 * @atid - connection active tid.
229 * return - send success/failure.
231 static int chcr_ktls_act_open_req6(struct sock
*sk
,
232 struct chcr_ktls_info
*tx_info
,
235 struct inet_sock
*inet
= inet_sk(sk
);
236 struct cpl_t6_act_open_req6
*cpl6
;
237 struct cpl_act_open_req6
*cpl
;
244 skb
= alloc_skb(len
, GFP_KERNEL
);
247 /* mark it a control pkt */
248 set_wr_txq(skb
, CPL_PRIORITY_CONTROL
, tx_info
->port_id
);
250 cpl6
= __skb_put_zero(skb
, len
);
251 cpl
= (struct cpl_act_open_req6
*)cpl6
;
253 qid_atid
= TID_QID_V(tx_info
->rx_qid
) | TID_TID_V(atid
);
254 OPCODE_TID(cpl
) = htonl(MK_OPCODE_TID(CPL_ACT_OPEN_REQ6
, qid_atid
));
255 cpl
->local_port
= inet
->inet_sport
;
256 cpl
->peer_port
= inet
->inet_dport
;
257 cpl
->local_ip_hi
= *(__be64
*)&sk
->sk_v6_rcv_saddr
.in6_u
.u6_addr8
[0];
258 cpl
->local_ip_lo
= *(__be64
*)&sk
->sk_v6_rcv_saddr
.in6_u
.u6_addr8
[8];
259 cpl
->peer_ip_hi
= *(__be64
*)&sk
->sk_v6_daddr
.in6_u
.u6_addr8
[0];
260 cpl
->peer_ip_lo
= *(__be64
*)&sk
->sk_v6_daddr
.in6_u
.u6_addr8
[8];
262 /* first 64 bit option field. */
263 options
= TCAM_BYPASS_F
| ULP_MODE_V(ULP_MODE_NONE
) | NON_OFFLOAD_F
|
264 SMAC_SEL_V(tx_info
->smt_idx
) | TX_CHAN_V(tx_info
->tx_chan
);
265 cpl
->opt0
= cpu_to_be64(options
);
266 /* next 64 bit option field. */
268 TX_QUEUE_V(tx_info
->adap
->params
.tp
.tx_modq
[tx_info
->tx_chan
]);
269 cpl
->opt2
= htonl(options
);
271 return cxgb4_l2t_send(tx_info
->netdev
, skb
, tx_info
->l2te
);
275 * chcr_setup_connection: create a TCB entry so that TP will form tcp packets.
277 * @tx_info - driver specific tls info.
278 * return: NET_TX_OK/NET_XMIT_DROP
280 static int chcr_setup_connection(struct sock
*sk
,
281 struct chcr_ktls_info
*tx_info
)
283 struct tid_info
*t
= &tx_info
->adap
->tids
;
286 atid
= cxgb4_alloc_atid(t
, tx_info
);
290 tx_info
->atid
= atid
;
291 tx_info
->ip_family
= sk
->sk_family
;
293 if (sk
->sk_family
== AF_INET
||
294 (sk
->sk_family
== AF_INET6
&& !sk
->sk_ipv6only
&&
295 ipv6_addr_type(&sk
->sk_v6_daddr
) == IPV6_ADDR_MAPPED
)) {
296 tx_info
->ip_family
= AF_INET
;
297 ret
= chcr_ktls_act_open_req(sk
, tx_info
, atid
);
299 tx_info
->ip_family
= AF_INET6
;
301 cxgb4_clip_get(tx_info
->netdev
,
302 (const u32
*)&sk
->sk_v6_rcv_saddr
.in6_u
.u6_addr8
,
306 ret
= chcr_ktls_act_open_req6(sk
, tx_info
, atid
);
309 /* if return type is NET_XMIT_CN, msg will be sent but delayed, mark ret
310 * success, if any other return type clear atid and return that failure.
313 if (ret
== NET_XMIT_CN
)
316 cxgb4_free_atid(t
, atid
);
320 /* update the connection state */
321 chcr_ktls_update_connection_state(tx_info
, KTLS_CONN_ACT_OPEN_REQ
);
327 * chcr_set_tcb_field: update tcb fields.
328 * @tx_info - driver specific tls info.
330 * @mask - TCB word related mask.
331 * @val - TCB word related value.
332 * @no_reply - set 1 if not looking for TP response.
334 static int chcr_set_tcb_field(struct chcr_ktls_info
*tx_info
, u16 word
,
335 u64 mask
, u64 val
, int no_reply
)
337 struct cpl_set_tcb_field
*req
;
340 skb
= alloc_skb(sizeof(struct cpl_set_tcb_field
), GFP_ATOMIC
);
344 req
= (struct cpl_set_tcb_field
*)__skb_put_zero(skb
, sizeof(*req
));
345 INIT_TP_WR_CPL(req
, CPL_SET_TCB_FIELD
, tx_info
->tid
);
346 req
->reply_ctrl
= htons(QUEUENO_V(tx_info
->rx_qid
) |
347 NO_REPLY_V(no_reply
));
348 req
->word_cookie
= htons(TCB_WORD_V(word
));
349 req
->mask
= cpu_to_be64(mask
);
350 req
->val
= cpu_to_be64(val
);
352 set_wr_txq(skb
, CPL_PRIORITY_CONTROL
, tx_info
->port_id
);
353 return cxgb4_ofld_send(tx_info
->netdev
, skb
);
357 * chcr_ktls_mark_tcb_close: mark tcb state to CLOSE
358 * @tx_info - driver specific tls info.
359 * return: NET_TX_OK/NET_XMIT_DROP.
361 static int chcr_ktls_mark_tcb_close(struct chcr_ktls_info
*tx_info
)
363 return chcr_set_tcb_field(tx_info
, TCB_T_STATE_W
,
364 TCB_T_STATE_V(TCB_T_STATE_M
),
365 CHCR_TCB_STATE_CLOSED
, 1);
369 * chcr_ktls_dev_del: call back for tls_dev_del.
370 * Remove the tid and l2t entry and close the connection.
371 * it per connection basis.
372 * @netdev - net device.
373 * @tls_cts - tls context.
374 * @direction - TX/RX crypto direction
376 static void chcr_ktls_dev_del(struct net_device
*netdev
,
377 struct tls_context
*tls_ctx
,
378 enum tls_offload_ctx_dir direction
)
380 struct chcr_ktls_ofld_ctx_tx
*tx_ctx
=
381 chcr_get_ktls_tx_context(tls_ctx
);
382 struct chcr_ktls_info
*tx_info
= tx_ctx
->chcr_info
;
389 spin_lock(&tx_info
->lock
);
390 tx_info
->connection_state
= KTLS_CONN_CLOSED
;
391 spin_unlock(&tx_info
->lock
);
393 /* clear l2t entry */
395 cxgb4_l2t_release(tx_info
->l2te
);
397 /* clear clip entry */
398 if (tx_info
->ip_family
== AF_INET6
)
399 cxgb4_clip_release(netdev
,
400 (const u32
*)&sk
->sk_v6_daddr
.in6_u
.u6_addr8
,
404 if (tx_info
->tid
!= -1) {
405 /* clear tcb state and then release tid */
406 chcr_ktls_mark_tcb_close(tx_info
);
407 cxgb4_remove_tid(&tx_info
->adap
->tids
, tx_info
->tx_chan
,
408 tx_info
->tid
, tx_info
->ip_family
);
411 atomic64_inc(&tx_info
->adap
->chcr_stats
.ktls_tx_connection_close
);
413 tx_ctx
->chcr_info
= NULL
;
417 * chcr_ktls_dev_add: call back for tls_dev_add.
418 * Create a tcb entry for TP. Also add l2t entry for the connection. And
419 * generate keys & save those keys locally.
420 * @netdev - net device.
421 * @tls_cts - tls context.
422 * @direction - TX/RX crypto direction
423 * return: SUCCESS/FAILURE.
425 static int chcr_ktls_dev_add(struct net_device
*netdev
, struct sock
*sk
,
426 enum tls_offload_ctx_dir direction
,
427 struct tls_crypto_info
*crypto_info
,
428 u32 start_offload_tcp_sn
)
430 struct tls_context
*tls_ctx
= tls_get_ctx(sk
);
431 struct chcr_ktls_ofld_ctx_tx
*tx_ctx
;
432 struct chcr_ktls_info
*tx_info
;
433 struct dst_entry
*dst
;
434 struct adapter
*adap
;
435 struct port_info
*pi
;
440 tx_ctx
= chcr_get_ktls_tx_context(tls_ctx
);
442 pi
= netdev_priv(netdev
);
444 if (direction
== TLS_OFFLOAD_CTX_DIR_RX
) {
445 pr_err("not expecting for RX direction\n");
449 if (tx_ctx
->chcr_info
) {
454 tx_info
= kvzalloc(sizeof(*tx_info
), GFP_KERNEL
);
460 spin_lock_init(&tx_info
->lock
);
462 /* clear connection state */
463 spin_lock(&tx_info
->lock
);
464 tx_info
->connection_state
= KTLS_CONN_CLOSED
;
465 spin_unlock(&tx_info
->lock
);
468 /* initialize tid and atid to -1, 0 is a also a valid id. */
472 tx_info
->adap
= adap
;
473 tx_info
->netdev
= netdev
;
474 tx_info
->first_qset
= pi
->first_qset
;
475 tx_info
->tx_chan
= pi
->tx_chan
;
476 tx_info
->smt_idx
= pi
->smt_idx
;
477 tx_info
->port_id
= pi
->port_id
;
479 tx_info
->rx_qid
= chcr_get_first_rx_qid(adap
);
480 if (unlikely(tx_info
->rx_qid
< 0))
483 tx_info
->prev_seq
= start_offload_tcp_sn
;
484 tx_info
->tcp_start_seq_number
= start_offload_tcp_sn
;
486 /* save crypto keys */
487 ret
= chcr_ktls_save_keys(tx_info
, crypto_info
, direction
);
492 if (sk
->sk_family
== AF_INET
||
493 (sk
->sk_family
== AF_INET6
&& !sk
->sk_ipv6only
&&
494 ipv6_addr_type(&sk
->sk_v6_daddr
) == IPV6_ADDR_MAPPED
)) {
495 memcpy(daaddr
, &sk
->sk_daddr
, 4);
497 memcpy(daaddr
, sk
->sk_v6_daddr
.in6_u
.u6_addr8
, 16);
500 /* get the l2t index */
501 dst
= sk_dst_get(sk
);
503 pr_err("DST entry not found\n");
506 n
= dst_neigh_lookup(dst
, daaddr
);
508 pr_err("neighbour not found\n");
512 tx_info
->l2te
= cxgb4_l2t_get(adap
->l2t
, n
, n
->dev
, 0);
517 if (!tx_info
->l2te
) {
518 pr_err("l2t entry not found\n");
522 tx_ctx
->chcr_info
= tx_info
;
524 /* create a filter and call cxgb4_l2t_send to send the packet out, which
525 * will take care of updating l2t entry in hw if not already done.
527 ret
= chcr_setup_connection(sk
, tx_info
);
531 atomic64_inc(&adap
->chcr_stats
.ktls_tx_connection_open
);
536 atomic64_inc(&adap
->chcr_stats
.ktls_tx_connection_fail
);
540 static const struct tlsdev_ops chcr_ktls_ops
= {
541 .tls_dev_add
= chcr_ktls_dev_add
,
542 .tls_dev_del
= chcr_ktls_dev_del
,
546 * chcr_enable_ktls: add NETIF_F_HW_TLS_TX flag in all the ports.
548 void chcr_enable_ktls(struct adapter
*adap
)
550 struct net_device
*netdev
;
553 for_each_port(adap
, i
) {
554 netdev
= adap
->port
[i
];
555 netdev
->features
|= NETIF_F_HW_TLS_TX
;
556 netdev
->hw_features
|= NETIF_F_HW_TLS_TX
;
557 netdev
->tlsdev_ops
= &chcr_ktls_ops
;
562 * chcr_disable_ktls: remove NETIF_F_HW_TLS_TX flag from all the ports.
564 void chcr_disable_ktls(struct adapter
*adap
)
566 struct net_device
*netdev
;
569 for_each_port(adap
, i
) {
570 netdev
= adap
->port
[i
];
571 netdev
->features
&= ~NETIF_F_HW_TLS_TX
;
572 netdev
->hw_features
&= ~NETIF_F_HW_TLS_TX
;
573 netdev
->tlsdev_ops
= NULL
;
578 * chcr_init_tcb_fields: Initialize tcb fields to handle TCP seq number
580 * @tx_info - driver specific tls info.
581 * return: NET_TX_OK/NET_XMIT_DROP
583 static int chcr_init_tcb_fields(struct chcr_ktls_info
*tx_info
)
587 /* set tcb in offload and bypass */
589 chcr_set_tcb_field(tx_info
, TCB_T_FLAGS_W
,
590 TCB_T_FLAGS_V(TF_CORE_BYPASS_F
| TF_NON_OFFLOAD_F
),
591 TCB_T_FLAGS_V(TF_CORE_BYPASS_F
), 1);
594 /* reset snd_una and snd_next fields in tcb */
595 ret
= chcr_set_tcb_field(tx_info
, TCB_SND_UNA_RAW_W
,
596 TCB_SND_NXT_RAW_V(TCB_SND_NXT_RAW_M
) |
597 TCB_SND_UNA_RAW_V(TCB_SND_UNA_RAW_M
),
603 ret
= chcr_set_tcb_field(tx_info
, TCB_SND_MAX_RAW_W
,
604 TCB_SND_MAX_RAW_V(TCB_SND_MAX_RAW_M
),
609 /* update l2t index and request for tp reply to confirm tcb is
610 * initialised to handle tx traffic.
612 ret
= chcr_set_tcb_field(tx_info
, TCB_L2T_IX_W
,
613 TCB_L2T_IX_V(TCB_L2T_IX_M
),
614 TCB_L2T_IX_V(tx_info
->l2te
->idx
), 0);
619 * chcr_ktls_cpl_act_open_rpl: connection reply received from TP.
621 int chcr_ktls_cpl_act_open_rpl(struct adapter
*adap
, unsigned char *input
)
623 const struct cpl_act_open_rpl
*p
= (void *)input
;
624 struct chcr_ktls_info
*tx_info
= NULL
;
625 unsigned int atid
, tid
, status
;
629 status
= AOPEN_STATUS_G(ntohl(p
->atid_status
));
630 atid
= TID_TID_G(AOPEN_ATID_G(ntohl(p
->atid_status
)));
633 tx_info
= lookup_atid(t
, atid
);
635 if (!tx_info
|| tx_info
->atid
!= atid
) {
636 pr_err("tx_info or atid is not correct\n");
642 cxgb4_insert_tid(t
, tx_info
, tx_info
->tid
, tx_info
->ip_family
);
644 cxgb4_free_atid(t
, atid
);
646 /* update the connection state */
647 chcr_ktls_update_connection_state(tx_info
,
648 KTLS_CONN_ACT_OPEN_RPL
);
654 * chcr_ktls_cpl_set_tcb_rpl: TCB reply received from TP.
656 int chcr_ktls_cpl_set_tcb_rpl(struct adapter
*adap
, unsigned char *input
)
658 const struct cpl_set_tcb_rpl
*p
= (void *)input
;
659 struct chcr_ktls_info
*tx_info
= NULL
;
666 tx_info
= lookup_tid(t
, tid
);
667 if (!tx_info
|| tx_info
->tid
!= tid
) {
668 pr_err("tx_info or atid is not correct\n");
671 /* update the connection state */
672 chcr_ktls_update_connection_state(tx_info
, KTLS_CONN_SET_TCB_RPL
);
677 * chcr_write_cpl_set_tcb_ulp: update tcb values.
678 * TCB is responsible to create tcp headers, so all the related values
679 * should be correctly updated.
680 * @tx_info - driver specific tls info.
681 * @q - tx queue on which packet is going out.
682 * @tid - TCB identifier.
683 * @pos - current index where should we start writing.
685 * @mask - TCB word related mask.
686 * @val - TCB word related value.
687 * @reply - set 1 if looking for TP response.
688 * return - next position to write.
690 static void *chcr_write_cpl_set_tcb_ulp(struct chcr_ktls_info
*tx_info
,
691 struct sge_eth_txq
*q
, u32 tid
,
692 void *pos
, u16 word
, u64 mask
,
695 struct cpl_set_tcb_field_core
*cpl
;
696 struct ulptx_idata
*idata
;
697 struct ulp_txpkt
*txpkt
;
698 void *save_pos
= NULL
;
702 left
= (void *)q
->q
.stat
- pos
;
703 if (unlikely(left
< CHCR_SET_TCB_FIELD_LEN
)) {
713 txpkt
->cmd_dest
= htonl(ULPTX_CMD_V(ULP_TX_PKT
) | ULP_TXPKT_DEST_V(0));
714 txpkt
->len
= htonl(DIV_ROUND_UP(CHCR_SET_TCB_FIELD_LEN
, 16));
716 /* ULPTX_IDATA sub-command */
717 idata
= (struct ulptx_idata
*)(txpkt
+ 1);
718 idata
->cmd_more
= htonl(ULPTX_CMD_V(ULP_TX_SC_IMM
));
719 idata
->len
= htonl(sizeof(*cpl
));
723 /* CPL_SET_TCB_FIELD */
724 OPCODE_TID(cpl
) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD
, tid
));
725 cpl
->reply_ctrl
= htons(QUEUENO_V(tx_info
->rx_qid
) |
727 cpl
->word_cookie
= htons(TCB_WORD_V(word
));
728 cpl
->mask
= cpu_to_be64(mask
);
729 cpl
->val
= cpu_to_be64(val
);
732 idata
= (struct ulptx_idata
*)(cpl
+ 1);
733 idata
->cmd_more
= htonl(ULPTX_CMD_V(ULP_TX_SC_NOOP
));
734 idata
->len
= htonl(0);
737 pos
= chcr_copy_to_txd(buf
, &q
->q
, save_pos
,
738 CHCR_SET_TCB_FIELD_LEN
);
740 /* check again if we are at the end of the queue */
741 if (left
== CHCR_SET_TCB_FIELD_LEN
)
751 * chcr_ktls_xmit_tcb_cpls: update tcb entry so that TP will create the header
752 * with updated values like tcp seq, ack, window etc.
753 * @tx_info - driver specific tls info.
758 * return: NETDEV_TX_BUSY/NET_TX_OK.
760 static int chcr_ktls_xmit_tcb_cpls(struct chcr_ktls_info
*tx_info
,
761 struct sge_eth_txq
*q
, u64 tcp_seq
,
762 u64 tcp_ack
, u64 tcp_win
)
764 bool first_wr
= ((tx_info
->prev_ack
== 0) && (tx_info
->prev_win
== 0));
765 u32 len
, cpl
= 0, ndesc
, wr_len
;
766 struct fw_ulptx_wr
*wr
;
770 wr_len
= sizeof(*wr
);
771 /* there can be max 4 cpls, check if we have enough credits */
772 len
= wr_len
+ 4 * roundup(CHCR_SET_TCB_FIELD_LEN
, 16);
773 ndesc
= DIV_ROUND_UP(len
, 64);
775 credits
= chcr_txq_avail(&q
->q
) - ndesc
;
776 if (unlikely(credits
< 0)) {
777 chcr_eth_txq_stop(q
);
778 return NETDEV_TX_BUSY
;
781 pos
= &q
->q
.desc
[q
->q
.pidx
];
782 /* make space for WR, we'll fill it later when we know all the cpls
783 * being sent out and have complete length.
787 /* update tx_max if its a re-transmit or the first wr */
788 if (first_wr
|| tcp_seq
!= tx_info
->prev_seq
) {
789 pos
= chcr_write_cpl_set_tcb_ulp(tx_info
, q
, tx_info
->tid
, pos
,
791 TCB_TX_MAX_V(TCB_TX_MAX_M
),
792 TCB_TX_MAX_V(tcp_seq
), 0);
795 /* reset snd una if it's a re-transmit pkt */
796 if (tcp_seq
!= tx_info
->prev_seq
) {
798 pos
= chcr_write_cpl_set_tcb_ulp(tx_info
, q
, tx_info
->tid
, pos
,
802 TCB_SND_UNA_RAW_V(0), 0);
803 atomic64_inc(&tx_info
->adap
->chcr_stats
.ktls_tx_ooo
);
807 if (first_wr
|| tx_info
->prev_ack
!= tcp_ack
) {
808 pos
= chcr_write_cpl_set_tcb_ulp(tx_info
, q
, tx_info
->tid
, pos
,
810 TCB_RCV_NXT_V(TCB_RCV_NXT_M
),
811 TCB_RCV_NXT_V(tcp_ack
), 0);
812 tx_info
->prev_ack
= tcp_ack
;
815 /* update receive window */
816 if (first_wr
|| tx_info
->prev_win
!= tcp_win
) {
817 pos
= chcr_write_cpl_set_tcb_ulp(tx_info
, q
, tx_info
->tid
, pos
,
819 TCB_RCV_WND_V(TCB_RCV_WND_M
),
820 TCB_RCV_WND_V(tcp_win
), 0);
821 tx_info
->prev_win
= tcp_win
;
826 /* get the actual length */
827 len
= wr_len
+ cpl
* roundup(CHCR_SET_TCB_FIELD_LEN
, 16);
829 wr
->op_to_compl
= htonl(FW_WR_OP_V(FW_ULPTX_WR
));
831 /* fill len in wr field */
832 wr
->flowid_len16
= htonl(FW_WR_LEN16_V(DIV_ROUND_UP(len
, 16)));
834 ndesc
= DIV_ROUND_UP(len
, 64);
835 chcr_txq_advance(&q
->q
, ndesc
);
836 cxgb4_ring_tx_db(tx_info
->adap
, &q
->q
, ndesc
);
843 * @nskb - new skb where the frags to be added.
844 * @skb - old skb from which frags will be copied.
846 static void chcr_ktls_skb_copy(struct sk_buff
*skb
, struct sk_buff
*nskb
)
850 for (i
= 0; i
< skb_shinfo(skb
)->nr_frags
; i
++) {
851 skb_shinfo(nskb
)->frags
[i
] = skb_shinfo(skb
)->frags
[i
];
852 __skb_frag_ref(&skb_shinfo(nskb
)->frags
[i
]);
855 skb_shinfo(nskb
)->nr_frags
= skb_shinfo(skb
)->nr_frags
;
856 nskb
->len
+= skb
->data_len
;
857 nskb
->data_len
= skb
->data_len
;
858 nskb
->truesize
+= skb
->data_len
;
862 * chcr_ktls_get_tx_flits
863 * returns number of flits to be sent out, it includes key context length, WR
864 * size and skb fragments.
867 chcr_ktls_get_tx_flits(const struct sk_buff
*skb
, unsigned int key_ctx_len
)
869 return chcr_sgl_len(skb_shinfo(skb
)->nr_frags
) +
870 DIV_ROUND_UP(key_ctx_len
+ CHCR_KTLS_WR_SIZE
, 8);
874 * chcr_ktls_check_tcp_options: To check if there is any TCP option availbale
875 * other than timestamp.
876 * @skb - skb contains partial record..
880 chcr_ktls_check_tcp_options(struct tcphdr
*tcp
)
882 int cnt
, opt
, optlen
;
885 cp
= (u_char
*)(tcp
+ 1);
886 cnt
= (tcp
->doff
<< 2) - sizeof(struct tcphdr
);
887 for (; cnt
> 0; cnt
-= optlen
, cp
+= optlen
) {
889 if (opt
== TCPOPT_EOL
)
891 if (opt
== TCPOPT_NOP
) {
897 if (optlen
< 2 || optlen
> cnt
)
911 * chcr_ktls_write_tcp_options : TP can't send out all the options, we need to
912 * send out separately.
913 * @tx_info - driver specific tls info.
914 * @skb - skb contains partial record..
916 * @tx_chan - channel number.
917 * return: NETDEV_TX_OK/NETDEV_TX_BUSY.
920 chcr_ktls_write_tcp_options(struct chcr_ktls_info
*tx_info
, struct sk_buff
*skb
,
921 struct sge_eth_txq
*q
, uint32_t tx_chan
)
923 struct fw_eth_tx_pkt_wr
*wr
;
924 struct cpl_tx_pkt_core
*cpl
;
925 u32 ctrl
, iplen
, maclen
;
935 iplen
= skb_network_header_len(skb
);
936 maclen
= skb_mac_header_len(skb
);
938 /* packet length = eth hdr len + ip hdr len + tcp hdr len
939 * (including options).
941 pktlen
= skb
->len
- skb
->data_len
;
943 ctrl
= sizeof(*cpl
) + pktlen
;
944 len16
= DIV_ROUND_UP(sizeof(*wr
) + ctrl
, 16);
945 /* check how many descriptors needed */
946 ndesc
= DIV_ROUND_UP(len16
, 4);
948 credits
= chcr_txq_avail(&q
->q
) - ndesc
;
949 if (unlikely(credits
< 0)) {
950 chcr_eth_txq_stop(q
);
951 return NETDEV_TX_BUSY
;
954 pos
= &q
->q
.desc
[q
->q
.pidx
];
957 /* Firmware work request header */
958 wr
->op_immdlen
= htonl(FW_WR_OP_V(FW_ETH_TX_PKT_WR
) |
959 FW_WR_IMMDLEN_V(ctrl
));
961 wr
->equiq_to_len16
= htonl(FW_WR_LEN16_V(len16
));
964 cpl
= (void *)(wr
+ 1);
967 cpl
->ctrl0
= htonl(TXPKT_OPCODE_V(CPL_TX_PKT
) | TXPKT_INTF_V(tx_chan
) |
968 TXPKT_PF_V(tx_info
->adap
->pf
));
970 cpl
->len
= htons(pktlen
);
971 /* checksum offload */
976 memcpy(buf
, skb
->data
, pktlen
);
977 if (tx_info
->ip_family
== AF_INET
) {
978 /* we need to correct ip header len */
979 ip
= (struct iphdr
*)(buf
+ maclen
);
980 ip
->tot_len
= htons(pktlen
- maclen
);
982 ip6
= (struct ipv6hdr
*)(buf
+ maclen
);
983 ip6
->payload_len
= htons(pktlen
- maclen
- iplen
);
985 /* now take care of the tcp header, if fin is not set then clear push
986 * bit as well, and if fin is set, it will be sent at the last so we
987 * need to update the tcp sequence number as per the last packet.
989 tcp
= (struct tcphdr
*)(buf
+ maclen
+ iplen
);
994 tcp
->seq
= htonl(tx_info
->prev_seq
);
996 chcr_copy_to_txd(buf
, &q
->q
, pos
, pktlen
);
998 chcr_txq_advance(&q
->q
, ndesc
);
999 cxgb4_ring_tx_db(tx_info
->adap
, &q
->q
, ndesc
);
1003 /* chcr_ktls_skb_shift - Shifts request length paged data from skb to another.
1004 * @tgt- buffer into which tail data gets added
1005 * @skb- buffer from which the paged data comes from
1006 * @shiftlen- shift up to this many bytes
1008 static int chcr_ktls_skb_shift(struct sk_buff
*tgt
, struct sk_buff
*skb
,
1011 skb_frag_t
*fragfrom
, *fragto
;
1014 WARN_ON(shiftlen
> skb
->data_len
);
1019 fragfrom
= &skb_shinfo(skb
)->frags
[from
];
1021 while ((todo
> 0) && (from
< skb_shinfo(skb
)->nr_frags
)) {
1022 fragfrom
= &skb_shinfo(skb
)->frags
[from
];
1023 fragto
= &skb_shinfo(tgt
)->frags
[to
];
1025 if (todo
>= skb_frag_size(fragfrom
)) {
1026 *fragto
= *fragfrom
;
1027 todo
-= skb_frag_size(fragfrom
);
1032 __skb_frag_ref(fragfrom
);
1033 skb_frag_page_copy(fragto
, fragfrom
);
1034 skb_frag_off_copy(fragto
, fragfrom
);
1035 skb_frag_size_set(fragto
, todo
);
1037 skb_frag_off_add(fragfrom
, todo
);
1038 skb_frag_size_sub(fragfrom
, todo
);
1046 /* Ready to "commit" this state change to tgt */
1047 skb_shinfo(tgt
)->nr_frags
= to
;
1049 /* Reposition in the original skb */
1051 while (from
< skb_shinfo(skb
)->nr_frags
)
1052 skb_shinfo(skb
)->frags
[to
++] = skb_shinfo(skb
)->frags
[from
++];
1054 skb_shinfo(skb
)->nr_frags
= to
;
1056 WARN_ON(todo
> 0 && !skb_shinfo(skb
)->nr_frags
);
1058 skb
->len
-= shiftlen
;
1059 skb
->data_len
-= shiftlen
;
1060 skb
->truesize
-= shiftlen
;
1061 tgt
->len
+= shiftlen
;
1062 tgt
->data_len
+= shiftlen
;
1063 tgt
->truesize
+= shiftlen
;
1069 * chcr_ktls_xmit_wr_complete: This sends out the complete record. If an skb
1070 * received has partial end part of the record, send out the complete record, so
1071 * that crypto block will be able to generate TAG/HASH.
1072 * @skb - segment which has complete or partial end part.
1073 * @tx_info - driver specific tls info.
1076 * @tcp_push - tcp push bit.
1077 * @mss - segment size.
1078 * return: NETDEV_TX_BUSY/NET_TX_OK.
1080 static int chcr_ktls_xmit_wr_complete(struct sk_buff
*skb
,
1081 struct chcr_ktls_info
*tx_info
,
1082 struct sge_eth_txq
*q
, u32 tcp_seq
,
1083 bool tcp_push
, u32 mss
)
1085 u32 len16
, wr_mid
= 0, flits
= 0, ndesc
, cipher_start
;
1086 struct adapter
*adap
= tx_info
->adap
;
1087 int credits
, left
, last_desc
;
1088 struct tx_sw_desc
*sgl_sdesc
;
1089 struct cpl_tx_data
*tx_data
;
1090 struct cpl_tx_sec_pdu
*cpl
;
1091 struct ulptx_idata
*idata
;
1092 struct ulp_txpkt
*ulptx
;
1093 struct fw_ulptx_wr
*wr
;
1097 /* get the number of flits required */
1098 flits
= chcr_ktls_get_tx_flits(skb
, tx_info
->key_ctx_len
);
1099 /* number of descriptors */
1100 ndesc
= chcr_flits_to_desc(flits
);
1101 /* check if enough credits available */
1102 credits
= chcr_txq_avail(&q
->q
) - ndesc
;
1103 if (unlikely(credits
< 0)) {
1104 chcr_eth_txq_stop(q
);
1105 return NETDEV_TX_BUSY
;
1108 if (unlikely(credits
< ETHTXQ_STOP_THRES
)) {
1109 /* Credits are below the threshold vaues, stop the queue after
1110 * injecting the Work Request for this packet.
1112 chcr_eth_txq_stop(q
);
1113 wr_mid
|= FW_WR_EQUEQ_F
| FW_WR_EQUIQ_F
;
1116 last_desc
= q
->q
.pidx
+ ndesc
- 1;
1117 if (last_desc
>= q
->q
.size
)
1118 last_desc
-= q
->q
.size
;
1119 sgl_sdesc
= &q
->q
.sdesc
[last_desc
];
1121 if (unlikely(cxgb4_map_skb(adap
->pdev_dev
, skb
, sgl_sdesc
->addr
) < 0)) {
1122 memset(sgl_sdesc
->addr
, 0, sizeof(sgl_sdesc
->addr
));
1124 return NETDEV_TX_BUSY
;
1127 pos
= &q
->q
.desc
[q
->q
.pidx
];
1128 end
= (u64
*)pos
+ flits
;
1131 /* WR will need len16 */
1132 len16
= DIV_ROUND_UP(flits
, 2);
1133 wr
->op_to_compl
= htonl(FW_WR_OP_V(FW_ULPTX_WR
));
1134 wr
->flowid_len16
= htonl(wr_mid
| FW_WR_LEN16_V(len16
));
1139 ulptx
->cmd_dest
= htonl(ULPTX_CMD_V(ULP_TX_PKT
) |
1140 ULP_TXPKT_CHANNELID_V(tx_info
->port_id
) |
1141 ULP_TXPKT_FID_V(q
->q
.cntxt_id
) |
1143 ulptx
->len
= htonl(len16
- 1);
1144 /* ULPTX_IDATA sub-command */
1145 idata
= (struct ulptx_idata
*)(ulptx
+ 1);
1146 idata
->cmd_more
= htonl(ULPTX_CMD_V(ULP_TX_SC_IMM
) | ULP_TX_SC_MORE_F
);
1147 /* idata length will include cpl_tx_sec_pdu + key context size +
1148 * cpl_tx_data header.
1150 idata
->len
= htonl(sizeof(*cpl
) + tx_info
->key_ctx_len
+
1153 cpl
= (struct cpl_tx_sec_pdu
*)(idata
+ 1);
1154 cpl
->op_ivinsrtofst
=
1155 htonl(CPL_TX_SEC_PDU_OPCODE_V(CPL_TX_SEC_PDU
) |
1156 CPL_TX_SEC_PDU_CPLLEN_V(CHCR_CPL_TX_SEC_PDU_LEN_64BIT
) |
1157 CPL_TX_SEC_PDU_PLACEHOLDER_V(1) |
1158 CPL_TX_SEC_PDU_IVINSRTOFST_V(TLS_HEADER_SIZE
+ 1));
1159 cpl
->pldlen
= htonl(skb
->data_len
);
1161 /* encryption should start after tls header size + iv size */
1162 cipher_start
= TLS_HEADER_SIZE
+ tx_info
->iv_size
+ 1;
1164 cpl
->aadstart_cipherstop_hi
=
1165 htonl(CPL_TX_SEC_PDU_AADSTART_V(1) |
1166 CPL_TX_SEC_PDU_AADSTOP_V(TLS_HEADER_SIZE
) |
1167 CPL_TX_SEC_PDU_CIPHERSTART_V(cipher_start
));
1169 /* authentication will also start after tls header + iv size */
1170 cpl
->cipherstop_lo_authinsert
=
1171 htonl(CPL_TX_SEC_PDU_AUTHSTART_V(cipher_start
) |
1172 CPL_TX_SEC_PDU_AUTHSTOP_V(TLS_CIPHER_AES_GCM_128_TAG_SIZE
) |
1173 CPL_TX_SEC_PDU_AUTHINSERT_V(TLS_CIPHER_AES_GCM_128_TAG_SIZE
));
1175 /* These two flits are actually a CPL_TLS_TX_SCMD_FMT. */
1176 cpl
->seqno_numivs
= htonl(tx_info
->scmd0_seqno_numivs
);
1177 cpl
->ivgen_hdrlen
= htonl(tx_info
->scmd0_ivgen_hdrlen
);
1178 cpl
->scmd1
= cpu_to_be64(tx_info
->record_no
);
1181 /* check if space left to fill the keys */
1182 left
= (void *)q
->q
.stat
- pos
;
1184 left
= (void *)end
- (void *)q
->q
.stat
;
1189 pos
= chcr_copy_to_txd(&tx_info
->key_ctx
, &q
->q
, pos
,
1190 tx_info
->key_ctx_len
);
1191 left
= (void *)q
->q
.stat
- pos
;
1194 left
= (void *)end
- (void *)q
->q
.stat
;
1199 tx_data
= (void *)pos
;
1200 OPCODE_TID(tx_data
) = htonl(MK_OPCODE_TID(CPL_TX_DATA
, tx_info
->tid
));
1201 tx_data
->len
= htonl(TX_DATA_MSS_V(mss
) | TX_LENGTH_V(skb
->data_len
));
1203 tx_data
->rsvd
= htonl(tcp_seq
);
1205 tx_data
->flags
= htonl(TX_BYPASS_F
);
1207 tx_data
->flags
|= htonl(TX_PUSH_F
| TX_SHOVE_F
);
1209 /* check left again, it might go beyond queue limit */
1211 left
= (void *)q
->q
.stat
- pos
;
1213 /* check the position again */
1215 left
= (void *)end
- (void *)q
->q
.stat
;
1220 /* send the complete packet except the header */
1221 cxgb4_write_sgl(skb
, &q
->q
, pos
, end
, skb
->len
- skb
->data_len
,
1223 sgl_sdesc
->skb
= skb
;
1225 chcr_txq_advance(&q
->q
, ndesc
);
1226 cxgb4_ring_tx_db(adap
, &q
->q
, ndesc
);
1227 atomic64_inc(&adap
->chcr_stats
.ktls_tx_send_records
);
1233 * chcr_ktls_xmit_wr_short: This is to send out partial records. If its
1234 * a middle part of a record, fetch the prior data to make it 16 byte aligned
1235 * and then only send it out.
1237 * @skb - skb contains partial record..
1238 * @tx_info - driver specific tls info.
1241 * @tcp_push - tcp push bit.
1242 * @mss - segment size.
1243 * @tls_rec_offset - offset from start of the tls record.
1244 * @perior_data - data before the current segment, required to make this record
1246 * @prior_data_len - prior_data length (less than 16)
1247 * return: NETDEV_TX_BUSY/NET_TX_OK.
1249 static int chcr_ktls_xmit_wr_short(struct sk_buff
*skb
,
1250 struct chcr_ktls_info
*tx_info
,
1251 struct sge_eth_txq
*q
,
1252 u32 tcp_seq
, bool tcp_push
, u32 mss
,
1253 u32 tls_rec_offset
, u8
*prior_data
,
1256 struct adapter
*adap
= tx_info
->adap
;
1257 u32 len16
, wr_mid
= 0, cipher_start
;
1258 unsigned int flits
= 0, ndesc
;
1259 int credits
, left
, last_desc
;
1260 struct tx_sw_desc
*sgl_sdesc
;
1261 struct cpl_tx_data
*tx_data
;
1262 struct cpl_tx_sec_pdu
*cpl
;
1263 struct ulptx_idata
*idata
;
1264 struct ulp_txpkt
*ulptx
;
1265 struct fw_ulptx_wr
*wr
;
1270 /* get the number of flits required, it's a partial record so 2 flits
1271 * (AES_BLOCK_SIZE) will be added.
1273 flits
= chcr_ktls_get_tx_flits(skb
, tx_info
->key_ctx_len
) + 2;
1274 /* get the correct 8 byte IV of this record */
1275 iv_record
= cpu_to_be64(tx_info
->iv
+ tx_info
->record_no
);
1276 /* If it's a middle record and not 16 byte aligned to run AES CTR, need
1277 * to make it 16 byte aligned. So atleadt 2 extra flits of immediate
1278 * data will be added.
1282 /* number of descriptors */
1283 ndesc
= chcr_flits_to_desc(flits
);
1284 /* check if enough credits available */
1285 credits
= chcr_txq_avail(&q
->q
) - ndesc
;
1286 if (unlikely(credits
< 0)) {
1287 chcr_eth_txq_stop(q
);
1288 return NETDEV_TX_BUSY
;
1291 if (unlikely(credits
< ETHTXQ_STOP_THRES
)) {
1292 chcr_eth_txq_stop(q
);
1293 wr_mid
|= FW_WR_EQUEQ_F
| FW_WR_EQUIQ_F
;
1296 last_desc
= q
->q
.pidx
+ ndesc
- 1;
1297 if (last_desc
>= q
->q
.size
)
1298 last_desc
-= q
->q
.size
;
1299 sgl_sdesc
= &q
->q
.sdesc
[last_desc
];
1301 if (unlikely(cxgb4_map_skb(adap
->pdev_dev
, skb
, sgl_sdesc
->addr
) < 0)) {
1302 memset(sgl_sdesc
->addr
, 0, sizeof(sgl_sdesc
->addr
));
1304 return NETDEV_TX_BUSY
;
1307 pos
= &q
->q
.desc
[q
->q
.pidx
];
1308 end
= (u64
*)pos
+ flits
;
1311 /* WR will need len16 */
1312 len16
= DIV_ROUND_UP(flits
, 2);
1313 wr
->op_to_compl
= htonl(FW_WR_OP_V(FW_ULPTX_WR
));
1314 wr
->flowid_len16
= htonl(wr_mid
| FW_WR_LEN16_V(len16
));
1319 ulptx
->cmd_dest
= htonl(ULPTX_CMD_V(ULP_TX_PKT
) |
1320 ULP_TXPKT_CHANNELID_V(tx_info
->port_id
) |
1321 ULP_TXPKT_FID_V(q
->q
.cntxt_id
) |
1323 ulptx
->len
= htonl(len16
- 1);
1324 /* ULPTX_IDATA sub-command */
1325 idata
= (struct ulptx_idata
*)(ulptx
+ 1);
1326 idata
->cmd_more
= htonl(ULPTX_CMD_V(ULP_TX_SC_IMM
) | ULP_TX_SC_MORE_F
);
1327 /* idata length will include cpl_tx_sec_pdu + key context size +
1328 * cpl_tx_data header.
1330 idata
->len
= htonl(sizeof(*cpl
) + tx_info
->key_ctx_len
+
1331 sizeof(*tx_data
) + AES_BLOCK_LEN
+ prior_data_len
);
1333 cpl
= (struct cpl_tx_sec_pdu
*)(idata
+ 1);
1334 /* cipher start will have tls header + iv size extra if its a header
1335 * part of tls record. else only 16 byte IV will be added.
1339 (!tls_rec_offset
? TLS_HEADER_SIZE
+ tx_info
->iv_size
: 0);
1341 cpl
->op_ivinsrtofst
=
1342 htonl(CPL_TX_SEC_PDU_OPCODE_V(CPL_TX_SEC_PDU
) |
1343 CPL_TX_SEC_PDU_CPLLEN_V(CHCR_CPL_TX_SEC_PDU_LEN_64BIT
) |
1344 CPL_TX_SEC_PDU_IVINSRTOFST_V(1));
1345 cpl
->pldlen
= htonl(skb
->data_len
+ AES_BLOCK_LEN
+ prior_data_len
);
1346 cpl
->aadstart_cipherstop_hi
=
1347 htonl(CPL_TX_SEC_PDU_CIPHERSTART_V(cipher_start
));
1348 cpl
->cipherstop_lo_authinsert
= 0;
1349 /* These two flits are actually a CPL_TLS_TX_SCMD_FMT. */
1350 cpl
->seqno_numivs
= htonl(tx_info
->scmd0_short_seqno_numivs
);
1351 cpl
->ivgen_hdrlen
= htonl(tx_info
->scmd0_short_ivgen_hdrlen
);
1355 /* check if space left to fill the keys */
1356 left
= (void *)q
->q
.stat
- pos
;
1358 left
= (void *)end
- (void *)q
->q
.stat
;
1363 pos
= chcr_copy_to_txd(&tx_info
->key_ctx
, &q
->q
, pos
,
1364 tx_info
->key_ctx_len
);
1365 left
= (void *)q
->q
.stat
- pos
;
1368 left
= (void *)end
- (void *)q
->q
.stat
;
1373 tx_data
= (void *)pos
;
1374 OPCODE_TID(tx_data
) = htonl(MK_OPCODE_TID(CPL_TX_DATA
, tx_info
->tid
));
1375 tx_data
->len
= htonl(TX_DATA_MSS_V(mss
) |
1376 TX_LENGTH_V(skb
->data_len
+ prior_data_len
));
1377 tx_data
->rsvd
= htonl(tcp_seq
);
1378 tx_data
->flags
= htonl(TX_BYPASS_F
);
1380 tx_data
->flags
|= htonl(TX_PUSH_F
| TX_SHOVE_F
);
1382 /* check left again, it might go beyond queue limit */
1384 left
= (void *)q
->q
.stat
- pos
;
1386 /* check the position again */
1388 left
= (void *)end
- (void *)q
->q
.stat
;
1392 /* copy the 16 byte IV for AES-CTR, which includes 4 bytes of salt, 8
1393 * bytes of actual IV and 4 bytes of 16 byte-sequence.
1395 memcpy(pos
, tx_info
->key_ctx
.salt
, tx_info
->salt_size
);
1396 memcpy(pos
+ tx_info
->salt_size
, &iv_record
, tx_info
->iv_size
);
1397 *(__be32
*)(pos
+ tx_info
->salt_size
+ tx_info
->iv_size
) =
1398 htonl(2 + (tls_rec_offset
? ((tls_rec_offset
-
1399 (TLS_HEADER_SIZE
+ tx_info
->iv_size
)) / AES_BLOCK_LEN
) : 0));
1402 /* Prior_data_len will always be less than 16 bytes, fill the
1403 * prio_data_len after AES_CTRL_BLOCK and clear the remaining length
1407 pos
= chcr_copy_to_txd(prior_data
, &q
->q
, pos
, 16);
1408 /* send the complete packet except the header */
1409 cxgb4_write_sgl(skb
, &q
->q
, pos
, end
, skb
->len
- skb
->data_len
,
1411 sgl_sdesc
->skb
= skb
;
1413 chcr_txq_advance(&q
->q
, ndesc
);
1414 cxgb4_ring_tx_db(adap
, &q
->q
, ndesc
);
1420 * chcr_ktls_tx_plaintxt: This handler will take care of the records which has
1421 * only plain text (only tls header and iv)
1422 * @tx_info - driver specific tls info.
1423 * @skb - skb contains partial record..
1425 * @mss - segment size.
1426 * @tcp_push - tcp push bit.
1428 * @port_id : port number
1429 * @perior_data - data before the current segment, required to make this record
1431 * @prior_data_len - prior_data length (less than 16)
1432 * return: NETDEV_TX_BUSY/NET_TX_OK.
1434 static int chcr_ktls_tx_plaintxt(struct chcr_ktls_info
*tx_info
,
1435 struct sk_buff
*skb
, u32 tcp_seq
, u32 mss
,
1436 bool tcp_push
, struct sge_eth_txq
*q
,
1437 u32 port_id
, u8
*prior_data
,
1440 int credits
, left
, len16
, last_desc
;
1441 unsigned int flits
= 0, ndesc
;
1442 struct tx_sw_desc
*sgl_sdesc
;
1443 struct cpl_tx_data
*tx_data
;
1444 struct ulptx_idata
*idata
;
1445 struct ulp_txpkt
*ulptx
;
1446 struct fw_ulptx_wr
*wr
;
1451 flits
= DIV_ROUND_UP(CHCR_PLAIN_TX_DATA_LEN
, 8);
1452 flits
+= chcr_sgl_len(skb_shinfo(skb
)->nr_frags
);
1455 /* WR will need len16 */
1456 len16
= DIV_ROUND_UP(flits
, 2);
1457 /* check how many descriptors needed */
1458 ndesc
= DIV_ROUND_UP(flits
, 8);
1460 credits
= chcr_txq_avail(&q
->q
) - ndesc
;
1461 if (unlikely(credits
< 0)) {
1462 chcr_eth_txq_stop(q
);
1463 return NETDEV_TX_BUSY
;
1466 if (unlikely(credits
< ETHTXQ_STOP_THRES
)) {
1467 chcr_eth_txq_stop(q
);
1468 wr_mid
|= FW_WR_EQUEQ_F
| FW_WR_EQUIQ_F
;
1471 last_desc
= q
->q
.pidx
+ ndesc
- 1;
1472 if (last_desc
>= q
->q
.size
)
1473 last_desc
-= q
->q
.size
;
1474 sgl_sdesc
= &q
->q
.sdesc
[last_desc
];
1476 if (unlikely(cxgb4_map_skb(tx_info
->adap
->pdev_dev
, skb
,
1477 sgl_sdesc
->addr
) < 0)) {
1478 memset(sgl_sdesc
->addr
, 0, sizeof(sgl_sdesc
->addr
));
1480 return NETDEV_TX_BUSY
;
1483 pos
= &q
->q
.desc
[q
->q
.pidx
];
1484 end
= (u64
*)pos
+ flits
;
1487 wr
->op_to_compl
= htonl(FW_WR_OP_V(FW_ULPTX_WR
));
1488 wr
->flowid_len16
= htonl(wr_mid
| FW_WR_LEN16_V(len16
));
1492 ulptx
= (struct ulp_txpkt
*)(wr
+ 1);
1493 ulptx
->cmd_dest
= htonl(ULPTX_CMD_V(ULP_TX_PKT
) |
1494 ULP_TXPKT_DATAMODIFY_V(0) |
1495 ULP_TXPKT_CHANNELID_V(tx_info
->port_id
) |
1496 ULP_TXPKT_DEST_V(0) |
1497 ULP_TXPKT_FID_V(q
->q
.cntxt_id
) | ULP_TXPKT_RO_V(1));
1498 ulptx
->len
= htonl(len16
- 1);
1499 /* ULPTX_IDATA sub-command */
1500 idata
= (struct ulptx_idata
*)(ulptx
+ 1);
1501 idata
->cmd_more
= htonl(ULPTX_CMD_V(ULP_TX_SC_IMM
) | ULP_TX_SC_MORE_F
);
1502 idata
->len
= htonl(sizeof(*tx_data
) + prior_data_len
);
1504 tx_data
= (struct cpl_tx_data
*)(idata
+ 1);
1505 OPCODE_TID(tx_data
) = htonl(MK_OPCODE_TID(CPL_TX_DATA
, tx_info
->tid
));
1506 tx_data
->len
= htonl(TX_DATA_MSS_V(mss
) |
1507 TX_LENGTH_V(skb
->data_len
+ prior_data_len
));
1508 /* set tcp seq number */
1509 tx_data
->rsvd
= htonl(tcp_seq
);
1510 tx_data
->flags
= htonl(TX_BYPASS_F
);
1512 tx_data
->flags
|= htonl(TX_PUSH_F
| TX_SHOVE_F
);
1515 /* apart from prior_data_len, we should set remaining part of 16 bytes
1519 pos
= chcr_copy_to_txd(prior_data
, &q
->q
, pos
, 16);
1521 /* check left again, it might go beyond queue limit */
1522 left
= (void *)q
->q
.stat
- pos
;
1524 /* check the position again */
1526 left
= (void *)end
- (void *)q
->q
.stat
;
1530 /* send the complete packet including the header */
1531 cxgb4_write_sgl(skb
, &q
->q
, pos
, end
, skb
->len
- skb
->data_len
,
1533 sgl_sdesc
->skb
= skb
;
1535 chcr_txq_advance(&q
->q
, ndesc
);
1536 cxgb4_ring_tx_db(tx_info
->adap
, &q
->q
, ndesc
);
1541 * chcr_ktls_copy_record_in_skb
1542 * @nskb - new skb where the frags to be added.
1543 * @record - specific record which has complete 16k record in frags.
1545 static void chcr_ktls_copy_record_in_skb(struct sk_buff
*nskb
,
1546 struct tls_record_info
*record
)
1550 for (i
= 0; i
< record
->num_frags
; i
++) {
1551 skb_shinfo(nskb
)->frags
[i
] = record
->frags
[i
];
1552 /* increase the frag ref count */
1553 __skb_frag_ref(&skb_shinfo(nskb
)->frags
[i
]);
1556 skb_shinfo(nskb
)->nr_frags
= record
->num_frags
;
1557 nskb
->data_len
= record
->len
;
1558 nskb
->len
+= record
->len
;
1559 nskb
->truesize
+= record
->len
;
1563 * chcr_ktls_update_snd_una: Reset the SEND_UNA. It will be done to avoid
1564 * sending the same segment again. It will discard the segment which is before
1565 * the current tx max.
1566 * @tx_info - driver specific tls info.
1568 * return: NET_TX_OK/NET_XMIT_DROP.
1570 static int chcr_ktls_update_snd_una(struct chcr_ktls_info
*tx_info
,
1571 struct sge_eth_txq
*q
)
1573 struct fw_ulptx_wr
*wr
;
1579 len
= sizeof(*wr
) + roundup(CHCR_SET_TCB_FIELD_LEN
, 16);
1580 ndesc
= DIV_ROUND_UP(len
, 64);
1582 credits
= chcr_txq_avail(&q
->q
) - ndesc
;
1583 if (unlikely(credits
< 0)) {
1584 chcr_eth_txq_stop(q
);
1585 return NETDEV_TX_BUSY
;
1588 pos
= &q
->q
.desc
[q
->q
.pidx
];
1592 wr
->op_to_compl
= htonl(FW_WR_OP_V(FW_ULPTX_WR
));
1594 /* fill len in wr field */
1595 wr
->flowid_len16
= htonl(FW_WR_LEN16_V(DIV_ROUND_UP(len
, 16)));
1599 pos
= chcr_write_cpl_set_tcb_ulp(tx_info
, q
, tx_info
->tid
, pos
,
1601 TCB_SND_UNA_RAW_V(TCB_SND_UNA_RAW_M
),
1602 TCB_SND_UNA_RAW_V(0), 0);
1604 chcr_txq_advance(&q
->q
, ndesc
);
1605 cxgb4_ring_tx_db(tx_info
->adap
, &q
->q
, ndesc
);
1611 * chcr_end_part_handler: This handler will handle the record which
1612 * is complete or if record's end part is received. T6 adapter has a issue that
1613 * it can't send out TAG with partial record so if its an end part then we have
1614 * to send TAG as well and for which we need to fetch the complete record and
1615 * send it to crypto module.
1616 * @tx_info - driver specific tls info.
1617 * @skb - skb contains partial record.
1618 * @record - complete record of 16K size.
1620 * @mss - segment size in which TP needs to chop a packet.
1621 * @tcp_push_no_fin - tcp push if fin is not set.
1623 * @tls_end_offset - offset from end of the record.
1624 * @last wr : check if this is the last part of the skb going out.
1625 * return: NETDEV_TX_OK/NETDEV_TX_BUSY.
1627 static int chcr_end_part_handler(struct chcr_ktls_info
*tx_info
,
1628 struct sk_buff
*skb
,
1629 struct tls_record_info
*record
,
1630 u32 tcp_seq
, int mss
, bool tcp_push_no_fin
,
1631 struct sge_eth_txq
*q
,
1632 u32 tls_end_offset
, bool last_wr
)
1634 struct sk_buff
*nskb
= NULL
;
1635 /* check if it is a complete record */
1636 if (tls_end_offset
== record
->len
) {
1638 atomic64_inc(&tx_info
->adap
->chcr_stats
.ktls_tx_complete_pkts
);
1640 dev_kfree_skb_any(skb
);
1642 nskb
= alloc_skb(0, GFP_KERNEL
);
1644 return NETDEV_TX_BUSY
;
1645 /* copy complete record in skb */
1646 chcr_ktls_copy_record_in_skb(nskb
, record
);
1647 /* packet is being sent from the beginning, update the tcp_seq
1650 tcp_seq
= tls_record_start_seq(record
);
1651 /* reset snd una, so the middle record won't send the already
1654 if (chcr_ktls_update_snd_una(tx_info
, q
))
1656 atomic64_inc(&tx_info
->adap
->chcr_stats
.ktls_tx_end_pkts
);
1659 if (chcr_ktls_xmit_wr_complete(nskb
, tx_info
, q
, tcp_seq
,
1660 (last_wr
&& tcp_push_no_fin
),
1666 dev_kfree_skb_any(nskb
);
1667 return NETDEV_TX_BUSY
;
1671 * chcr_short_record_handler: This handler will take care of the records which
1672 * doesn't have end part (1st part or the middle part(/s) of a record). In such
1673 * cases, AES CTR will be used in place of AES GCM to send out partial packet.
1674 * This partial record might be the first part of the record, or the middle
1675 * part. In case of middle record we should fetch the prior data to make it 16
1676 * byte aligned. If it has a partial tls header or iv then get to the start of
1677 * tls header. And if it has partial TAG, then remove the complete TAG and send
1679 * There is one more possibility that it gets a partial header, send that
1680 * portion as a plaintext.
1681 * @tx_info - driver specific tls info.
1682 * @skb - skb contains partial record..
1683 * @record - complete record of 16K size.
1685 * @mss - segment size in which TP needs to chop a packet.
1686 * @tcp_push_no_fin - tcp push if fin is not set.
1688 * @tls_end_offset - offset from end of the record.
1689 * return: NETDEV_TX_OK/NETDEV_TX_BUSY.
1691 static int chcr_short_record_handler(struct chcr_ktls_info
*tx_info
,
1692 struct sk_buff
*skb
,
1693 struct tls_record_info
*record
,
1694 u32 tcp_seq
, int mss
, bool tcp_push_no_fin
,
1695 struct sge_eth_txq
*q
, u32 tls_end_offset
)
1697 u32 tls_rec_offset
= tcp_seq
- tls_record_start_seq(record
);
1698 u8 prior_data
[16] = {0};
1699 u32 prior_data_len
= 0;
1702 /* check if the skb is ending in middle of tag/HASH, its a big
1703 * trouble, send the packet before the HASH.
1705 int remaining_record
= tls_end_offset
- skb
->data_len
;
1707 if (remaining_record
> 0 &&
1708 remaining_record
< TLS_CIPHER_AES_GCM_128_TAG_SIZE
) {
1709 int trimmed_len
= skb
->data_len
-
1710 (TLS_CIPHER_AES_GCM_128_TAG_SIZE
- remaining_record
);
1711 struct sk_buff
*tmp_skb
= NULL
;
1712 /* don't process the pkt if it is only a partial tag */
1713 if (skb
->data_len
< TLS_CIPHER_AES_GCM_128_TAG_SIZE
)
1716 WARN_ON(trimmed_len
> skb
->data_len
);
1718 /* shift to those many bytes */
1719 tmp_skb
= alloc_skb(0, GFP_KERNEL
);
1720 if (unlikely(!tmp_skb
))
1723 chcr_ktls_skb_shift(tmp_skb
, skb
, trimmed_len
);
1724 /* free the last trimmed portion */
1725 dev_kfree_skb_any(skb
);
1727 atomic64_inc(&tx_info
->adap
->chcr_stats
.ktls_tx_trimmed_pkts
);
1729 data_len
= skb
->data_len
;
1730 /* check if the middle record's start point is 16 byte aligned. CTR
1731 * needs 16 byte aligned start point to start encryption.
1733 if (tls_rec_offset
) {
1734 /* there is an offset from start, means its a middle record */
1737 if (tls_rec_offset
< (TLS_HEADER_SIZE
+ tx_info
->iv_size
)) {
1738 prior_data_len
= tls_rec_offset
;
1744 (TLS_HEADER_SIZE
+ tx_info
->iv_size
))
1746 remaining
= tls_rec_offset
- prior_data_len
;
1749 /* if prior_data_len is not zero, means we need to fetch prior
1750 * data to make this record 16 byte aligned, or we need to reach
1753 if (prior_data_len
) {
1758 int frag_size
= 0, frag_delta
= 0;
1760 while (remaining
> 0) {
1761 frag_size
= skb_frag_size(&record
->frags
[i
]);
1762 if (remaining
< frag_size
)
1765 remaining
-= frag_size
;
1768 f
= &record
->frags
[i
];
1769 vaddr
= kmap_atomic(skb_frag_page(f
));
1771 data
= vaddr
+ skb_frag_off(f
) + remaining
;
1772 frag_delta
= skb_frag_size(f
) - remaining
;
1774 if (frag_delta
>= prior_data_len
) {
1775 memcpy(prior_data
, data
, prior_data_len
);
1776 kunmap_atomic(vaddr
);
1778 memcpy(prior_data
, data
, frag_delta
);
1779 kunmap_atomic(vaddr
);
1780 /* get the next page */
1781 f
= &record
->frags
[i
+ 1];
1782 vaddr
= kmap_atomic(skb_frag_page(f
));
1783 data
= vaddr
+ skb_frag_off(f
);
1784 memcpy(prior_data
+ frag_delta
,
1785 data
, (prior_data_len
- frag_delta
));
1786 kunmap_atomic(vaddr
);
1788 /* reset tcp_seq as per the prior_data_required len */
1789 tcp_seq
-= prior_data_len
;
1790 /* include prio_data_len for further calculation.
1792 data_len
+= prior_data_len
;
1794 /* reset snd una, so the middle record won't send the already
1797 if (chcr_ktls_update_snd_una(tx_info
, q
))
1799 atomic64_inc(&tx_info
->adap
->chcr_stats
.ktls_tx_middle_pkts
);
1801 /* Else means, its a partial first part of the record. Check if
1802 * its only the header, don't need to send for encryption then.
1804 if (data_len
<= TLS_HEADER_SIZE
+ tx_info
->iv_size
) {
1805 if (chcr_ktls_tx_plaintxt(tx_info
, skb
, tcp_seq
, mss
,
1814 atomic64_inc(&tx_info
->adap
->chcr_stats
.ktls_tx_start_pkts
);
1817 if (chcr_ktls_xmit_wr_short(skb
, tx_info
, q
, tcp_seq
, tcp_push_no_fin
,
1818 mss
, tls_rec_offset
, prior_data
,
1825 dev_kfree_skb_any(skb
);
1826 return NETDEV_TX_BUSY
;
1829 /* nic tls TX handler */
1830 int chcr_ktls_xmit(struct sk_buff
*skb
, struct net_device
*dev
)
1832 struct chcr_ktls_ofld_ctx_tx
*tx_ctx
;
1833 struct tcphdr
*th
= tcp_hdr(skb
);
1834 int data_len
, qidx
, ret
= 0, mss
;
1835 struct tls_record_info
*record
;
1836 struct chcr_stats_debug
*stats
;
1837 struct chcr_ktls_info
*tx_info
;
1838 u32 tls_end_offset
, tcp_seq
;
1839 struct tls_context
*tls_ctx
;
1840 struct sk_buff
*local_skb
;
1841 int new_connection_state
;
1842 struct sge_eth_txq
*q
;
1843 struct adapter
*adap
;
1844 unsigned long flags
;
1846 tcp_seq
= ntohl(th
->seq
);
1848 mss
= skb_is_gso(skb
) ? skb_shinfo(skb
)->gso_size
: skb
->data_len
;
1850 /* check if we haven't set it for ktls offload */
1851 if (!skb
->sk
|| !tls_is_sk_tx_device_offloaded(skb
->sk
))
1854 tls_ctx
= tls_get_ctx(skb
->sk
);
1855 if (unlikely(tls_ctx
->netdev
!= dev
))
1858 tx_ctx
= chcr_get_ktls_tx_context(tls_ctx
);
1859 tx_info
= tx_ctx
->chcr_info
;
1861 if (unlikely(!tx_info
))
1864 /* check the connection state, we don't need to pass new connection
1865 * state, state machine will check and update the new state if it is
1866 * stuck due to responses not received from HW.
1867 * Start the tx handling only if state is KTLS_CONN_TX_READY.
1869 new_connection_state
= chcr_ktls_update_connection_state(tx_info
, 0);
1870 if (new_connection_state
!= KTLS_CONN_TX_READY
)
1873 /* don't touch the original skb, make a new skb to extract each records
1874 * and send them separately.
1876 local_skb
= alloc_skb(0, GFP_KERNEL
);
1878 if (unlikely(!local_skb
))
1879 return NETDEV_TX_BUSY
;
1881 adap
= tx_info
->adap
;
1882 stats
= &adap
->chcr_stats
;
1884 qidx
= skb
->queue_mapping
;
1885 q
= &adap
->sge
.ethtxq
[qidx
+ tx_info
->first_qset
];
1886 cxgb4_reclaim_completed_tx(adap
, &q
->q
, true);
1887 /* if tcp options are set but finish is not send the options first */
1888 if (!th
->fin
&& chcr_ktls_check_tcp_options(th
)) {
1889 ret
= chcr_ktls_write_tcp_options(tx_info
, skb
, q
,
1892 return NETDEV_TX_BUSY
;
1895 ret
= chcr_ktls_xmit_tcb_cpls(tx_info
, q
, ntohl(th
->seq
),
1899 dev_kfree_skb_any(local_skb
);
1900 return NETDEV_TX_BUSY
;
1903 /* copy skb contents into local skb */
1904 chcr_ktls_skb_copy(skb
, local_skb
);
1906 /* go through the skb and send only one record at a time. */
1907 data_len
= skb
->data_len
;
1908 /* TCP segments can be in received either complete or partial.
1909 * chcr_end_part_handler will handle cases if complete record or end
1910 * part of the record is received. Incase of partial end part of record,
1911 * we will send the complete record again.
1917 cxgb4_reclaim_completed_tx(adap
, &q
->q
, true);
1919 spin_lock_irqsave(&tx_ctx
->base
.lock
, flags
);
1920 /* fetch the tls record */
1921 record
= tls_get_record(&tx_ctx
->base
, tcp_seq
,
1922 &tx_info
->record_no
);
1923 /* By the time packet reached to us, ACK is received, and record
1924 * won't be found in that case, handle it gracefully.
1926 if (unlikely(!record
)) {
1927 spin_unlock_irqrestore(&tx_ctx
->base
.lock
, flags
);
1928 atomic64_inc(&stats
->ktls_tx_drop_no_sync_data
);
1932 if (unlikely(tls_record_is_start_marker(record
))) {
1933 spin_unlock_irqrestore(&tx_ctx
->base
.lock
, flags
);
1934 atomic64_inc(&stats
->ktls_tx_skip_no_sync_data
);
1938 /* increase page reference count of the record, so that there
1939 * won't be any chance of page free in middle if in case stack
1940 * receives ACK and try to delete the record.
1942 for (i
= 0; i
< record
->num_frags
; i
++)
1943 __skb_frag_ref(&record
->frags
[i
]);
1945 spin_unlock_irqrestore(&tx_ctx
->base
.lock
, flags
);
1947 tls_end_offset
= record
->end_seq
- tcp_seq
;
1949 pr_debug("seq 0x%x, end_seq 0x%x prev_seq 0x%x, datalen 0x%x\n",
1950 tcp_seq
, record
->end_seq
, tx_info
->prev_seq
, data_len
);
1951 /* if a tls record is finishing in this SKB */
1952 if (tls_end_offset
<= data_len
) {
1953 struct sk_buff
*nskb
= NULL
;
1955 if (tls_end_offset
< data_len
) {
1956 nskb
= alloc_skb(0, GFP_KERNEL
);
1957 if (unlikely(!nskb
)) {
1962 chcr_ktls_skb_shift(nskb
, local_skb
,
1965 /* its the only record in this skb, directly
1970 ret
= chcr_end_part_handler(tx_info
, nskb
, record
,
1972 (!th
->fin
&& th
->psh
), q
,
1974 (nskb
== local_skb
));
1976 if (ret
&& nskb
!= local_skb
)
1977 dev_kfree_skb_any(local_skb
);
1979 data_len
-= tls_end_offset
;
1980 /* tcp_seq increment is required to handle next record.
1982 tcp_seq
+= tls_end_offset
;
1984 ret
= chcr_short_record_handler(tx_info
, local_skb
,
1985 record
, tcp_seq
, mss
,
1986 (!th
->fin
&& th
->psh
),
1991 /* clear the frag ref count which increased locally before */
1992 for (i
= 0; i
< record
->num_frags
; i
++) {
1993 /* clear the frag ref count */
1994 __skb_frag_unref(&record
->frags
[i
]);
1996 /* if any failure, come out from the loop. */
1999 /* length should never be less than 0 */
2000 WARN_ON(data_len
< 0);
2002 } while (data_len
> 0);
2004 tx_info
->prev_seq
= ntohl(th
->seq
) + skb
->data_len
;
2006 atomic64_inc(&stats
->ktls_tx_encrypted_packets
);
2007 atomic64_add(skb
->data_len
, &stats
->ktls_tx_encrypted_bytes
);
2009 /* tcp finish is set, send a separate tcp msg including all the options
2013 chcr_ktls_write_tcp_options(tx_info
, skb
, q
, tx_info
->tx_chan
);
2016 dev_kfree_skb_any(skb
);
2017 return NETDEV_TX_OK
;
2019 #endif /* CONFIG_CHELSIO_TLS_DEVICE */