]>
Commit | Line | Data |
---|---|---|
2303f994 PK |
1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* Multipath TCP | |
3 | * | |
4 | * Copyright (c) 2017 - 2019, Intel Corporation. | |
5 | */ | |
6 | ||
79c0949e PK |
7 | #define pr_fmt(fmt) "MPTCP: " fmt |
8 | ||
2303f994 PK |
9 | #include <linux/kernel.h> |
10 | #include <linux/module.h> | |
11 | #include <linux/netdevice.h> | |
f296234c | 12 | #include <crypto/algapi.h> |
2303f994 PK |
13 | #include <net/sock.h> |
14 | #include <net/inet_common.h> | |
15 | #include <net/inet_hashtables.h> | |
16 | #include <net/protocol.h> | |
17 | #include <net/tcp.h> | |
cec37a6e PK |
18 | #if IS_ENABLED(CONFIG_MPTCP_IPV6) |
19 | #include <net/ip6_route.h> | |
20 | #endif | |
2303f994 PK |
21 | #include <net/mptcp.h> |
22 | #include "protocol.h" | |
fc518953 FW |
23 | #include "mib.h" |
24 | ||
25 | static void SUBFLOW_REQ_INC_STATS(struct request_sock *req, | |
26 | enum linux_mptcp_mib_field field) | |
27 | { | |
28 | MPTCP_INC_STATS(sock_net(req_to_sk(req)), field); | |
29 | } | |
2303f994 | 30 | |
79c0949e PK |
31 | static int subflow_rebuild_header(struct sock *sk) |
32 | { | |
33 | struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk); | |
ec3edaa7 | 34 | int local_id, err = 0; |
79c0949e PK |
35 | |
36 | if (subflow->request_mptcp && !subflow->token) { | |
37 | pr_debug("subflow=%p", sk); | |
38 | err = mptcp_token_new_connect(sk); | |
ec3edaa7 PK |
39 | } else if (subflow->request_join && !subflow->local_nonce) { |
40 | struct mptcp_sock *msk = (struct mptcp_sock *)subflow->conn; | |
41 | ||
42 | pr_debug("subflow=%p", sk); | |
43 | ||
44 | do { | |
45 | get_random_bytes(&subflow->local_nonce, sizeof(u32)); | |
46 | } while (!subflow->local_nonce); | |
47 | ||
48 | if (subflow->local_id) | |
49 | goto out; | |
50 | ||
51 | local_id = mptcp_pm_get_local_id(msk, (struct sock_common *)sk); | |
52 | if (local_id < 0) | |
53 | return -EINVAL; | |
54 | ||
55 | subflow->local_id = local_id; | |
79c0949e PK |
56 | } |
57 | ||
ec3edaa7 | 58 | out: |
79c0949e PK |
59 | if (err) |
60 | return err; | |
61 | ||
62 | return subflow->icsk_af_ops->rebuild_header(sk); | |
63 | } | |
64 | ||
65 | static void subflow_req_destructor(struct request_sock *req) | |
66 | { | |
67 | struct mptcp_subflow_request_sock *subflow_req = mptcp_subflow_rsk(req); | |
68 | ||
69 | pr_debug("subflow_req=%p", subflow_req); | |
70 | ||
71 | if (subflow_req->mp_capable) | |
72 | mptcp_token_destroy_request(subflow_req->token); | |
73 | tcp_request_sock_ops.destructor(req); | |
74 | } | |
75 | ||
f296234c PK |
76 | static void subflow_generate_hmac(u64 key1, u64 key2, u32 nonce1, u32 nonce2, |
77 | void *hmac) | |
78 | { | |
79 | u8 msg[8]; | |
80 | ||
81 | put_unaligned_be32(nonce1, &msg[0]); | |
82 | put_unaligned_be32(nonce2, &msg[4]); | |
83 | ||
84 | mptcp_crypto_hmac_sha(key1, key2, msg, 8, hmac); | |
85 | } | |
86 | ||
87 | /* validate received token and create truncated hmac and nonce for SYN-ACK */ | |
88 | static bool subflow_token_join_request(struct request_sock *req, | |
89 | const struct sk_buff *skb) | |
90 | { | |
91 | struct mptcp_subflow_request_sock *subflow_req = mptcp_subflow_rsk(req); | |
92 | u8 hmac[MPTCPOPT_HMAC_LEN]; | |
93 | struct mptcp_sock *msk; | |
94 | int local_id; | |
95 | ||
96 | msk = mptcp_token_get_sock(subflow_req->token); | |
97 | if (!msk) { | |
fc518953 | 98 | SUBFLOW_REQ_INC_STATS(req, MPTCP_MIB_JOINNOTOKEN); |
f296234c PK |
99 | return false; |
100 | } | |
101 | ||
102 | local_id = mptcp_pm_get_local_id(msk, (struct sock_common *)req); | |
103 | if (local_id < 0) { | |
104 | sock_put((struct sock *)msk); | |
105 | return false; | |
106 | } | |
107 | subflow_req->local_id = local_id; | |
108 | ||
109 | get_random_bytes(&subflow_req->local_nonce, sizeof(u32)); | |
110 | ||
111 | subflow_generate_hmac(msk->local_key, msk->remote_key, | |
112 | subflow_req->local_nonce, | |
113 | subflow_req->remote_nonce, hmac); | |
114 | ||
115 | subflow_req->thmac = get_unaligned_be64(hmac); | |
116 | ||
117 | sock_put((struct sock *)msk); | |
118 | return true; | |
119 | } | |
120 | ||
cec37a6e PK |
121 | static void subflow_init_req(struct request_sock *req, |
122 | const struct sock *sk_listener, | |
123 | struct sk_buff *skb) | |
124 | { | |
125 | struct mptcp_subflow_context *listener = mptcp_subflow_ctx(sk_listener); | |
126 | struct mptcp_subflow_request_sock *subflow_req = mptcp_subflow_rsk(req); | |
cfde141e | 127 | struct mptcp_options_received mp_opt; |
cec37a6e PK |
128 | |
129 | pr_debug("subflow_req=%p, listener=%p", subflow_req, listener); | |
130 | ||
cfde141e | 131 | mptcp_get_options(skb, &mp_opt); |
cec37a6e PK |
132 | |
133 | subflow_req->mp_capable = 0; | |
f296234c | 134 | subflow_req->mp_join = 0; |
cec37a6e PK |
135 | |
136 | #ifdef CONFIG_TCP_MD5SIG | |
137 | /* no MPTCP if MD5SIG is enabled on this socket or we may run out of | |
138 | * TCP option space. | |
139 | */ | |
140 | if (rcu_access_pointer(tcp_sk(sk_listener)->md5sig_info)) | |
141 | return; | |
142 | #endif | |
143 | ||
cfde141e | 144 | if (mp_opt.mp_capable) { |
fc518953 FW |
145 | SUBFLOW_REQ_INC_STATS(req, MPTCP_MIB_MPCAPABLEPASSIVE); |
146 | ||
cfde141e | 147 | if (mp_opt.mp_join) |
fc518953 | 148 | return; |
cfde141e | 149 | } else if (mp_opt.mp_join) { |
fc518953 FW |
150 | SUBFLOW_REQ_INC_STATS(req, MPTCP_MIB_JOINSYNRX); |
151 | } | |
f296234c | 152 | |
cfde141e | 153 | if (mp_opt.mp_capable && listener->request_mptcp) { |
79c0949e PK |
154 | int err; |
155 | ||
156 | err = mptcp_token_new_request(req); | |
157 | if (err == 0) | |
158 | subflow_req->mp_capable = 1; | |
159 | ||
648ef4b8 | 160 | subflow_req->ssn_offset = TCP_SKB_CB(skb)->seq; |
cfde141e | 161 | } else if (mp_opt.mp_join && listener->request_mptcp) { |
ec3edaa7 | 162 | subflow_req->ssn_offset = TCP_SKB_CB(skb)->seq; |
f296234c | 163 | subflow_req->mp_join = 1; |
cfde141e PA |
164 | subflow_req->backup = mp_opt.backup; |
165 | subflow_req->remote_id = mp_opt.join_id; | |
166 | subflow_req->token = mp_opt.token; | |
167 | subflow_req->remote_nonce = mp_opt.nonce; | |
f296234c PK |
168 | pr_debug("token=%u, remote_nonce=%u", subflow_req->token, |
169 | subflow_req->remote_nonce); | |
170 | if (!subflow_token_join_request(req, skb)) { | |
171 | subflow_req->mp_join = 0; | |
172 | // @@ need to trigger RST | |
173 | } | |
cec37a6e PK |
174 | } |
175 | } | |
176 | ||
177 | static void subflow_v4_init_req(struct request_sock *req, | |
178 | const struct sock *sk_listener, | |
179 | struct sk_buff *skb) | |
180 | { | |
181 | tcp_rsk(req)->is_mptcp = 1; | |
182 | ||
183 | tcp_request_sock_ipv4_ops.init_req(req, sk_listener, skb); | |
184 | ||
185 | subflow_init_req(req, sk_listener, skb); | |
186 | } | |
187 | ||
188 | #if IS_ENABLED(CONFIG_MPTCP_IPV6) | |
189 | static void subflow_v6_init_req(struct request_sock *req, | |
190 | const struct sock *sk_listener, | |
191 | struct sk_buff *skb) | |
192 | { | |
193 | tcp_rsk(req)->is_mptcp = 1; | |
194 | ||
195 | tcp_request_sock_ipv6_ops.init_req(req, sk_listener, skb); | |
196 | ||
197 | subflow_init_req(req, sk_listener, skb); | |
198 | } | |
199 | #endif | |
200 | ||
ec3edaa7 PK |
201 | /* validate received truncated hmac and create hmac for third ACK */ |
202 | static bool subflow_thmac_valid(struct mptcp_subflow_context *subflow) | |
203 | { | |
204 | u8 hmac[MPTCPOPT_HMAC_LEN]; | |
205 | u64 thmac; | |
206 | ||
207 | subflow_generate_hmac(subflow->remote_key, subflow->local_key, | |
208 | subflow->remote_nonce, subflow->local_nonce, | |
209 | hmac); | |
210 | ||
211 | thmac = get_unaligned_be64(hmac); | |
212 | pr_debug("subflow=%p, token=%u, thmac=%llu, subflow->thmac=%llu\n", | |
213 | subflow, subflow->token, | |
214 | (unsigned long long)thmac, | |
215 | (unsigned long long)subflow->thmac); | |
216 | ||
217 | return thmac == subflow->thmac; | |
218 | } | |
219 | ||
cec37a6e PK |
220 | static void subflow_finish_connect(struct sock *sk, const struct sk_buff *skb) |
221 | { | |
222 | struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk); | |
cfde141e | 223 | struct mptcp_options_received mp_opt; |
c3c123d1 | 224 | struct sock *parent = subflow->conn; |
263e1201 | 225 | struct tcp_sock *tp = tcp_sk(sk); |
cec37a6e PK |
226 | |
227 | subflow->icsk_af_ops->sk_rx_dst_set(sk, skb); | |
228 | ||
1200832c | 229 | if (inet_sk_state_load(parent) == TCP_SYN_SENT) { |
c3c123d1 DC |
230 | inet_sk_state_store(parent, TCP_ESTABLISHED); |
231 | parent->sk_state_change(parent); | |
232 | } | |
233 | ||
263e1201 PA |
234 | /* be sure no special action on any packet other than syn-ack */ |
235 | if (subflow->conn_finished) | |
236 | return; | |
237 | ||
238 | subflow->conn_finished = 1; | |
239 | ||
cfde141e PA |
240 | mptcp_get_options(skb, &mp_opt); |
241 | if (subflow->request_mptcp && mp_opt.mp_capable) { | |
263e1201 PA |
242 | subflow->mp_capable = 1; |
243 | subflow->can_ack = 1; | |
cfde141e | 244 | subflow->remote_key = mp_opt.sndr_key; |
263e1201 PA |
245 | pr_debug("subflow=%p, remote_key=%llu", subflow, |
246 | subflow->remote_key); | |
cfde141e | 247 | } else if (subflow->request_join && mp_opt.mp_join) { |
263e1201 | 248 | subflow->mp_join = 1; |
cfde141e PA |
249 | subflow->thmac = mp_opt.thmac; |
250 | subflow->remote_nonce = mp_opt.nonce; | |
263e1201 PA |
251 | pr_debug("subflow=%p, thmac=%llu, remote_nonce=%u", subflow, |
252 | subflow->thmac, subflow->remote_nonce); | |
253 | } else if (subflow->request_mptcp) { | |
254 | tp->is_mptcp = 0; | |
255 | } | |
256 | ||
257 | if (!tp->is_mptcp) | |
ec3edaa7 PK |
258 | return; |
259 | ||
260 | if (subflow->mp_capable) { | |
cec37a6e PK |
261 | pr_debug("subflow=%p, remote_key=%llu", mptcp_subflow_ctx(sk), |
262 | subflow->remote_key); | |
263 | mptcp_finish_connect(sk); | |
648ef4b8 MM |
264 | |
265 | if (skb) { | |
266 | pr_debug("synack seq=%u", TCP_SKB_CB(skb)->seq); | |
267 | subflow->ssn_offset = TCP_SKB_CB(skb)->seq; | |
268 | } | |
ec3edaa7 PK |
269 | } else if (subflow->mp_join) { |
270 | pr_debug("subflow=%p, thmac=%llu, remote_nonce=%u", | |
271 | subflow, subflow->thmac, | |
272 | subflow->remote_nonce); | |
273 | if (!subflow_thmac_valid(subflow)) { | |
fc518953 | 274 | MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_JOINACKMAC); |
ec3edaa7 PK |
275 | subflow->mp_join = 0; |
276 | goto do_reset; | |
277 | } | |
278 | ||
279 | subflow_generate_hmac(subflow->local_key, subflow->remote_key, | |
280 | subflow->local_nonce, | |
281 | subflow->remote_nonce, | |
282 | subflow->hmac); | |
283 | ||
284 | if (skb) | |
285 | subflow->ssn_offset = TCP_SKB_CB(skb)->seq; | |
286 | ||
287 | if (!mptcp_finish_join(sk)) | |
288 | goto do_reset; | |
289 | ||
fc518953 | 290 | MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_JOINSYNACKRX); |
ec3edaa7 PK |
291 | } else { |
292 | do_reset: | |
293 | tcp_send_active_reset(sk, GFP_ATOMIC); | |
294 | tcp_done(sk); | |
cec37a6e PK |
295 | } |
296 | } | |
297 | ||
298 | static struct request_sock_ops subflow_request_sock_ops; | |
299 | static struct tcp_request_sock_ops subflow_request_sock_ipv4_ops; | |
300 | ||
301 | static int subflow_v4_conn_request(struct sock *sk, struct sk_buff *skb) | |
302 | { | |
303 | struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk); | |
304 | ||
305 | pr_debug("subflow=%p", subflow); | |
306 | ||
307 | /* Never answer to SYNs sent to broadcast or multicast */ | |
308 | if (skb_rtable(skb)->rt_flags & (RTCF_BROADCAST | RTCF_MULTICAST)) | |
309 | goto drop; | |
310 | ||
311 | return tcp_conn_request(&subflow_request_sock_ops, | |
312 | &subflow_request_sock_ipv4_ops, | |
313 | sk, skb); | |
314 | drop: | |
315 | tcp_listendrop(sk); | |
316 | return 0; | |
317 | } | |
318 | ||
319 | #if IS_ENABLED(CONFIG_MPTCP_IPV6) | |
320 | static struct tcp_request_sock_ops subflow_request_sock_ipv6_ops; | |
321 | static struct inet_connection_sock_af_ops subflow_v6_specific; | |
322 | static struct inet_connection_sock_af_ops subflow_v6m_specific; | |
323 | ||
324 | static int subflow_v6_conn_request(struct sock *sk, struct sk_buff *skb) | |
325 | { | |
326 | struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk); | |
327 | ||
328 | pr_debug("subflow=%p", subflow); | |
329 | ||
330 | if (skb->protocol == htons(ETH_P_IP)) | |
331 | return subflow_v4_conn_request(sk, skb); | |
332 | ||
333 | if (!ipv6_unicast_destination(skb)) | |
334 | goto drop; | |
335 | ||
336 | return tcp_conn_request(&subflow_request_sock_ops, | |
337 | &subflow_request_sock_ipv6_ops, sk, skb); | |
338 | ||
339 | drop: | |
340 | tcp_listendrop(sk); | |
341 | return 0; /* don't send reset */ | |
342 | } | |
343 | #endif | |
344 | ||
f296234c PK |
345 | /* validate hmac received in third ACK */ |
346 | static bool subflow_hmac_valid(const struct request_sock *req, | |
cfde141e | 347 | const struct mptcp_options_received *mp_opt) |
f296234c PK |
348 | { |
349 | const struct mptcp_subflow_request_sock *subflow_req; | |
350 | u8 hmac[MPTCPOPT_HMAC_LEN]; | |
351 | struct mptcp_sock *msk; | |
352 | bool ret; | |
353 | ||
354 | subflow_req = mptcp_subflow_rsk(req); | |
355 | msk = mptcp_token_get_sock(subflow_req->token); | |
356 | if (!msk) | |
357 | return false; | |
358 | ||
359 | subflow_generate_hmac(msk->remote_key, msk->local_key, | |
360 | subflow_req->remote_nonce, | |
361 | subflow_req->local_nonce, hmac); | |
362 | ||
363 | ret = true; | |
cfde141e | 364 | if (crypto_memneq(hmac, mp_opt->hmac, sizeof(hmac))) |
f296234c PK |
365 | ret = false; |
366 | ||
367 | sock_put((struct sock *)msk); | |
368 | return ret; | |
369 | } | |
370 | ||
df1036da FW |
371 | static void mptcp_sock_destruct(struct sock *sk) |
372 | { | |
373 | /* if new mptcp socket isn't accepted, it is free'd | |
374 | * from the tcp listener sockets request queue, linked | |
375 | * from req->sk. The tcp socket is released. | |
376 | * This calls the ULP release function which will | |
377 | * also remove the mptcp socket, via | |
378 | * sock_put(ctx->conn). | |
379 | * | |
380 | * Problem is that the mptcp socket will not be in | |
381 | * SYN_RECV state and doesn't have SOCK_DEAD flag. | |
382 | * Both result in warnings from inet_sock_destruct. | |
383 | */ | |
384 | ||
385 | if (sk->sk_state == TCP_SYN_RECV) { | |
386 | sk->sk_state = TCP_CLOSE; | |
387 | WARN_ON_ONCE(sk->sk_socket); | |
388 | sock_orphan(sk); | |
389 | } | |
390 | ||
391 | inet_sock_destruct(sk); | |
392 | } | |
393 | ||
9f5ca6a5 FW |
394 | static void mptcp_force_close(struct sock *sk) |
395 | { | |
396 | inet_sk_state_store(sk, TCP_CLOSE); | |
397 | sk_common_release(sk); | |
398 | } | |
399 | ||
4c8941de PA |
400 | static void subflow_ulp_fallback(struct sock *sk, |
401 | struct mptcp_subflow_context *old_ctx) | |
402 | { | |
403 | struct inet_connection_sock *icsk = inet_csk(sk); | |
404 | ||
405 | mptcp_subflow_tcp_fallback(sk, old_ctx); | |
406 | icsk->icsk_ulp_ops = NULL; | |
407 | rcu_assign_pointer(icsk->icsk_ulp_data, NULL); | |
408 | tcp_sk(sk)->is_mptcp = 0; | |
409 | } | |
410 | ||
cec37a6e PK |
411 | static struct sock *subflow_syn_recv_sock(const struct sock *sk, |
412 | struct sk_buff *skb, | |
413 | struct request_sock *req, | |
414 | struct dst_entry *dst, | |
415 | struct request_sock *req_unhash, | |
416 | bool *own_req) | |
417 | { | |
418 | struct mptcp_subflow_context *listener = mptcp_subflow_ctx(sk); | |
cc7972ea | 419 | struct mptcp_subflow_request_sock *subflow_req; |
cfde141e | 420 | struct mptcp_options_received mp_opt; |
f296234c | 421 | bool fallback_is_fatal = false; |
58b09919 | 422 | struct sock *new_msk = NULL; |
4c8941de | 423 | bool fallback = false; |
cec37a6e PK |
424 | struct sock *child; |
425 | ||
426 | pr_debug("listener=%p, req=%p, conn=%p", listener, req, listener->conn); | |
427 | ||
cfde141e PA |
428 | /* we need later a valid 'mp_capable' value even when options are not |
429 | * parsed | |
430 | */ | |
431 | mp_opt.mp_capable = 0; | |
ae2dd716 FW |
432 | if (tcp_rsk(req)->is_mptcp == 0) |
433 | goto create_child; | |
434 | ||
d22f4988 | 435 | /* if the sk is MP_CAPABLE, we try to fetch the client key */ |
cc7972ea CP |
436 | subflow_req = mptcp_subflow_rsk(req); |
437 | if (subflow_req->mp_capable) { | |
d22f4988 CP |
438 | if (TCP_SKB_CB(skb)->seq != subflow_req->ssn_offset + 1) { |
439 | /* here we can receive and accept an in-window, | |
440 | * out-of-order pkt, which will not carry the MP_CAPABLE | |
441 | * opt even on mptcp enabled paths | |
442 | */ | |
58b09919 | 443 | goto create_msk; |
d22f4988 CP |
444 | } |
445 | ||
cfde141e PA |
446 | mptcp_get_options(skb, &mp_opt); |
447 | if (!mp_opt.mp_capable) { | |
4c8941de | 448 | fallback = true; |
58b09919 | 449 | goto create_child; |
d22f4988 | 450 | } |
58b09919 PA |
451 | |
452 | create_msk: | |
cfde141e | 453 | new_msk = mptcp_sk_clone(listener->conn, &mp_opt, req); |
58b09919 | 454 | if (!new_msk) |
4c8941de | 455 | fallback = true; |
f296234c PK |
456 | } else if (subflow_req->mp_join) { |
457 | fallback_is_fatal = true; | |
cfde141e PA |
458 | mptcp_get_options(skb, &mp_opt); |
459 | if (!mp_opt.mp_join || | |
460 | !subflow_hmac_valid(req, &mp_opt)) { | |
fc518953 | 461 | SUBFLOW_REQ_INC_STATS(req, MPTCP_MIB_JOINACKMAC); |
f296234c | 462 | return NULL; |
fc518953 | 463 | } |
cc7972ea | 464 | } |
cec37a6e | 465 | |
d22f4988 | 466 | create_child: |
cec37a6e PK |
467 | child = listener->icsk_af_ops->syn_recv_sock(sk, skb, req, dst, |
468 | req_unhash, own_req); | |
469 | ||
470 | if (child && *own_req) { | |
79c0949e PK |
471 | struct mptcp_subflow_context *ctx = mptcp_subflow_ctx(child); |
472 | ||
4c8941de PA |
473 | /* we need to fallback on ctx allocation failure and on pre-reqs |
474 | * checking above. In the latter scenario we additionally need | |
475 | * to reset the context to non MPTCP status. | |
79c0949e | 476 | */ |
4c8941de | 477 | if (!ctx || fallback) { |
f296234c PK |
478 | if (fallback_is_fatal) |
479 | goto close_child; | |
4c8941de PA |
480 | |
481 | if (ctx) { | |
482 | subflow_ulp_fallback(child, ctx); | |
483 | kfree_rcu(ctx, rcu); | |
484 | } | |
58b09919 | 485 | goto out; |
f296234c | 486 | } |
79c0949e PK |
487 | |
488 | if (ctx->mp_capable) { | |
58b09919 PA |
489 | /* new mpc subflow takes ownership of the newly |
490 | * created mptcp socket | |
491 | */ | |
df1036da | 492 | new_msk->sk_destruct = mptcp_sock_destruct; |
1b1c7a0e | 493 | mptcp_pm_new_connection(mptcp_sk(new_msk), 1); |
58b09919 PA |
494 | ctx->conn = new_msk; |
495 | new_msk = NULL; | |
fca5c82c PA |
496 | |
497 | /* with OoO packets we can reach here without ingress | |
498 | * mpc option | |
499 | */ | |
cfde141e PA |
500 | ctx->remote_key = mp_opt.sndr_key; |
501 | ctx->fully_established = mp_opt.mp_capable; | |
502 | ctx->can_ack = mp_opt.mp_capable; | |
f296234c PK |
503 | } else if (ctx->mp_join) { |
504 | struct mptcp_sock *owner; | |
505 | ||
506 | owner = mptcp_token_get_sock(ctx->token); | |
507 | if (!owner) | |
508 | goto close_child; | |
509 | ||
510 | ctx->conn = (struct sock *)owner; | |
511 | if (!mptcp_finish_join(child)) | |
512 | goto close_child; | |
fc518953 FW |
513 | |
514 | SUBFLOW_REQ_INC_STATS(req, MPTCP_MIB_JOINACKRX); | |
cec37a6e PK |
515 | } |
516 | } | |
517 | ||
58b09919 PA |
518 | out: |
519 | /* dispose of the left over mptcp master, if any */ | |
520 | if (unlikely(new_msk)) | |
9f5ca6a5 | 521 | mptcp_force_close(new_msk); |
4c8941de PA |
522 | |
523 | /* check for expected invariant - should never trigger, just help | |
524 | * catching eariler subtle bugs | |
525 | */ | |
ac2b47fb | 526 | WARN_ON_ONCE(child && *own_req && tcp_sk(child)->is_mptcp && |
4c8941de PA |
527 | (!mptcp_subflow_ctx(child) || |
528 | !mptcp_subflow_ctx(child)->conn)); | |
cec37a6e | 529 | return child; |
f296234c PK |
530 | |
531 | close_child: | |
532 | tcp_send_active_reset(child, GFP_ATOMIC); | |
533 | inet_csk_prepare_forced_close(child); | |
534 | tcp_done(child); | |
535 | return NULL; | |
cec37a6e PK |
536 | } |
537 | ||
538 | static struct inet_connection_sock_af_ops subflow_specific; | |
539 | ||
648ef4b8 MM |
540 | enum mapping_status { |
541 | MAPPING_OK, | |
542 | MAPPING_INVALID, | |
543 | MAPPING_EMPTY, | |
544 | MAPPING_DATA_FIN | |
545 | }; | |
546 | ||
547 | static u64 expand_seq(u64 old_seq, u16 old_data_len, u64 seq) | |
548 | { | |
549 | if ((u32)seq == (u32)old_seq) | |
550 | return old_seq; | |
551 | ||
552 | /* Assume map covers data not mapped yet. */ | |
553 | return seq | ((old_seq + old_data_len + 1) & GENMASK_ULL(63, 32)); | |
554 | } | |
555 | ||
556 | static void warn_bad_map(struct mptcp_subflow_context *subflow, u32 ssn) | |
557 | { | |
558 | WARN_ONCE(1, "Bad mapping: ssn=%d map_seq=%d map_data_len=%d", | |
559 | ssn, subflow->map_subflow_seq, subflow->map_data_len); | |
560 | } | |
561 | ||
562 | static bool skb_is_fully_mapped(struct sock *ssk, struct sk_buff *skb) | |
563 | { | |
564 | struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk); | |
565 | unsigned int skb_consumed; | |
566 | ||
567 | skb_consumed = tcp_sk(ssk)->copied_seq - TCP_SKB_CB(skb)->seq; | |
568 | if (WARN_ON_ONCE(skb_consumed >= skb->len)) | |
569 | return true; | |
570 | ||
571 | return skb->len - skb_consumed <= subflow->map_data_len - | |
572 | mptcp_subflow_get_map_offset(subflow); | |
573 | } | |
574 | ||
575 | static bool validate_mapping(struct sock *ssk, struct sk_buff *skb) | |
576 | { | |
577 | struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk); | |
578 | u32 ssn = tcp_sk(ssk)->copied_seq - subflow->ssn_offset; | |
579 | ||
580 | if (unlikely(before(ssn, subflow->map_subflow_seq))) { | |
581 | /* Mapping covers data later in the subflow stream, | |
582 | * currently unsupported. | |
583 | */ | |
584 | warn_bad_map(subflow, ssn); | |
585 | return false; | |
586 | } | |
587 | if (unlikely(!before(ssn, subflow->map_subflow_seq + | |
588 | subflow->map_data_len))) { | |
589 | /* Mapping does covers past subflow data, invalid */ | |
590 | warn_bad_map(subflow, ssn + skb->len); | |
591 | return false; | |
592 | } | |
593 | return true; | |
594 | } | |
595 | ||
596 | static enum mapping_status get_mapping_status(struct sock *ssk) | |
597 | { | |
598 | struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk); | |
599 | struct mptcp_ext *mpext; | |
600 | struct sk_buff *skb; | |
601 | u16 data_len; | |
602 | u64 map_seq; | |
603 | ||
604 | skb = skb_peek(&ssk->sk_receive_queue); | |
605 | if (!skb) | |
606 | return MAPPING_EMPTY; | |
607 | ||
608 | mpext = mptcp_get_ext(skb); | |
609 | if (!mpext || !mpext->use_map) { | |
610 | if (!subflow->map_valid && !skb->len) { | |
611 | /* the TCP stack deliver 0 len FIN pkt to the receive | |
612 | * queue, that is the only 0len pkts ever expected here, | |
613 | * and we can admit no mapping only for 0 len pkts | |
614 | */ | |
615 | if (!(TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN)) | |
616 | WARN_ONCE(1, "0len seq %d:%d flags %x", | |
617 | TCP_SKB_CB(skb)->seq, | |
618 | TCP_SKB_CB(skb)->end_seq, | |
619 | TCP_SKB_CB(skb)->tcp_flags); | |
620 | sk_eat_skb(ssk, skb); | |
621 | return MAPPING_EMPTY; | |
622 | } | |
623 | ||
624 | if (!subflow->map_valid) | |
625 | return MAPPING_INVALID; | |
626 | ||
627 | goto validate_seq; | |
628 | } | |
629 | ||
630 | pr_debug("seq=%llu is64=%d ssn=%u data_len=%u data_fin=%d", | |
631 | mpext->data_seq, mpext->dsn64, mpext->subflow_seq, | |
632 | mpext->data_len, mpext->data_fin); | |
633 | ||
634 | data_len = mpext->data_len; | |
635 | if (data_len == 0) { | |
636 | pr_err("Infinite mapping not handled"); | |
fc518953 | 637 | MPTCP_INC_STATS(sock_net(ssk), MPTCP_MIB_INFINITEMAPRX); |
648ef4b8 MM |
638 | return MAPPING_INVALID; |
639 | } | |
640 | ||
641 | if (mpext->data_fin == 1) { | |
642 | if (data_len == 1) { | |
643 | pr_debug("DATA_FIN with no payload"); | |
644 | if (subflow->map_valid) { | |
645 | /* A DATA_FIN might arrive in a DSS | |
646 | * option before the previous mapping | |
647 | * has been fully consumed. Continue | |
648 | * handling the existing mapping. | |
649 | */ | |
650 | skb_ext_del(skb, SKB_EXT_MPTCP); | |
651 | return MAPPING_OK; | |
652 | } else { | |
653 | return MAPPING_DATA_FIN; | |
654 | } | |
655 | } | |
656 | ||
657 | /* Adjust for DATA_FIN using 1 byte of sequence space */ | |
658 | data_len--; | |
659 | } | |
660 | ||
661 | if (!mpext->dsn64) { | |
662 | map_seq = expand_seq(subflow->map_seq, subflow->map_data_len, | |
663 | mpext->data_seq); | |
664 | pr_debug("expanded seq=%llu", subflow->map_seq); | |
665 | } else { | |
666 | map_seq = mpext->data_seq; | |
667 | } | |
668 | ||
669 | if (subflow->map_valid) { | |
670 | /* Allow replacing only with an identical map */ | |
671 | if (subflow->map_seq == map_seq && | |
672 | subflow->map_subflow_seq == mpext->subflow_seq && | |
673 | subflow->map_data_len == data_len) { | |
674 | skb_ext_del(skb, SKB_EXT_MPTCP); | |
675 | return MAPPING_OK; | |
676 | } | |
677 | ||
678 | /* If this skb data are fully covered by the current mapping, | |
679 | * the new map would need caching, which is not supported | |
680 | */ | |
fc518953 FW |
681 | if (skb_is_fully_mapped(ssk, skb)) { |
682 | MPTCP_INC_STATS(sock_net(ssk), MPTCP_MIB_DSSNOMATCH); | |
648ef4b8 | 683 | return MAPPING_INVALID; |
fc518953 | 684 | } |
648ef4b8 MM |
685 | |
686 | /* will validate the next map after consuming the current one */ | |
687 | return MAPPING_OK; | |
688 | } | |
689 | ||
690 | subflow->map_seq = map_seq; | |
691 | subflow->map_subflow_seq = mpext->subflow_seq; | |
692 | subflow->map_data_len = data_len; | |
693 | subflow->map_valid = 1; | |
d22f4988 | 694 | subflow->mpc_map = mpext->mpc_map; |
648ef4b8 MM |
695 | pr_debug("new map seq=%llu subflow_seq=%u data_len=%u", |
696 | subflow->map_seq, subflow->map_subflow_seq, | |
697 | subflow->map_data_len); | |
698 | ||
699 | validate_seq: | |
700 | /* we revalidate valid mapping on new skb, because we must ensure | |
701 | * the current skb is completely covered by the available mapping | |
702 | */ | |
703 | if (!validate_mapping(ssk, skb)) | |
704 | return MAPPING_INVALID; | |
705 | ||
706 | skb_ext_del(skb, SKB_EXT_MPTCP); | |
707 | return MAPPING_OK; | |
708 | } | |
709 | ||
bfae9dae FW |
710 | static int subflow_read_actor(read_descriptor_t *desc, |
711 | struct sk_buff *skb, | |
712 | unsigned int offset, size_t len) | |
713 | { | |
714 | size_t copy_len = min(desc->count, len); | |
715 | ||
716 | desc->count -= copy_len; | |
717 | ||
718 | pr_debug("flushed %zu bytes, %zu left", copy_len, desc->count); | |
719 | return copy_len; | |
720 | } | |
721 | ||
648ef4b8 MM |
722 | static bool subflow_check_data_avail(struct sock *ssk) |
723 | { | |
724 | struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk); | |
725 | enum mapping_status status; | |
726 | struct mptcp_sock *msk; | |
727 | struct sk_buff *skb; | |
728 | ||
729 | pr_debug("msk=%p ssk=%p data_avail=%d skb=%p", subflow->conn, ssk, | |
730 | subflow->data_avail, skb_peek(&ssk->sk_receive_queue)); | |
731 | if (subflow->data_avail) | |
732 | return true; | |
733 | ||
648ef4b8 MM |
734 | msk = mptcp_sk(subflow->conn); |
735 | for (;;) { | |
736 | u32 map_remaining; | |
737 | size_t delta; | |
738 | u64 ack_seq; | |
739 | u64 old_ack; | |
740 | ||
741 | status = get_mapping_status(ssk); | |
742 | pr_debug("msk=%p ssk=%p status=%d", msk, ssk, status); | |
743 | if (status == MAPPING_INVALID) { | |
744 | ssk->sk_err = EBADMSG; | |
745 | goto fatal; | |
746 | } | |
747 | ||
748 | if (status != MAPPING_OK) | |
749 | return false; | |
750 | ||
751 | skb = skb_peek(&ssk->sk_receive_queue); | |
752 | if (WARN_ON_ONCE(!skb)) | |
753 | return false; | |
754 | ||
d22f4988 CP |
755 | /* if msk lacks the remote key, this subflow must provide an |
756 | * MP_CAPABLE-based mapping | |
757 | */ | |
758 | if (unlikely(!READ_ONCE(msk->can_ack))) { | |
759 | if (!subflow->mpc_map) { | |
760 | ssk->sk_err = EBADMSG; | |
761 | goto fatal; | |
762 | } | |
763 | WRITE_ONCE(msk->remote_key, subflow->remote_key); | |
764 | WRITE_ONCE(msk->ack_seq, subflow->map_seq); | |
765 | WRITE_ONCE(msk->can_ack, true); | |
766 | } | |
767 | ||
648ef4b8 MM |
768 | old_ack = READ_ONCE(msk->ack_seq); |
769 | ack_seq = mptcp_subflow_get_mapped_dsn(subflow); | |
770 | pr_debug("msk ack_seq=%llx subflow ack_seq=%llx", old_ack, | |
771 | ack_seq); | |
772 | if (ack_seq == old_ack) | |
773 | break; | |
774 | ||
775 | /* only accept in-sequence mapping. Old values are spurious | |
776 | * retransmission; we can hit "future" values on active backup | |
777 | * subflow switch, we relay on retransmissions to get | |
778 | * in-sequence data. | |
779 | * Cuncurrent subflows support will require subflow data | |
780 | * reordering | |
781 | */ | |
782 | map_remaining = subflow->map_data_len - | |
783 | mptcp_subflow_get_map_offset(subflow); | |
784 | if (before64(ack_seq, old_ack)) | |
785 | delta = min_t(size_t, old_ack - ack_seq, map_remaining); | |
786 | else | |
787 | delta = min_t(size_t, ack_seq - old_ack, map_remaining); | |
788 | ||
789 | /* discard mapped data */ | |
790 | pr_debug("discarding %zu bytes, current map len=%d", delta, | |
791 | map_remaining); | |
792 | if (delta) { | |
648ef4b8 MM |
793 | read_descriptor_t desc = { |
794 | .count = delta, | |
648ef4b8 MM |
795 | }; |
796 | int ret; | |
797 | ||
bfae9dae | 798 | ret = tcp_read_sock(ssk, &desc, subflow_read_actor); |
648ef4b8 MM |
799 | if (ret < 0) { |
800 | ssk->sk_err = -ret; | |
801 | goto fatal; | |
802 | } | |
803 | if (ret < delta) | |
804 | return false; | |
805 | if (delta == map_remaining) | |
806 | subflow->map_valid = 0; | |
807 | } | |
808 | } | |
809 | return true; | |
810 | ||
811 | fatal: | |
812 | /* fatal protocol error, close the socket */ | |
813 | /* This barrier is coupled with smp_rmb() in tcp_poll() */ | |
814 | smp_wmb(); | |
815 | ssk->sk_error_report(ssk); | |
816 | tcp_set_state(ssk, TCP_CLOSE); | |
817 | tcp_send_active_reset(ssk, GFP_ATOMIC); | |
818 | return false; | |
819 | } | |
820 | ||
821 | bool mptcp_subflow_data_available(struct sock *sk) | |
822 | { | |
823 | struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk); | |
824 | struct sk_buff *skb; | |
825 | ||
826 | /* check if current mapping is still valid */ | |
827 | if (subflow->map_valid && | |
828 | mptcp_subflow_get_map_offset(subflow) >= subflow->map_data_len) { | |
829 | subflow->map_valid = 0; | |
830 | subflow->data_avail = 0; | |
831 | ||
832 | pr_debug("Done with mapping: seq=%u data_len=%u", | |
833 | subflow->map_subflow_seq, | |
834 | subflow->map_data_len); | |
835 | } | |
836 | ||
837 | if (!subflow_check_data_avail(sk)) { | |
838 | subflow->data_avail = 0; | |
839 | return false; | |
840 | } | |
841 | ||
842 | skb = skb_peek(&sk->sk_receive_queue); | |
843 | subflow->data_avail = skb && | |
844 | before(tcp_sk(sk)->copied_seq, TCP_SKB_CB(skb)->end_seq); | |
845 | return subflow->data_avail; | |
846 | } | |
847 | ||
848 | static void subflow_data_ready(struct sock *sk) | |
849 | { | |
850 | struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk); | |
851 | struct sock *parent = subflow->conn; | |
852 | ||
f296234c | 853 | if (!subflow->mp_capable && !subflow->mp_join) { |
648ef4b8 MM |
854 | subflow->tcp_data_ready(sk); |
855 | ||
dc093db5 | 856 | parent->sk_data_ready(parent); |
648ef4b8 MM |
857 | return; |
858 | } | |
859 | ||
101f6f85 | 860 | if (mptcp_subflow_data_available(sk)) |
2e52213c | 861 | mptcp_data_ready(parent, sk); |
648ef4b8 MM |
862 | } |
863 | ||
864 | static void subflow_write_space(struct sock *sk) | |
865 | { | |
866 | struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk); | |
867 | struct sock *parent = subflow->conn; | |
868 | ||
869 | sk_stream_write_space(sk); | |
dc093db5 | 870 | if (sk_stream_is_writeable(sk)) { |
1891c4a0 FW |
871 | set_bit(MPTCP_SEND_SPACE, &mptcp_sk(parent)->flags); |
872 | smp_mb__after_atomic(); | |
873 | /* set SEND_SPACE before sk_stream_write_space clears NOSPACE */ | |
648ef4b8 MM |
874 | sk_stream_write_space(parent); |
875 | } | |
876 | } | |
877 | ||
cec37a6e PK |
878 | static struct inet_connection_sock_af_ops * |
879 | subflow_default_af_ops(struct sock *sk) | |
880 | { | |
881 | #if IS_ENABLED(CONFIG_MPTCP_IPV6) | |
882 | if (sk->sk_family == AF_INET6) | |
883 | return &subflow_v6_specific; | |
884 | #endif | |
885 | return &subflow_specific; | |
886 | } | |
887 | ||
cec37a6e | 888 | #if IS_ENABLED(CONFIG_MPTCP_IPV6) |
31484d56 GU |
889 | void mptcpv6_handle_mapped(struct sock *sk, bool mapped) |
890 | { | |
cec37a6e PK |
891 | struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk); |
892 | struct inet_connection_sock *icsk = inet_csk(sk); | |
893 | struct inet_connection_sock_af_ops *target; | |
894 | ||
895 | target = mapped ? &subflow_v6m_specific : subflow_default_af_ops(sk); | |
896 | ||
897 | pr_debug("subflow=%p family=%d ops=%p target=%p mapped=%d", | |
edc7e489 | 898 | subflow, sk->sk_family, icsk->icsk_af_ops, target, mapped); |
cec37a6e PK |
899 | |
900 | if (likely(icsk->icsk_af_ops == target)) | |
901 | return; | |
902 | ||
903 | subflow->icsk_af_ops = icsk->icsk_af_ops; | |
904 | icsk->icsk_af_ops = target; | |
cec37a6e | 905 | } |
31484d56 | 906 | #endif |
cec37a6e | 907 | |
ec3edaa7 PK |
908 | static void mptcp_info2sockaddr(const struct mptcp_addr_info *info, |
909 | struct sockaddr_storage *addr) | |
910 | { | |
911 | memset(addr, 0, sizeof(*addr)); | |
912 | addr->ss_family = info->family; | |
913 | if (addr->ss_family == AF_INET) { | |
914 | struct sockaddr_in *in_addr = (struct sockaddr_in *)addr; | |
915 | ||
916 | in_addr->sin_addr = info->addr; | |
917 | in_addr->sin_port = info->port; | |
918 | } | |
919 | #if IS_ENABLED(CONFIG_MPTCP_IPV6) | |
920 | else if (addr->ss_family == AF_INET6) { | |
921 | struct sockaddr_in6 *in6_addr = (struct sockaddr_in6 *)addr; | |
922 | ||
923 | in6_addr->sin6_addr = info->addr6; | |
924 | in6_addr->sin6_port = info->port; | |
925 | } | |
926 | #endif | |
927 | } | |
928 | ||
929 | int __mptcp_subflow_connect(struct sock *sk, int ifindex, | |
930 | const struct mptcp_addr_info *loc, | |
931 | const struct mptcp_addr_info *remote) | |
932 | { | |
933 | struct mptcp_sock *msk = mptcp_sk(sk); | |
934 | struct mptcp_subflow_context *subflow; | |
935 | struct sockaddr_storage addr; | |
936 | struct socket *sf; | |
937 | u32 remote_token; | |
938 | int addrlen; | |
939 | int err; | |
940 | ||
941 | if (sk->sk_state != TCP_ESTABLISHED) | |
942 | return -ENOTCONN; | |
943 | ||
944 | err = mptcp_subflow_create_socket(sk, &sf); | |
945 | if (err) | |
946 | return err; | |
947 | ||
948 | subflow = mptcp_subflow_ctx(sf->sk); | |
949 | subflow->remote_key = msk->remote_key; | |
950 | subflow->local_key = msk->local_key; | |
951 | subflow->token = msk->token; | |
952 | mptcp_info2sockaddr(loc, &addr); | |
953 | ||
954 | addrlen = sizeof(struct sockaddr_in); | |
955 | #if IS_ENABLED(CONFIG_MPTCP_IPV6) | |
956 | if (loc->family == AF_INET6) | |
957 | addrlen = sizeof(struct sockaddr_in6); | |
958 | #endif | |
959 | sf->sk->sk_bound_dev_if = ifindex; | |
960 | err = kernel_bind(sf, (struct sockaddr *)&addr, addrlen); | |
961 | if (err) | |
962 | goto failed; | |
963 | ||
964 | mptcp_crypto_key_sha(subflow->remote_key, &remote_token, NULL); | |
965 | pr_debug("msk=%p remote_token=%u", msk, remote_token); | |
966 | subflow->remote_token = remote_token; | |
967 | subflow->local_id = loc->id; | |
968 | subflow->request_join = 1; | |
969 | subflow->request_bkup = 1; | |
970 | mptcp_info2sockaddr(remote, &addr); | |
971 | ||
972 | err = kernel_connect(sf, (struct sockaddr *)&addr, addrlen, O_NONBLOCK); | |
973 | if (err && err != -EINPROGRESS) | |
974 | goto failed; | |
975 | ||
976 | spin_lock_bh(&msk->join_list_lock); | |
977 | list_add_tail(&subflow->node, &msk->join_list); | |
978 | spin_unlock_bh(&msk->join_list_lock); | |
979 | ||
980 | return err; | |
981 | ||
982 | failed: | |
983 | sock_release(sf); | |
984 | return err; | |
985 | } | |
986 | ||
2303f994 PK |
987 | int mptcp_subflow_create_socket(struct sock *sk, struct socket **new_sock) |
988 | { | |
989 | struct mptcp_subflow_context *subflow; | |
990 | struct net *net = sock_net(sk); | |
991 | struct socket *sf; | |
992 | int err; | |
993 | ||
cec37a6e PK |
994 | err = sock_create_kern(net, sk->sk_family, SOCK_STREAM, IPPROTO_TCP, |
995 | &sf); | |
2303f994 PK |
996 | if (err) |
997 | return err; | |
998 | ||
999 | lock_sock(sf->sk); | |
1000 | ||
1001 | /* kernel sockets do not by default acquire net ref, but TCP timer | |
1002 | * needs it. | |
1003 | */ | |
1004 | sf->sk->sk_net_refcnt = 1; | |
1005 | get_net(net); | |
f6f7d8cf | 1006 | #ifdef CONFIG_PROC_FS |
2303f994 | 1007 | this_cpu_add(*net->core.sock_inuse, 1); |
f6f7d8cf | 1008 | #endif |
2303f994 PK |
1009 | err = tcp_set_ulp(sf->sk, "mptcp"); |
1010 | release_sock(sf->sk); | |
1011 | ||
1012 | if (err) | |
1013 | return err; | |
1014 | ||
7d14b0d2 PA |
1015 | /* the newly created socket really belongs to the owning MPTCP master |
1016 | * socket, even if for additional subflows the allocation is performed | |
1017 | * by a kernel workqueue. Adjust inode references, so that the | |
1018 | * procfs/diag interaces really show this one belonging to the correct | |
1019 | * user. | |
1020 | */ | |
1021 | SOCK_INODE(sf)->i_ino = SOCK_INODE(sk->sk_socket)->i_ino; | |
1022 | SOCK_INODE(sf)->i_uid = SOCK_INODE(sk->sk_socket)->i_uid; | |
1023 | SOCK_INODE(sf)->i_gid = SOCK_INODE(sk->sk_socket)->i_gid; | |
1024 | ||
2303f994 PK |
1025 | subflow = mptcp_subflow_ctx(sf->sk); |
1026 | pr_debug("subflow=%p", subflow); | |
1027 | ||
1028 | *new_sock = sf; | |
79c0949e | 1029 | sock_hold(sk); |
2303f994 PK |
1030 | subflow->conn = sk; |
1031 | ||
1032 | return 0; | |
1033 | } | |
1034 | ||
1035 | static struct mptcp_subflow_context *subflow_create_ctx(struct sock *sk, | |
1036 | gfp_t priority) | |
1037 | { | |
1038 | struct inet_connection_sock *icsk = inet_csk(sk); | |
1039 | struct mptcp_subflow_context *ctx; | |
1040 | ||
1041 | ctx = kzalloc(sizeof(*ctx), priority); | |
1042 | if (!ctx) | |
1043 | return NULL; | |
1044 | ||
1045 | rcu_assign_pointer(icsk->icsk_ulp_data, ctx); | |
cec37a6e | 1046 | INIT_LIST_HEAD(&ctx->node); |
2303f994 PK |
1047 | |
1048 | pr_debug("subflow=%p", ctx); | |
1049 | ||
1050 | ctx->tcp_sock = sk; | |
1051 | ||
1052 | return ctx; | |
1053 | } | |
1054 | ||
648ef4b8 MM |
1055 | static void __subflow_state_change(struct sock *sk) |
1056 | { | |
1057 | struct socket_wq *wq; | |
1058 | ||
1059 | rcu_read_lock(); | |
1060 | wq = rcu_dereference(sk->sk_wq); | |
1061 | if (skwq_has_sleeper(wq)) | |
1062 | wake_up_interruptible_all(&wq->wait); | |
1063 | rcu_read_unlock(); | |
1064 | } | |
1065 | ||
1066 | static bool subflow_is_done(const struct sock *sk) | |
1067 | { | |
1068 | return sk->sk_shutdown & RCV_SHUTDOWN || sk->sk_state == TCP_CLOSE; | |
1069 | } | |
1070 | ||
1071 | static void subflow_state_change(struct sock *sk) | |
1072 | { | |
1073 | struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk); | |
dc093db5 | 1074 | struct sock *parent = subflow->conn; |
648ef4b8 MM |
1075 | |
1076 | __subflow_state_change(sk); | |
1077 | ||
1078 | /* as recvmsg() does not acquire the subflow socket for ssk selection | |
1079 | * a fin packet carrying a DSS can be unnoticed if we don't trigger | |
1080 | * the data available machinery here. | |
1081 | */ | |
dc093db5 | 1082 | if (subflow->mp_capable && mptcp_subflow_data_available(sk)) |
2e52213c | 1083 | mptcp_data_ready(parent, sk); |
648ef4b8 | 1084 | |
dc093db5 | 1085 | if (!(parent->sk_shutdown & RCV_SHUTDOWN) && |
648ef4b8 MM |
1086 | !subflow->rx_eof && subflow_is_done(sk)) { |
1087 | subflow->rx_eof = 1; | |
59832e24 | 1088 | mptcp_subflow_eof(parent); |
648ef4b8 MM |
1089 | } |
1090 | } | |
1091 | ||
2303f994 PK |
1092 | static int subflow_ulp_init(struct sock *sk) |
1093 | { | |
cec37a6e | 1094 | struct inet_connection_sock *icsk = inet_csk(sk); |
2303f994 PK |
1095 | struct mptcp_subflow_context *ctx; |
1096 | struct tcp_sock *tp = tcp_sk(sk); | |
1097 | int err = 0; | |
1098 | ||
1099 | /* disallow attaching ULP to a socket unless it has been | |
1100 | * created with sock_create_kern() | |
1101 | */ | |
1102 | if (!sk->sk_kern_sock) { | |
1103 | err = -EOPNOTSUPP; | |
1104 | goto out; | |
1105 | } | |
1106 | ||
1107 | ctx = subflow_create_ctx(sk, GFP_KERNEL); | |
1108 | if (!ctx) { | |
1109 | err = -ENOMEM; | |
1110 | goto out; | |
1111 | } | |
1112 | ||
1113 | pr_debug("subflow=%p, family=%d", ctx, sk->sk_family); | |
1114 | ||
1115 | tp->is_mptcp = 1; | |
cec37a6e PK |
1116 | ctx->icsk_af_ops = icsk->icsk_af_ops; |
1117 | icsk->icsk_af_ops = subflow_default_af_ops(sk); | |
648ef4b8 MM |
1118 | ctx->tcp_data_ready = sk->sk_data_ready; |
1119 | ctx->tcp_state_change = sk->sk_state_change; | |
1120 | ctx->tcp_write_space = sk->sk_write_space; | |
1121 | sk->sk_data_ready = subflow_data_ready; | |
1122 | sk->sk_write_space = subflow_write_space; | |
1123 | sk->sk_state_change = subflow_state_change; | |
2303f994 PK |
1124 | out: |
1125 | return err; | |
1126 | } | |
1127 | ||
1128 | static void subflow_ulp_release(struct sock *sk) | |
1129 | { | |
1130 | struct mptcp_subflow_context *ctx = mptcp_subflow_ctx(sk); | |
1131 | ||
1132 | if (!ctx) | |
1133 | return; | |
1134 | ||
79c0949e PK |
1135 | if (ctx->conn) |
1136 | sock_put(ctx->conn); | |
1137 | ||
2303f994 PK |
1138 | kfree_rcu(ctx, rcu); |
1139 | } | |
1140 | ||
cec37a6e PK |
1141 | static void subflow_ulp_clone(const struct request_sock *req, |
1142 | struct sock *newsk, | |
1143 | const gfp_t priority) | |
1144 | { | |
1145 | struct mptcp_subflow_request_sock *subflow_req = mptcp_subflow_rsk(req); | |
1146 | struct mptcp_subflow_context *old_ctx = mptcp_subflow_ctx(newsk); | |
1147 | struct mptcp_subflow_context *new_ctx; | |
1148 | ||
f296234c PK |
1149 | if (!tcp_rsk(req)->is_mptcp || |
1150 | (!subflow_req->mp_capable && !subflow_req->mp_join)) { | |
648ef4b8 | 1151 | subflow_ulp_fallback(newsk, old_ctx); |
cec37a6e PK |
1152 | return; |
1153 | } | |
1154 | ||
1155 | new_ctx = subflow_create_ctx(newsk, priority); | |
edc7e489 | 1156 | if (!new_ctx) { |
648ef4b8 | 1157 | subflow_ulp_fallback(newsk, old_ctx); |
cec37a6e PK |
1158 | return; |
1159 | } | |
1160 | ||
1161 | new_ctx->conn_finished = 1; | |
1162 | new_ctx->icsk_af_ops = old_ctx->icsk_af_ops; | |
648ef4b8 MM |
1163 | new_ctx->tcp_data_ready = old_ctx->tcp_data_ready; |
1164 | new_ctx->tcp_state_change = old_ctx->tcp_state_change; | |
1165 | new_ctx->tcp_write_space = old_ctx->tcp_write_space; | |
58b09919 PA |
1166 | new_ctx->rel_write_seq = 1; |
1167 | new_ctx->tcp_sock = newsk; | |
1168 | ||
f296234c PK |
1169 | if (subflow_req->mp_capable) { |
1170 | /* see comments in subflow_syn_recv_sock(), MPTCP connection | |
1171 | * is fully established only after we receive the remote key | |
1172 | */ | |
1173 | new_ctx->mp_capable = 1; | |
f296234c PK |
1174 | new_ctx->local_key = subflow_req->local_key; |
1175 | new_ctx->token = subflow_req->token; | |
1176 | new_ctx->ssn_offset = subflow_req->ssn_offset; | |
1177 | new_ctx->idsn = subflow_req->idsn; | |
1178 | } else if (subflow_req->mp_join) { | |
ec3edaa7 | 1179 | new_ctx->ssn_offset = subflow_req->ssn_offset; |
f296234c PK |
1180 | new_ctx->mp_join = 1; |
1181 | new_ctx->fully_established = 1; | |
1182 | new_ctx->backup = subflow_req->backup; | |
1183 | new_ctx->local_id = subflow_req->local_id; | |
1184 | new_ctx->token = subflow_req->token; | |
1185 | new_ctx->thmac = subflow_req->thmac; | |
1186 | } | |
cec37a6e PK |
1187 | } |
1188 | ||
2303f994 PK |
1189 | static struct tcp_ulp_ops subflow_ulp_ops __read_mostly = { |
1190 | .name = "mptcp", | |
1191 | .owner = THIS_MODULE, | |
1192 | .init = subflow_ulp_init, | |
1193 | .release = subflow_ulp_release, | |
cec37a6e | 1194 | .clone = subflow_ulp_clone, |
2303f994 PK |
1195 | }; |
1196 | ||
cec37a6e PK |
1197 | static int subflow_ops_init(struct request_sock_ops *subflow_ops) |
1198 | { | |
1199 | subflow_ops->obj_size = sizeof(struct mptcp_subflow_request_sock); | |
1200 | subflow_ops->slab_name = "request_sock_subflow"; | |
1201 | ||
1202 | subflow_ops->slab = kmem_cache_create(subflow_ops->slab_name, | |
1203 | subflow_ops->obj_size, 0, | |
1204 | SLAB_ACCOUNT | | |
1205 | SLAB_TYPESAFE_BY_RCU, | |
1206 | NULL); | |
1207 | if (!subflow_ops->slab) | |
1208 | return -ENOMEM; | |
1209 | ||
79c0949e PK |
1210 | subflow_ops->destructor = subflow_req_destructor; |
1211 | ||
cec37a6e PK |
1212 | return 0; |
1213 | } | |
1214 | ||
2303f994 PK |
1215 | void mptcp_subflow_init(void) |
1216 | { | |
cec37a6e PK |
1217 | subflow_request_sock_ops = tcp_request_sock_ops; |
1218 | if (subflow_ops_init(&subflow_request_sock_ops) != 0) | |
1219 | panic("MPTCP: failed to init subflow request sock ops\n"); | |
1220 | ||
1221 | subflow_request_sock_ipv4_ops = tcp_request_sock_ipv4_ops; | |
1222 | subflow_request_sock_ipv4_ops.init_req = subflow_v4_init_req; | |
1223 | ||
1224 | subflow_specific = ipv4_specific; | |
1225 | subflow_specific.conn_request = subflow_v4_conn_request; | |
1226 | subflow_specific.syn_recv_sock = subflow_syn_recv_sock; | |
1227 | subflow_specific.sk_rx_dst_set = subflow_finish_connect; | |
79c0949e | 1228 | subflow_specific.rebuild_header = subflow_rebuild_header; |
cec37a6e PK |
1229 | |
1230 | #if IS_ENABLED(CONFIG_MPTCP_IPV6) | |
1231 | subflow_request_sock_ipv6_ops = tcp_request_sock_ipv6_ops; | |
1232 | subflow_request_sock_ipv6_ops.init_req = subflow_v6_init_req; | |
1233 | ||
1234 | subflow_v6_specific = ipv6_specific; | |
1235 | subflow_v6_specific.conn_request = subflow_v6_conn_request; | |
1236 | subflow_v6_specific.syn_recv_sock = subflow_syn_recv_sock; | |
1237 | subflow_v6_specific.sk_rx_dst_set = subflow_finish_connect; | |
79c0949e | 1238 | subflow_v6_specific.rebuild_header = subflow_rebuild_header; |
cec37a6e PK |
1239 | |
1240 | subflow_v6m_specific = subflow_v6_specific; | |
1241 | subflow_v6m_specific.queue_xmit = ipv4_specific.queue_xmit; | |
1242 | subflow_v6m_specific.send_check = ipv4_specific.send_check; | |
1243 | subflow_v6m_specific.net_header_len = ipv4_specific.net_header_len; | |
1244 | subflow_v6m_specific.mtu_reduced = ipv4_specific.mtu_reduced; | |
1245 | subflow_v6m_specific.net_frag_header_len = 0; | |
1246 | #endif | |
1247 | ||
5147dfb5 DC |
1248 | mptcp_diag_subflow_init(&subflow_ulp_ops); |
1249 | ||
2303f994 PK |
1250 | if (tcp_register_ulp(&subflow_ulp_ops) != 0) |
1251 | panic("MPTCP: failed to register subflows to ULP\n"); | |
1252 | } |