]>
Commit | Line | Data |
---|---|---|
f870fa0b MM |
1 | /* SPDX-License-Identifier: GPL-2.0 */ |
2 | /* Multipath TCP | |
3 | * | |
4 | * Copyright (c) 2017 - 2019, Intel Corporation. | |
5 | */ | |
6 | ||
7 | #ifndef __MPTCP_PROTOCOL_H | |
8 | #define __MPTCP_PROTOCOL_H | |
9 | ||
79c0949e PK |
10 | #include <linux/random.h> |
11 | #include <net/tcp.h> | |
12 | #include <net/inet_connection_sock.h> | |
b911c97c | 13 | #include <uapi/linux/mptcp.h> |
9ab4807c | 14 | #include <net/genetlink.h> |
79c0949e | 15 | |
cc7972ea | 16 | #define MPTCP_SUPPORTED_VERSION 1 |
eda7acdd PK |
17 | |
18 | /* MPTCP option bits */ | |
19 | #define OPTION_MPTCP_MPC_SYN BIT(0) | |
20 | #define OPTION_MPTCP_MPC_SYNACK BIT(1) | |
21 | #define OPTION_MPTCP_MPC_ACK BIT(2) | |
f296234c PK |
22 | #define OPTION_MPTCP_MPJ_SYN BIT(3) |
23 | #define OPTION_MPTCP_MPJ_SYNACK BIT(4) | |
24 | #define OPTION_MPTCP_MPJ_ACK BIT(5) | |
3df523ab | 25 | #define OPTION_MPTCP_ADD_ADDR BIT(6) |
fef6b7ec GT |
26 | #define OPTION_MPTCP_RM_ADDR BIT(7) |
27 | #define OPTION_MPTCP_FASTCLOSE BIT(8) | |
28 | #define OPTION_MPTCP_PRIO BIT(9) | |
29 | #define OPTION_MPTCP_RST BIT(10) | |
1bff1e43 | 30 | #define OPTION_MPTCP_DSS BIT(11) |
c25aeb4e | 31 | #define OPTION_MPTCP_FAIL BIT(12) |
eda7acdd | 32 | |
74c7dfbe PA |
33 | #define OPTION_MPTCP_CSUMREQD BIT(13) |
34 | ||
35 | #define OPTIONS_MPTCP_MPC (OPTION_MPTCP_MPC_SYN | OPTION_MPTCP_MPC_SYNACK | \ | |
36 | OPTION_MPTCP_MPC_ACK) | |
37 | #define OPTIONS_MPTCP_MPJ (OPTION_MPTCP_MPJ_SYN | OPTION_MPTCP_MPJ_SYNACK | \ | |
780aa120 | 38 | OPTION_MPTCP_MPJ_ACK) |
74c7dfbe | 39 | |
eda7acdd PK |
40 | /* MPTCP option subtypes */ |
41 | #define MPTCPOPT_MP_CAPABLE 0 | |
42 | #define MPTCPOPT_MP_JOIN 1 | |
43 | #define MPTCPOPT_DSS 2 | |
44 | #define MPTCPOPT_ADD_ADDR 3 | |
45 | #define MPTCPOPT_RM_ADDR 4 | |
46 | #define MPTCPOPT_MP_PRIO 5 | |
47 | #define MPTCPOPT_MP_FAIL 6 | |
48 | #define MPTCPOPT_MP_FASTCLOSE 7 | |
dc87efdb | 49 | #define MPTCPOPT_RST 8 |
eda7acdd PK |
50 | |
51 | /* MPTCP suboption lengths */ | |
cc7972ea | 52 | #define TCPOLEN_MPTCP_MPC_SYN 4 |
eda7acdd PK |
53 | #define TCPOLEN_MPTCP_MPC_SYNACK 12 |
54 | #define TCPOLEN_MPTCP_MPC_ACK 20 | |
cc7972ea | 55 | #define TCPOLEN_MPTCP_MPC_ACK_DATA 22 |
f296234c PK |
56 | #define TCPOLEN_MPTCP_MPJ_SYN 12 |
57 | #define TCPOLEN_MPTCP_MPJ_SYNACK 16 | |
58 | #define TCPOLEN_MPTCP_MPJ_ACK 24 | |
6d0060f6 | 59 | #define TCPOLEN_MPTCP_DSS_BASE 4 |
648ef4b8 | 60 | #define TCPOLEN_MPTCP_DSS_ACK32 4 |
6d0060f6 | 61 | #define TCPOLEN_MPTCP_DSS_ACK64 8 |
648ef4b8 | 62 | #define TCPOLEN_MPTCP_DSS_MAP32 10 |
6d0060f6 MM |
63 | #define TCPOLEN_MPTCP_DSS_MAP64 14 |
64 | #define TCPOLEN_MPTCP_DSS_CHECKSUM 2 | |
3df523ab | 65 | #define TCPOLEN_MPTCP_ADD_ADDR 16 |
27ab92d9 | 66 | #define TCPOLEN_MPTCP_ADD_ADDR_PORT 18 |
3df523ab | 67 | #define TCPOLEN_MPTCP_ADD_ADDR_BASE 8 |
27ab92d9 | 68 | #define TCPOLEN_MPTCP_ADD_ADDR_BASE_PORT 10 |
3df523ab | 69 | #define TCPOLEN_MPTCP_ADD_ADDR6 28 |
27ab92d9 | 70 | #define TCPOLEN_MPTCP_ADD_ADDR6_PORT 30 |
3df523ab | 71 | #define TCPOLEN_MPTCP_ADD_ADDR6_BASE 20 |
27ab92d9 DC |
72 | #define TCPOLEN_MPTCP_ADD_ADDR6_BASE_PORT 22 |
73 | #define TCPOLEN_MPTCP_PORT_LEN 2 | |
74 | #define TCPOLEN_MPTCP_PORT_ALIGN 2 | |
6445e17a | 75 | #define TCPOLEN_MPTCP_RM_ADDR_BASE 3 |
ec99a470 DC |
76 | #define TCPOLEN_MPTCP_PRIO 3 |
77 | #define TCPOLEN_MPTCP_PRIO_ALIGN 4 | |
50c504a2 | 78 | #define TCPOLEN_MPTCP_FASTCLOSE 12 |
dc87efdb | 79 | #define TCPOLEN_MPTCP_RST 4 |
c25aeb4e | 80 | #define TCPOLEN_MPTCP_FAIL 12 |
eda7acdd | 81 | |
208e8f66 GT |
82 | #define TCPOLEN_MPTCP_MPC_ACK_DATA_CSUM (TCPOLEN_MPTCP_DSS_CHECKSUM + TCPOLEN_MPTCP_MPC_ACK_DATA) |
83 | ||
ec3edaa7 | 84 | /* MPTCP MP_JOIN flags */ |
f296234c | 85 | #define MPTCPOPT_BACKUP BIT(0) |
ec3edaa7 | 86 | #define MPTCPOPT_THMAC_LEN 8 |
f296234c | 87 | |
eda7acdd PK |
88 | /* MPTCP MP_CAPABLE flags */ |
89 | #define MPTCP_VERSION_MASK (0x0F) | |
90 | #define MPTCP_CAP_CHECKSUM_REQD BIT(7) | |
91 | #define MPTCP_CAP_EXTENSIBILITY BIT(6) | |
bab6b88e | 92 | #define MPTCP_CAP_DENY_JOIN_ID0 BIT(5) |
65492c5a | 93 | #define MPTCP_CAP_HMAC_SHA256 BIT(0) |
bab6b88e | 94 | #define MPTCP_CAP_FLAG_MASK (0x1F) |
eda7acdd | 95 | |
6d0060f6 MM |
96 | /* MPTCP DSS flags */ |
97 | #define MPTCP_DSS_DATA_FIN BIT(4) | |
98 | #define MPTCP_DSS_DSN64 BIT(3) | |
99 | #define MPTCP_DSS_HAS_MAP BIT(2) | |
100 | #define MPTCP_DSS_ACK64 BIT(1) | |
101 | #define MPTCP_DSS_HAS_ACK BIT(0) | |
648ef4b8 MM |
102 | #define MPTCP_DSS_FLAG_MASK (0x1F) |
103 | ||
3df523ab PK |
104 | /* MPTCP ADD_ADDR flags */ |
105 | #define MPTCP_ADDR_ECHO BIT(0) | |
3df523ab | 106 | |
40453a5c GT |
107 | /* MPTCP MP_PRIO flags */ |
108 | #define MPTCP_PRIO_BKUP BIT(0) | |
109 | ||
dc87efdb FW |
110 | /* MPTCP TCPRST flags */ |
111 | #define MPTCP_RST_TRANSIENT BIT(0) | |
112 | ||
e9d09bac | 113 | /* MPTCP socket atomic flags */ |
8edf0864 | 114 | #define MPTCP_NOSPACE 1 |
3b1d6210 | 115 | #define MPTCP_WORK_RTX 2 |
e1ff9e82 | 116 | #define MPTCP_FALLBACK_DONE 4 |
0e4f35d7 | 117 | #define MPTCP_WORK_CLOSE_SUBFLOW 5 |
e9d09bac PA |
118 | |
119 | /* MPTCP socket release cb flags */ | |
120 | #define MPTCP_PUSH_PENDING 1 | |
121 | #define MPTCP_CLEAN_UNA 2 | |
122 | #define MPTCP_ERROR_REPORT 3 | |
123 | #define MPTCP_RETRANSMIT 4 | |
124 | #define MPTCP_FLUSH_JOIN_LIST 5 | |
125 | #define MPTCP_CONNECTED 6 | |
6d0060f6 | 126 | |
dfc8d060 DS |
127 | struct mptcp_skb_cb { |
128 | u64 map_seq; | |
129 | u64 end_seq; | |
130 | u32 offset; | |
131 | u8 has_rxtstamp:1; | |
132 | }; | |
133 | ||
134 | #define MPTCP_SKB_CB(__skb) ((struct mptcp_skb_cb *)&((__skb)->cb[0])) | |
135 | ||
eaa2ffab PA |
136 | static inline bool before64(__u64 seq1, __u64 seq2) |
137 | { | |
138 | return (__s64)(seq1 - seq2) < 0; | |
139 | } | |
140 | ||
141 | #define after64(seq2, seq1) before64(seq1, seq2) | |
142 | ||
cfde141e PA |
143 | struct mptcp_options_received { |
144 | u64 sndr_key; | |
145 | u64 rcvr_key; | |
146 | u64 data_ack; | |
147 | u64 data_seq; | |
148 | u32 subflow_seq; | |
149 | u16 data_len; | |
208e8f66 | 150 | __sum16 csum; |
74c7dfbe | 151 | u16 suboptions; |
cfde141e PA |
152 | u32 token; |
153 | u32 nonce; | |
a086aeba | 154 | u16 use_map:1, |
cfde141e PA |
155 | dsn64:1, |
156 | data_fin:1, | |
157 | use_ack:1, | |
158 | ack64:1, | |
159 | mpc_map:1, | |
a086aeba PA |
160 | reset_reason:4, |
161 | reset_transient:1, | |
162 | echo:1, | |
163 | backup:1, | |
164 | deny_join_id0:1, | |
cfde141e | 165 | __unused:2; |
a086aeba PA |
166 | u8 join_id; |
167 | u64 thmac; | |
168 | u8 hmac[MPTCPOPT_HMAC_LEN]; | |
f7dafee1 | 169 | struct mptcp_addr_info addr; |
5c4a824d | 170 | struct mptcp_rm_list rm_list; |
cfde141e | 171 | u64 ahmac; |
5580d41b | 172 | u64 fail_seq; |
cfde141e PA |
173 | }; |
174 | ||
3df523ab PK |
175 | static inline __be32 mptcp_option(u8 subopt, u8 len, u8 nib, u8 field) |
176 | { | |
177 | return htonl((TCPOPT_MPTCP << 24) | (len << 16) | (subopt << 12) | | |
178 | ((nib & 0xF) << 8) | field); | |
179 | } | |
180 | ||
1b1c7a0e PK |
181 | enum mptcp_pm_status { |
182 | MPTCP_PM_ADD_ADDR_RECEIVED, | |
84dfe367 | 183 | MPTCP_PM_ADD_ADDR_SEND_ACK, |
d0876b22 | 184 | MPTCP_PM_RM_ADDR_RECEIVED, |
1b1c7a0e PK |
185 | MPTCP_PM_ESTABLISHED, |
186 | MPTCP_PM_SUBFLOW_ESTABLISHED, | |
86e39e04 PA |
187 | MPTCP_PM_ALREADY_ESTABLISHED, /* persistent status, set after ESTABLISHED event */ |
188 | MPTCP_PM_MPC_ENDPOINT_ACCOUNTED /* persistent status, set after MPC local address is | |
189 | * accounted int id_avail_bitmap | |
190 | */ | |
1b1c7a0e PK |
191 | }; |
192 | ||
d85a8fde MM |
193 | enum mptcp_pm_type { |
194 | MPTCP_PM_TYPE_KERNEL = 0, | |
195 | MPTCP_PM_TYPE_USERSPACE, | |
196 | ||
197 | __MPTCP_PM_TYPE_NR, | |
198 | __MPTCP_PM_TYPE_MAX = __MPTCP_PM_TYPE_NR - 1, | |
199 | }; | |
200 | ||
86e39e04 PA |
201 | /* Status bits below MPTCP_PM_ALREADY_ESTABLISHED need pm worker actions */ |
202 | #define MPTCP_PM_WORK_MASK ((1 << MPTCP_PM_ALREADY_ESTABLISHED) - 1) | |
203 | ||
13ad9f01 | 204 | enum mptcp_addr_signal_status { |
d91d322a GT |
205 | MPTCP_ADD_ADDR_SIGNAL, |
206 | MPTCP_ADD_ADDR_ECHO, | |
42842a42 | 207 | MPTCP_RM_ADDR_SIGNAL, |
d91d322a GT |
208 | }; |
209 | ||
86e39e04 PA |
210 | /* max value of mptcp_addr_info.id */ |
211 | #define MPTCP_PM_MAX_ADDR_ID U8_MAX | |
212 | ||
1b1c7a0e PK |
213 | struct mptcp_pm_data { |
214 | struct mptcp_addr_info local; | |
215 | struct mptcp_addr_info remote; | |
b6c08380 | 216 | struct list_head anno_list; |
4638de5a | 217 | struct list_head userspace_pm_local_addr_list; |
1b1c7a0e PK |
218 | |
219 | spinlock_t lock; /*protects the whole PM data */ | |
220 | ||
13ad9f01 | 221 | u8 addr_signal; |
1b1c7a0e PK |
222 | bool server_side; |
223 | bool work_pending; | |
224 | bool accept_addr; | |
225 | bool accept_subflow; | |
df377be3 | 226 | bool remote_deny_join_id0; |
1b1c7a0e PK |
227 | u8 add_addr_signaled; |
228 | u8 add_addr_accepted; | |
229 | u8 local_addr_used; | |
d85a8fde | 230 | u8 pm_type; |
1b1c7a0e | 231 | u8 subflows; |
1b1c7a0e | 232 | u8 status; |
86e39e04 | 233 | DECLARE_BITMAP(id_avail_bitmap, MPTCP_PM_MAX_ADDR_ID + 1); |
cbde2787 | 234 | struct mptcp_rm_list rm_list_tx; |
b5c55f33 | 235 | struct mptcp_rm_list rm_list_rx; |
1b1c7a0e PK |
236 | }; |
237 | ||
4638de5a KM |
238 | struct mptcp_pm_addr_entry { |
239 | struct list_head list; | |
240 | struct mptcp_addr_info addr; | |
241 | u8 flags; | |
242 | int ifindex; | |
243 | struct socket *lsk; | |
244 | }; | |
245 | ||
18b683bf PA |
246 | struct mptcp_data_frag { |
247 | struct list_head list; | |
248 | u64 data_seq; | |
f0e6a4cf PA |
249 | u16 data_len; |
250 | u16 offset; | |
251 | u16 overhead; | |
252 | u16 already_sent; | |
18b683bf PA |
253 | struct page *page; |
254 | }; | |
255 | ||
f870fa0b MM |
256 | /* MPTCP connection sock */ |
257 | struct mptcp_sock { | |
258 | /* inet_connection_sock must be the first member */ | |
259 | struct inet_connection_sock sk; | |
cec37a6e PK |
260 | u64 local_key; |
261 | u64 remote_key; | |
6d0060f6 | 262 | u64 write_seq; |
38967f42 | 263 | u64 bytes_sent; |
eaa2ffab | 264 | u64 snd_nxt; |
38967f42 | 265 | u64 bytes_received; |
6d0060f6 | 266 | u64 ack_seq; |
f3589be0 | 267 | atomic64_t rcv_wnd_sent; |
3721b9b6 | 268 | u64 rcv_data_fin_seq; |
38967f42 | 269 | u64 bytes_retrans; |
6511882c | 270 | int rmem_fwd_alloc; |
d5f49190 | 271 | int snd_burst; |
ea4ca586 | 272 | int old_wspace; |
1e1d9d6f PA |
273 | u64 recovery_snd_nxt; /* in recovery mode accept up to this seq; |
274 | * recovery related fields are under data_lock | |
275 | * protection | |
276 | */ | |
38967f42 | 277 | u64 bytes_acked; |
7439d687 PA |
278 | u64 snd_una; |
279 | u64 wnd_end; | |
b51f9b80 | 280 | unsigned long timer_ival; |
79c0949e | 281 | u32 token; |
87952603 | 282 | int rmem_released; |
648ef4b8 | 283 | unsigned long flags; |
e9d09bac PA |
284 | unsigned long cb_flags; |
285 | unsigned long push_pending; | |
1e1d9d6f | 286 | bool recovery; /* closing subflow write queue reinjected */ |
d22f4988 | 287 | bool can_ack; |
b93df08c | 288 | bool fully_established; |
3721b9b6 | 289 | bool rcv_data_fin; |
7279da61 | 290 | bool snd_data_fin_enable; |
50c504a2 | 291 | bool rcv_fastclose; |
37198e93 | 292 | bool use_64bit_ack; /* Set when we received a 64-bit DSN */ |
752e9067 | 293 | bool csum_enabled; |
0530020a | 294 | bool allow_infinite_fallback; |
3ad14f54 | 295 | u8 mpc_endpoint_id; |
4f6e14bd MG |
296 | u8 recvmsg_inq:1, |
297 | cork:1, | |
54f1944e | 298 | nodelay:1, |
b6985b9b | 299 | fastopening:1, |
39880bd8 PA |
300 | in_accept_queue:1, |
301 | free_first:1; | |
80992017 | 302 | struct work_struct work; |
ab174ad8 PA |
303 | struct sk_buff *ooo_last_skb; |
304 | struct rb_root out_of_order_queue; | |
87952603 | 305 | struct sk_buff_head receive_queue; |
cec37a6e | 306 | struct list_head conn_list; |
18b683bf | 307 | struct list_head rtx_queue; |
f0e6a4cf | 308 | struct mptcp_data_frag *first_pending; |
ec3edaa7 | 309 | struct list_head join_list; |
39880bd8 PA |
310 | struct sock *first; /* The mptcp ops can safely dereference, using suitable |
311 | * ONCE annotation, the subflow outside the socket | |
312 | * lock as such sock is freed after close(). | |
313 | */ | |
1b1c7a0e | 314 | struct mptcp_pm_data pm; |
1730b2b2 | 315 | struct mptcp_sched_ops *sched; |
a6b118fe FW |
316 | struct { |
317 | u32 space; /* bytes copied in last measurement window */ | |
318 | u32 copied; /* bytes copied in this measurement window */ | |
319 | u64 time; /* start time of measurement window */ | |
320 | u64 rtt_us; /* last maximum rtt of subflows */ | |
321 | } rcvq_space; | |
b8dc6d6c | 322 | u8 scaling_ratio; |
df00b087 | 323 | |
6f06b4d4 PA |
324 | u32 subflow_id; |
325 | u32 setsockopt_seq; | |
20b5759f | 326 | char ca_name[TCP_CA_NAME_MAX]; |
f870fa0b MM |
327 | }; |
328 | ||
87952603 PA |
329 | #define mptcp_data_lock(sk) spin_lock_bh(&(sk)->sk_lock.slock) |
330 | #define mptcp_data_unlock(sk) spin_unlock_bh(&(sk)->sk_lock.slock) | |
331 | ||
cec37a6e PK |
332 | #define mptcp_for_each_subflow(__msk, __subflow) \ |
333 | list_for_each_entry(__subflow, &((__msk)->conn_list), node) | |
5efbf6f7 MB |
334 | #define mptcp_for_each_subflow_safe(__msk, __subflow, __tmp) \ |
335 | list_for_each_entry_safe(__subflow, __tmp, &((__msk)->conn_list), node) | |
cec37a6e | 336 | |
3abc05d9 FW |
337 | static inline void msk_owned_by_me(const struct mptcp_sock *msk) |
338 | { | |
339 | sock_owned_by_me((const struct sock *)msk); | |
340 | } | |
341 | ||
403a40f2 | 342 | #define mptcp_sk(ptr) container_of_const(ptr, struct mptcp_sock, sk.icsk_inet.sk) |
f870fa0b | 343 | |
ce599c51 PA |
344 | /* the msk socket don't use the backlog, also account for the bulk |
345 | * free memory | |
346 | */ | |
347 | static inline int __mptcp_rmem(const struct sock *sk) | |
348 | { | |
349 | return atomic_read(&sk->sk_rmem_alloc) - READ_ONCE(mptcp_sk(sk)->rmem_released); | |
350 | } | |
351 | ||
b8dc6d6c PA |
352 | static inline int mptcp_win_from_space(const struct sock *sk, int space) |
353 | { | |
354 | return __tcp_win_from_space(mptcp_sk(sk)->scaling_ratio, space); | |
355 | } | |
356 | ||
ea4ca586 PA |
357 | static inline int __mptcp_space(const struct sock *sk) |
358 | { | |
b8dc6d6c | 359 | return mptcp_win_from_space(sk, READ_ONCE(sk->sk_rcvbuf) - __mptcp_rmem(sk)); |
ea4ca586 PA |
360 | } |
361 | ||
f0e6a4cf PA |
362 | static inline struct mptcp_data_frag *mptcp_send_head(const struct sock *sk) |
363 | { | |
364 | const struct mptcp_sock *msk = mptcp_sk(sk); | |
365 | ||
366 | return READ_ONCE(msk->first_pending); | |
367 | } | |
368 | ||
369 | static inline struct mptcp_data_frag *mptcp_send_next(struct sock *sk) | |
370 | { | |
371 | struct mptcp_sock *msk = mptcp_sk(sk); | |
372 | struct mptcp_data_frag *cur; | |
373 | ||
374 | cur = msk->first_pending; | |
375 | return list_is_last(&cur->list, &msk->rtx_queue) ? NULL : | |
376 | list_next_entry(cur, list); | |
377 | } | |
378 | ||
379 | static inline struct mptcp_data_frag *mptcp_pending_tail(const struct sock *sk) | |
380 | { | |
403a40f2 | 381 | const struct mptcp_sock *msk = mptcp_sk(sk); |
f0e6a4cf PA |
382 | |
383 | if (!msk->first_pending) | |
384 | return NULL; | |
385 | ||
386 | if (WARN_ON_ONCE(list_empty(&msk->rtx_queue))) | |
387 | return NULL; | |
388 | ||
389 | return list_last_entry(&msk->rtx_queue, struct mptcp_data_frag, list); | |
390 | } | |
391 | ||
403a40f2 | 392 | static inline struct mptcp_data_frag *mptcp_rtx_head(struct sock *sk) |
18b683bf PA |
393 | { |
394 | struct mptcp_sock *msk = mptcp_sk(sk); | |
395 | ||
64b9cea7 | 396 | if (msk->snd_una == READ_ONCE(msk->snd_nxt)) |
18b683bf PA |
397 | return NULL; |
398 | ||
a386bc5b | 399 | return list_first_entry_or_null(&msk->rtx_queue, struct mptcp_data_frag, list); |
7948f6cc FW |
400 | } |
401 | ||
d0cc2987 GT |
402 | struct csum_pseudo_header { |
403 | __be64 data_seq; | |
404 | __be32 subflow_seq; | |
405 | __be16 data_len; | |
406 | __sum16 csum; | |
407 | }; | |
408 | ||
cec37a6e PK |
409 | struct mptcp_subflow_request_sock { |
410 | struct tcp_request_sock sk; | |
d22f4988 | 411 | u16 mp_capable : 1, |
cec37a6e | 412 | mp_join : 1, |
06fe1719 | 413 | backup : 1, |
bab6b88e GT |
414 | csum_reqd : 1, |
415 | allow_join_id0 : 1; | |
1b1c7a0e | 416 | u8 local_id; |
f296234c | 417 | u8 remote_id; |
cec37a6e | 418 | u64 local_key; |
79c0949e PK |
419 | u64 idsn; |
420 | u32 token; | |
648ef4b8 | 421 | u32 ssn_offset; |
f296234c PK |
422 | u64 thmac; |
423 | u32 local_nonce; | |
424 | u32 remote_nonce; | |
8fd4de12 | 425 | struct mptcp_sock *msk; |
2c5ebd00 | 426 | struct hlist_nulls_node token_node; |
cec37a6e PK |
427 | }; |
428 | ||
429 | static inline struct mptcp_subflow_request_sock * | |
430 | mptcp_subflow_rsk(const struct request_sock *rsk) | |
431 | { | |
432 | return (struct mptcp_subflow_request_sock *)rsk; | |
433 | } | |
434 | ||
6719331c PA |
435 | enum mptcp_data_avail { |
436 | MPTCP_SUBFLOW_NODATA, | |
437 | MPTCP_SUBFLOW_DATA_AVAIL, | |
6719331c PA |
438 | }; |
439 | ||
b19bc294 PA |
440 | struct mptcp_delegated_action { |
441 | struct napi_struct napi; | |
442 | struct list_head head; | |
443 | }; | |
444 | ||
445 | DECLARE_PER_CPU(struct mptcp_delegated_action, mptcp_delegated_actions); | |
446 | ||
a5efdbce PA |
447 | #define MPTCP_DELEGATE_SCHEDULED 0 |
448 | #define MPTCP_DELEGATE_SEND 1 | |
449 | #define MPTCP_DELEGATE_ACK 2 | |
b19bc294 | 450 | |
a5efdbce | 451 | #define MPTCP_DELEGATE_ACTIONS_MASK (~BIT(MPTCP_DELEGATE_SCHEDULED)) |
2303f994 PK |
452 | /* MPTCP subflow context */ |
453 | struct mptcp_subflow_context { | |
cec37a6e | 454 | struct list_head node;/* conn_list of subflows */ |
b29fcfb5 | 455 | |
63ec72bd | 456 | struct_group(reset, |
b29fcfb5 | 457 | |
3ce0852c | 458 | unsigned long avg_pacing_rate; /* protected by msk socket lock */ |
cec37a6e PK |
459 | u64 local_key; |
460 | u64 remote_key; | |
79c0949e | 461 | u64 idsn; |
648ef4b8 | 462 | u64 map_seq; |
cc7972ea | 463 | u32 snd_isn; |
79c0949e | 464 | u32 token; |
6d0060f6 | 465 | u32 rel_write_seq; |
648ef4b8 MM |
466 | u32 map_subflow_seq; |
467 | u32 ssn_offset; | |
468 | u32 map_data_len; | |
dd8bcd17 PA |
469 | __wsum map_data_csum; |
470 | u32 map_csum_len; | |
cec37a6e | 471 | u32 request_mptcp : 1, /* send MP_CAPABLE */ |
ec3edaa7 PK |
472 | request_join : 1, /* send MP_JOIN */ |
473 | request_bkup : 1, | |
cec37a6e | 474 | mp_capable : 1, /* remote is MPTCP capable */ |
f296234c | 475 | mp_join : 1, /* remote is JOINing */ |
0be534f5 | 476 | fully_established : 1, /* path validated */ |
f296234c | 477 | pm_notified : 1, /* PM hook called for established status */ |
648ef4b8 MM |
478 | conn_finished : 1, |
479 | map_valid : 1, | |
dd8bcd17 PA |
480 | map_csum_reqd : 1, |
481 | map_data_fin : 1, | |
d22f4988 | 482 | mpc_map : 1, |
f296234c | 483 | backup : 1, |
06706542 | 484 | send_mp_prio : 1, |
c25aeb4e | 485 | send_mp_fail : 1, |
f284c0c7 | 486 | send_fastclose : 1, |
1e39e5a3 | 487 | send_infinite_map : 1, |
b3ea6b27 | 488 | remote_key_valid : 1, /* received the peer key from */ |
ff5a0b42 | 489 | disposable : 1, /* ctx can be free at ulp release time */ |
4cf86ae8 | 490 | stale : 1, /* unable to snd/rcv data, do not use for xmit */ |
ae66fb2b | 491 | local_id_valid : 1, /* local_id is correctly initialized */ |
dfc8d060 DS |
492 | valid_csum_seen : 1, /* at least one csum validated */ |
493 | is_mptfo : 1, /* subflow is doing TFO */ | |
b7535cfe | 494 | __unused : 9; |
6719331c | 495 | enum mptcp_data_avail data_avail; |
fce68b03 | 496 | bool scheduled; |
f296234c PK |
497 | u32 remote_nonce; |
498 | u64 thmac; | |
499 | u32 local_nonce; | |
ec3edaa7 | 500 | u32 remote_token; |
b3ea6b27 PA |
501 | union { |
502 | u8 hmac[MPTCPOPT_HMAC_LEN]; /* MPJ subflow only */ | |
503 | u64 iasn; /* initial ack sequence number, MPC subflows only */ | |
504 | }; | |
f296234c PK |
505 | u8 local_id; |
506 | u8 remote_id; | |
dc87efdb FW |
507 | u8 reset_seen:1; |
508 | u8 reset_transient:1; | |
509 | u8 reset_reason:4; | |
71b7dec2 | 510 | u8 stale_count; |
648ef4b8 | 511 | |
6f06b4d4 PA |
512 | u32 subflow_id; |
513 | ||
b19bc294 | 514 | long delegated_status; |
76a13b31 | 515 | unsigned long fail_tout; |
b29fcfb5 | 516 | |
63ec72bd | 517 | ); |
b29fcfb5 | 518 | |
b19bc294 PA |
519 | struct list_head delegated_node; /* link into delegated_action, protected by local BH */ |
520 | ||
71b7dec2 PA |
521 | u32 setsockopt_seq; |
522 | u32 stale_rcv_tstamp; | |
df00b087 | 523 | |
2303f994 PK |
524 | struct sock *tcp_sock; /* tcp sk backpointer */ |
525 | struct sock *conn; /* parent mptcp_sock */ | |
cec37a6e | 526 | const struct inet_connection_sock_af_ops *icsk_af_ops; |
648ef4b8 | 527 | void (*tcp_state_change)(struct sock *sk); |
15cc1045 | 528 | void (*tcp_error_report)(struct sock *sk); |
648ef4b8 | 529 | |
2303f994 PK |
530 | struct rcu_head rcu; |
531 | }; | |
532 | ||
533 | static inline struct mptcp_subflow_context * | |
534 | mptcp_subflow_ctx(const struct sock *sk) | |
535 | { | |
536 | struct inet_connection_sock *icsk = inet_csk(sk); | |
537 | ||
538 | /* Use RCU on icsk_ulp_data only for sock diag code */ | |
539 | return (__force struct mptcp_subflow_context *)icsk->icsk_ulp_data; | |
540 | } | |
541 | ||
542 | static inline struct sock * | |
543 | mptcp_subflow_tcp_sock(const struct mptcp_subflow_context *subflow) | |
544 | { | |
545 | return subflow->tcp_sock; | |
546 | } | |
547 | ||
b29fcfb5 PA |
548 | static inline void |
549 | mptcp_subflow_ctx_reset(struct mptcp_subflow_context *subflow) | |
550 | { | |
63ec72bd | 551 | memset(&subflow->reset, 0, sizeof(subflow->reset)); |
b29fcfb5 PA |
552 | subflow->request_mptcp = 1; |
553 | } | |
554 | ||
648ef4b8 MM |
555 | static inline u64 |
556 | mptcp_subflow_get_map_offset(const struct mptcp_subflow_context *subflow) | |
557 | { | |
558 | return tcp_sk(mptcp_subflow_tcp_sock(subflow))->copied_seq - | |
559 | subflow->ssn_offset - | |
560 | subflow->map_subflow_seq; | |
561 | } | |
562 | ||
563 | static inline u64 | |
564 | mptcp_subflow_get_mapped_dsn(const struct mptcp_subflow_context *subflow) | |
565 | { | |
566 | return subflow->map_seq + mptcp_subflow_get_map_offset(subflow); | |
567 | } | |
568 | ||
a5efdbce | 569 | void mptcp_subflow_process_delegated(struct sock *ssk, long actions); |
b19bc294 | 570 | |
bcd97734 | 571 | static inline void mptcp_subflow_delegate(struct mptcp_subflow_context *subflow, int action) |
b19bc294 | 572 | { |
a5efdbce | 573 | long old, set_bits = BIT(MPTCP_DELEGATE_SCHEDULED) | BIT(action); |
b19bc294 PA |
574 | struct mptcp_delegated_action *delegated; |
575 | bool schedule; | |
576 | ||
bcd97734 PA |
577 | /* the caller held the subflow bh socket lock */ |
578 | lockdep_assert_in_softirq(); | |
579 | ||
a5efdbce PA |
580 | /* The implied barrier pairs with tcp_release_cb_override() |
581 | * mptcp_napi_poll(), and ensures the below list check sees list | |
582 | * updates done prior to delegated status bits changes | |
b19bc294 | 583 | */ |
a5efdbce PA |
584 | old = set_mask_bits(&subflow->delegated_status, 0, set_bits); |
585 | if (!(old & BIT(MPTCP_DELEGATE_SCHEDULED))) { | |
586 | if (WARN_ON_ONCE(!list_empty(&subflow->delegated_node))) | |
b19bc294 PA |
587 | return; |
588 | ||
b19bc294 PA |
589 | delegated = this_cpu_ptr(&mptcp_delegated_actions); |
590 | schedule = list_empty(&delegated->head); | |
591 | list_add_tail(&subflow->delegated_node, &delegated->head); | |
592 | sock_hold(mptcp_subflow_tcp_sock(subflow)); | |
593 | if (schedule) | |
594 | napi_schedule(&delegated->napi); | |
595 | } | |
596 | } | |
597 | ||
598 | static inline struct mptcp_subflow_context * | |
599 | mptcp_subflow_delegated_next(struct mptcp_delegated_action *delegated) | |
600 | { | |
601 | struct mptcp_subflow_context *ret; | |
602 | ||
603 | if (list_empty(&delegated->head)) | |
604 | return NULL; | |
605 | ||
606 | ret = list_first_entry(&delegated->head, struct mptcp_subflow_context, delegated_node); | |
607 | list_del_init(&ret->delegated_node); | |
608 | return ret; | |
609 | } | |
610 | ||
6da14d74 PA |
611 | int mptcp_is_enabled(const struct net *net); |
612 | unsigned int mptcp_get_add_addr_timeout(const struct net *net); | |
613 | int mptcp_is_checksum_enabled(const struct net *net); | |
614 | int mptcp_allow_join_id0(const struct net *net); | |
ff5a0b42 | 615 | unsigned int mptcp_stale_loss_cnt(const struct net *net); |
6bb63ccc | 616 | int mptcp_get_pm_type(const struct net *net); |
e3b2870b | 617 | const char *mptcp_get_scheduler(const struct net *net); |
b93df08c | 618 | void mptcp_subflow_fully_established(struct mptcp_subflow_context *subflow, |
b3ea6b27 | 619 | const struct mptcp_options_received *mp_opt); |
1e1d9d6f | 620 | bool __mptcp_retransmit_pending_data(struct sock *sk); |
8b38217a | 621 | void mptcp_check_and_set_pending(struct sock *sk); |
ff5a0b42 | 622 | void __mptcp_push_pending(struct sock *sk, unsigned int flags); |
648ef4b8 | 623 | bool mptcp_subflow_data_available(struct sock *sk); |
d39dceca | 624 | void __init mptcp_subflow_init(void); |
d0876b22 | 625 | void mptcp_subflow_shutdown(struct sock *sk, struct sock *ssk, int how); |
a141e02e FW |
626 | void mptcp_close_ssk(struct sock *sk, struct sock *ssk, |
627 | struct mptcp_subflow_context *subflow); | |
a6574302 | 628 | void __mptcp_subflow_send_ack(struct sock *ssk); |
d5824847 | 629 | void mptcp_subflow_reset(struct sock *ssk); |
2a6a870e | 630 | void mptcp_subflow_queue_clean(struct sock *sk, struct sock *ssk); |
866f26f2 | 631 | void mptcp_sock_graft(struct sock *sk, struct socket *parent); |
3f326a82 | 632 | struct sock *__mptcp_nmpc_sk(struct mptcp_sock *msk); |
26d3e21c | 633 | bool __mptcp_close(struct sock *sk, long timeout); |
30e51b92 | 634 | void mptcp_cancel_work(struct sock *sk); |
63740448 | 635 | void __mptcp_unaccepted_force_close(struct sock *sk); |
36b122ba | 636 | void mptcp_set_owner_r(struct sk_buff *skb, struct sock *sk); |
ec3edaa7 | 637 | |
4638de5a KM |
638 | bool mptcp_addresses_equal(const struct mptcp_addr_info *a, |
639 | const struct mptcp_addr_info *b, bool use_port); | |
dc886bce | 640 | void mptcp_local_address(const struct sock_common *skc, struct mptcp_addr_info *addr); |
4638de5a | 641 | |
ec3edaa7 | 642 | /* called with sk socket lock held */ |
ef0da3b8 | 643 | int __mptcp_subflow_connect(struct sock *sk, const struct mptcp_addr_info *loc, |
ee285257 | 644 | const struct mptcp_addr_info *remote); |
6bc1fe7d PA |
645 | int mptcp_subflow_create_socket(struct sock *sk, unsigned short family, |
646 | struct socket **new_sock); | |
1729cf18 GT |
647 | void mptcp_info2sockaddr(const struct mptcp_addr_info *info, |
648 | struct sockaddr_storage *addr, | |
649 | unsigned short family); | |
740ebe35 GT |
650 | struct mptcp_sched_ops *mptcp_sched_find(const char *name); |
651 | int mptcp_register_scheduler(struct mptcp_sched_ops *sched); | |
652 | void mptcp_unregister_scheduler(struct mptcp_sched_ops *sched); | |
ed1ad86b | 653 | void mptcp_sched_init(void); |
1730b2b2 GT |
654 | int mptcp_init_sched(struct mptcp_sock *msk, |
655 | struct mptcp_sched_ops *sched); | |
656 | void mptcp_release_sched(struct mptcp_sock *msk); | |
fce68b03 GT |
657 | void mptcp_subflow_set_scheduled(struct mptcp_subflow_context *subflow, |
658 | bool scheduled); | |
07336a87 GT |
659 | struct sock *mptcp_subflow_get_send(struct mptcp_sock *msk); |
660 | struct sock *mptcp_subflow_get_retrans(struct mptcp_sock *msk); | |
661 | int mptcp_sched_get_send(struct mptcp_sock *msk); | |
662 | int mptcp_sched_get_retrans(struct mptcp_sock *msk); | |
2303f994 | 663 | |
c886d702 | 664 | static inline bool __tcp_can_send(const struct sock *ssk) |
43f1140b | 665 | { |
c886d702 PA |
666 | /* only send if our side has not closed yet */ |
667 | return ((1 << inet_sk_state_load(ssk)) & (TCPF_ESTABLISHED | TCPF_CLOSE_WAIT)); | |
668 | } | |
43f1140b | 669 | |
c886d702 PA |
670 | static inline bool __mptcp_subflow_active(struct mptcp_subflow_context *subflow) |
671 | { | |
43f1140b GT |
672 | /* can't send if JOIN hasn't completed yet (i.e. is usable for mptcp) */ |
673 | if (subflow->request_join && !subflow->fully_established) | |
674 | return false; | |
675 | ||
c886d702 | 676 | return __tcp_can_send(mptcp_subflow_tcp_sock(subflow)); |
43f1140b GT |
677 | } |
678 | ||
ff5a0b42 PA |
679 | void mptcp_subflow_set_active(struct mptcp_subflow_context *subflow); |
680 | ||
681 | bool mptcp_subflow_active(struct mptcp_subflow_context *subflow); | |
682 | ||
b6985b9b PA |
683 | void mptcp_subflow_drop_ctx(struct sock *ssk); |
684 | ||
648ef4b8 MM |
685 | static inline void mptcp_subflow_tcp_fallback(struct sock *sk, |
686 | struct mptcp_subflow_context *ctx) | |
687 | { | |
952382c6 | 688 | sk->sk_data_ready = sock_def_readable; |
648ef4b8 | 689 | sk->sk_state_change = ctx->tcp_state_change; |
952382c6 | 690 | sk->sk_write_space = sk_stream_write_space; |
15cc1045 | 691 | sk->sk_error_report = ctx->tcp_error_report; |
648ef4b8 MM |
692 | |
693 | inet_csk(sk)->icsk_af_ops = ctx->icsk_af_ops; | |
694 | } | |
695 | ||
d39dceca | 696 | void __init mptcp_proto_init(void); |
784325e9 | 697 | #if IS_ENABLED(CONFIG_MPTCP_IPV6) |
d39dceca | 698 | int __init mptcp_proto_v6_init(void); |
784325e9 | 699 | #endif |
648ef4b8 | 700 | |
7e8b88ec PA |
701 | struct sock *mptcp_sk_clone_init(const struct sock *sk, |
702 | const struct mptcp_options_received *mp_opt, | |
703 | struct sock *ssk, | |
704 | struct request_sock *req); | |
0799e21b | 705 | void mptcp_get_options(const struct sk_buff *skb, |
cfde141e | 706 | struct mptcp_options_received *mp_opt); |
cec37a6e PK |
707 | |
708 | void mptcp_finish_connect(struct sock *sk); | |
490274b4 | 709 | void __mptcp_set_connected(struct sock *sk); |
f6909dc1 | 710 | void mptcp_reset_tout_timer(struct mptcp_sock *msk, unsigned long fail_tout); |
27e5ccc2 PA |
711 | |
712 | static inline void mptcp_stop_tout_timer(struct sock *sk) | |
713 | { | |
714 | if (!inet_csk(sk)->icsk_mtup.probe_timestamp) | |
715 | return; | |
716 | ||
717 | sk_stop_timer(sk, &sk->sk_timer); | |
718 | inet_csk(sk)->icsk_mtup.probe_timestamp = 0; | |
719 | } | |
720 | ||
721 | static inline void mptcp_set_close_tout(struct sock *sk, unsigned long tout) | |
722 | { | |
723 | /* avoid 0 timestamp, as that means no close timeout */ | |
724 | inet_csk(sk)->icsk_mtup.probe_timestamp = tout ? : 1; | |
725 | } | |
726 | ||
727 | static inline void mptcp_start_tout_timer(struct sock *sk) | |
728 | { | |
729 | mptcp_set_close_tout(sk, tcp_jiffies32); | |
730 | mptcp_reset_tout_timer(mptcp_sk(sk), 0); | |
731 | } | |
732 | ||
b93df08c PA |
733 | static inline bool mptcp_is_fully_established(struct sock *sk) |
734 | { | |
735 | return inet_sk_state_load(sk) == TCP_ESTABLISHED && | |
736 | READ_ONCE(mptcp_sk(sk)->fully_established); | |
737 | } | |
a6b118fe | 738 | void mptcp_rcv_space_init(struct mptcp_sock *msk, const struct sock *ssk); |
2e52213c | 739 | void mptcp_data_ready(struct sock *sk, struct sock *ssk); |
f296234c | 740 | bool mptcp_finish_join(struct sock *sk); |
ba8f48f7 | 741 | bool mptcp_schedule_work(struct sock *sk); |
0abdde82 PA |
742 | int mptcp_setsockopt(struct sock *sk, int level, int optname, |
743 | sockptr_t optval, unsigned int optlen); | |
744 | int mptcp_getsockopt(struct sock *sk, int level, int optname, | |
745 | char __user *optval, int __user *option); | |
746 | ||
1502328f PA |
747 | u64 __mptcp_expand_seq(u64 old_seq, u64 cur_seq); |
748 | static inline u64 mptcp_expand_seq(u64 old_seq, u64 cur_seq, bool use_64bit) | |
749 | { | |
750 | if (use_64bit) | |
751 | return cur_seq; | |
752 | ||
753 | return __mptcp_expand_seq(old_seq, cur_seq); | |
754 | } | |
219d0499 | 755 | void __mptcp_check_push(struct sock *sk, struct sock *ssk); |
7439d687 | 756 | void __mptcp_data_acked(struct sock *sk); |
15cc1045 | 757 | void __mptcp_error_report(struct sock *sk); |
1a49b2c2 | 758 | bool mptcp_update_rcv_data_fin(struct mptcp_sock *msk, u64 data_fin_seq, bool use_64bit); |
e16163b6 PA |
759 | static inline bool mptcp_data_fin_enabled(const struct mptcp_sock *msk) |
760 | { | |
761 | return READ_ONCE(msk->snd_data_fin_enable) && | |
762 | READ_ONCE(msk->write_seq) == READ_ONCE(msk->snd_nxt); | |
763 | } | |
764 | ||
5cf92bba PA |
765 | static inline bool mptcp_propagate_sndbuf(struct sock *sk, struct sock *ssk) |
766 | { | |
767 | if ((sk->sk_userlocks & SOCK_SNDBUF_LOCK) || ssk->sk_sndbuf <= READ_ONCE(sk->sk_sndbuf)) | |
768 | return false; | |
769 | ||
770 | WRITE_ONCE(sk->sk_sndbuf, ssk->sk_sndbuf); | |
771 | return true; | |
772 | } | |
773 | ||
774 | static inline void mptcp_write_space(struct sock *sk) | |
775 | { | |
776 | if (sk_stream_is_writeable(sk)) { | |
777 | /* pairs with memory barrier in mptcp_poll */ | |
778 | smp_mb(); | |
779 | if (test_and_clear_bit(MPTCP_NOSPACE, &mptcp_sk(sk)->flags)) | |
780 | sk_stream_write_space(sk); | |
781 | } | |
782 | } | |
783 | ||
c0bf3c6a | 784 | void mptcp_destroy_common(struct mptcp_sock *msk, unsigned int flags); |
cec37a6e | 785 | |
c68a0cd1 JW |
786 | #define MPTCP_TOKEN_MAX_RETRIES 4 |
787 | ||
2c5ebd00 PA |
788 | void __init mptcp_token_init(void); |
789 | static inline void mptcp_token_init_request(struct request_sock *req) | |
790 | { | |
791 | mptcp_subflow_rsk(req)->token_node.pprev = NULL; | |
792 | } | |
793 | ||
79c0949e | 794 | int mptcp_token_new_request(struct request_sock *req); |
2c5ebd00 | 795 | void mptcp_token_destroy_request(struct request_sock *req); |
294de909 | 796 | int mptcp_token_new_connect(struct sock *ssk); |
2c5ebd00 PA |
797 | void mptcp_token_accept(struct mptcp_subflow_request_sock *r, |
798 | struct mptcp_sock *msk); | |
c83a47e5 | 799 | bool mptcp_token_exists(u32 token); |
ea1300b9 | 800 | struct mptcp_sock *mptcp_token_get_sock(struct net *net, u32 token); |
96d890da PA |
801 | struct mptcp_sock *mptcp_token_iter_next(const struct net *net, long *s_slot, |
802 | long *s_num); | |
2c5ebd00 | 803 | void mptcp_token_destroy(struct mptcp_sock *msk); |
79c0949e PK |
804 | |
805 | void mptcp_crypto_key_sha(u64 key, u32 *token, u64 *idsn); | |
79c0949e | 806 | |
3df523ab | 807 | void mptcp_crypto_hmac_sha(u64 key1, u64 key2, u8 *msg, int len, void *hmac); |
ba2c89e0 | 808 | __sum16 __mptcp_make_csum(u64 data_seq, u32 subflow_seq, u16 data_len, __wsum sum); |
79c0949e | 809 | |
d39dceca | 810 | void __init mptcp_pm_init(void); |
1b1c7a0e | 811 | void mptcp_pm_data_init(struct mptcp_sock *msk); |
b29fcfb5 | 812 | void mptcp_pm_data_reset(struct mptcp_sock *msk); |
702c2f64 FW |
813 | int mptcp_pm_parse_addr(struct nlattr *attr, struct genl_info *info, |
814 | struct mptcp_addr_info *addr); | |
9ab4807c KM |
815 | int mptcp_pm_parse_entry(struct nlattr *attr, struct genl_info *info, |
816 | bool require_family, | |
817 | struct mptcp_pm_addr_entry *entry); | |
fb00ee4f MB |
818 | bool mptcp_pm_addr_families_match(const struct sock *sk, |
819 | const struct mptcp_addr_info *loc, | |
820 | const struct mptcp_addr_info *rem); | |
71b7dec2 | 821 | void mptcp_pm_subflow_chk_stale(const struct mptcp_sock *msk, struct sock *ssk); |
ff5a0b42 | 822 | void mptcp_pm_nl_subflow_chk_stale(const struct mptcp_sock *msk, struct sock *ssk); |
6c714f1b | 823 | void mptcp_pm_new_connection(struct mptcp_sock *msk, const struct sock *ssk, int server_side); |
7a486c44 | 824 | void mptcp_pm_fully_established(struct mptcp_sock *msk, const struct sock *ssk); |
1b1c7a0e PK |
825 | bool mptcp_pm_allow_new_subflow(struct mptcp_sock *msk); |
826 | void mptcp_pm_connection_closed(struct mptcp_sock *msk); | |
62535200 | 827 | void mptcp_pm_subflow_established(struct mptcp_sock *msk); |
a88c9e49 PA |
828 | bool mptcp_pm_nl_check_work_pending(struct mptcp_sock *msk); |
829 | void mptcp_pm_subflow_check_next(struct mptcp_sock *msk, const struct sock *ssk, | |
830 | const struct mptcp_subflow_context *subflow); | |
d1ace2d9 | 831 | void mptcp_pm_add_addr_received(const struct sock *ssk, |
1b1c7a0e | 832 | const struct mptcp_addr_info *addr); |
557963c3 | 833 | void mptcp_pm_add_addr_echoed(struct mptcp_sock *msk, |
90d93088 | 834 | const struct mptcp_addr_info *addr); |
84dfe367 | 835 | void mptcp_pm_add_addr_send_ack(struct mptcp_sock *msk); |
b46a0238 | 836 | void mptcp_pm_nl_addr_send_ack(struct mptcp_sock *msk); |
5c4a824d GT |
837 | void mptcp_pm_rm_addr_received(struct mptcp_sock *msk, |
838 | const struct mptcp_rm_list *rm_list); | |
40453a5c | 839 | void mptcp_pm_mp_prio_received(struct sock *sk, u8 bkup); |
5580d41b | 840 | void mptcp_pm_mp_fail_received(struct sock *sk, u64 fail_seq); |
892f396c KM |
841 | int mptcp_pm_nl_mp_prio_send_ack(struct mptcp_sock *msk, |
842 | struct mptcp_addr_info *addr, | |
843 | struct mptcp_addr_info *rem, | |
844 | u8 bkup); | |
9ab4807c | 845 | bool mptcp_pm_alloc_anno_list(struct mptcp_sock *msk, |
528cb5f2 | 846 | const struct mptcp_addr_info *addr); |
b6c08380 | 847 | void mptcp_pm_free_anno_list(struct mptcp_sock *msk); |
5bc56388 | 848 | bool mptcp_pm_sport_in_anno_list(struct mptcp_sock *msk, const struct sock *sk); |
00cfd77b GT |
849 | struct mptcp_pm_add_entry * |
850 | mptcp_pm_del_add_timer(struct mptcp_sock *msk, | |
90d93088 | 851 | const struct mptcp_addr_info *addr, bool check_id); |
d88c476f | 852 | struct mptcp_pm_add_entry * |
90d93088 PA |
853 | mptcp_lookup_anno_list_by_saddr(const struct mptcp_sock *msk, |
854 | const struct mptcp_addr_info *addr); | |
8b201370 KM |
855 | int mptcp_pm_get_flags_and_ifindex_by_id(struct mptcp_sock *msk, |
856 | unsigned int id, | |
ee285257 | 857 | u8 *flags, int *ifindex); |
f40be0db GT |
858 | int mptcp_pm_nl_get_flags_and_ifindex_by_id(struct mptcp_sock *msk, unsigned int id, |
859 | u8 *flags, int *ifindex); | |
8b201370 KM |
860 | int mptcp_userspace_pm_get_flags_and_ifindex_by_id(struct mptcp_sock *msk, |
861 | unsigned int id, | |
862 | u8 *flags, int *ifindex); | |
6ba7ce89 GT |
863 | int mptcp_pm_set_flags(struct net *net, struct nlattr *token, |
864 | struct mptcp_pm_addr_entry *loc, | |
865 | struct mptcp_pm_addr_entry *rem, u8 bkup); | |
866 | int mptcp_pm_nl_set_flags(struct net *net, struct mptcp_pm_addr_entry *addr, u8 bkup); | |
892f396c KM |
867 | int mptcp_userspace_pm_set_flags(struct net *net, struct nlattr *token, |
868 | struct mptcp_pm_addr_entry *loc, | |
869 | struct mptcp_pm_addr_entry *rem, u8 bkup); | |
1b1c7a0e | 870 | int mptcp_pm_announce_addr(struct mptcp_sock *msk, |
6a6c05a8 | 871 | const struct mptcp_addr_info *addr, |
f7efc777 | 872 | bool echo); |
cbde2787 | 873 | int mptcp_pm_remove_addr(struct mptcp_sock *msk, const struct mptcp_rm_list *rm_list); |
ddd14bb8 | 874 | int mptcp_pm_remove_subflow(struct mptcp_sock *msk, const struct mptcp_rm_list *rm_list); |
8b1c94da | 875 | void mptcp_pm_remove_addrs(struct mptcp_sock *msk, struct list_head *rm_list); |
d9a4594e KM |
876 | void mptcp_pm_remove_addrs_and_subflows(struct mptcp_sock *msk, |
877 | struct list_head *rm_list); | |
878 | ||
4638de5a | 879 | void mptcp_free_local_addr_list(struct mptcp_sock *msk); |
9ab4807c | 880 | int mptcp_nl_cmd_announce(struct sk_buff *skb, struct genl_info *info); |
d9a4594e | 881 | int mptcp_nl_cmd_remove(struct sk_buff *skb, struct genl_info *info); |
702c2f64 FW |
882 | int mptcp_nl_cmd_sf_create(struct sk_buff *skb, struct genl_info *info); |
883 | int mptcp_nl_cmd_sf_destroy(struct sk_buff *skb, struct genl_info *info); | |
1b1c7a0e | 884 | |
b911c97c FW |
885 | void mptcp_event(enum mptcp_event_type type, const struct mptcp_sock *msk, |
886 | const struct sock *ssk, gfp_t gfp); | |
d1ace2d9 | 887 | void mptcp_event_addr_announced(const struct sock *ssk, const struct mptcp_addr_info *info); |
b911c97c | 888 | void mptcp_event_addr_removed(const struct mptcp_sock *msk, u8 id); |
f8c9dfbd GT |
889 | void mptcp_event_pm_listener(const struct sock *ssk, |
890 | enum mptcp_event_type event); | |
4d25247d | 891 | bool mptcp_userspace_pm_active(const struct mptcp_sock *msk); |
b911c97c | 892 | |
dfc8d060 DS |
893 | void mptcp_fastopen_gen_msk_ackseq(struct mptcp_sock *msk, struct mptcp_subflow_context *subflow, |
894 | const struct mptcp_options_received *mp_opt); | |
36b122ba DS |
895 | void mptcp_fastopen_subflow_synack_set_params(struct mptcp_subflow_context *subflow, |
896 | struct request_sock *req); | |
dfc8d060 | 897 | |
f643b803 | 898 | static inline bool mptcp_pm_should_add_signal(struct mptcp_sock *msk) |
18fc1a92 YL |
899 | { |
900 | return READ_ONCE(msk->pm.addr_signal) & | |
901 | (BIT(MPTCP_ADD_ADDR_SIGNAL) | BIT(MPTCP_ADD_ADDR_ECHO)); | |
902 | } | |
903 | ||
904 | static inline bool mptcp_pm_should_add_signal_addr(struct mptcp_sock *msk) | |
1b1c7a0e | 905 | { |
13ad9f01 | 906 | return READ_ONCE(msk->pm.addr_signal) & BIT(MPTCP_ADD_ADDR_SIGNAL); |
d91d322a GT |
907 | } |
908 | ||
909 | static inline bool mptcp_pm_should_add_signal_echo(struct mptcp_sock *msk) | |
910 | { | |
13ad9f01 | 911 | return READ_ONCE(msk->pm.addr_signal) & BIT(MPTCP_ADD_ADDR_ECHO); |
1b1c7a0e PK |
912 | } |
913 | ||
5cb104ae GT |
914 | static inline bool mptcp_pm_should_rm_signal(struct mptcp_sock *msk) |
915 | { | |
13ad9f01 | 916 | return READ_ONCE(msk->pm.addr_signal) & BIT(MPTCP_RM_ADDR_SIGNAL); |
5cb104ae GT |
917 | } |
918 | ||
14b06811 MM |
919 | static inline bool mptcp_pm_is_userspace(const struct mptcp_sock *msk) |
920 | { | |
921 | return READ_ONCE(msk->pm.pm_type) == MPTCP_PM_TYPE_USERSPACE; | |
922 | } | |
923 | ||
4d25247d KM |
924 | static inline bool mptcp_pm_is_kernel(const struct mptcp_sock *msk) |
925 | { | |
926 | return READ_ONCE(msk->pm.pm_type) == MPTCP_PM_TYPE_KERNEL; | |
927 | } | |
928 | ||
4a2777a8 | 929 | static inline unsigned int mptcp_add_addr_len(int family, bool echo, bool port) |
1b1c7a0e | 930 | { |
2ec72fae GT |
931 | u8 len = TCPOLEN_MPTCP_ADD_ADDR_BASE; |
932 | ||
933 | if (family == AF_INET6) | |
934 | len = TCPOLEN_MPTCP_ADD_ADDR6_BASE; | |
935 | if (!echo) | |
936 | len += MPTCPOPT_THMAC_LEN; | |
27ab92d9 | 937 | /* account for 2 trailing 'nop' options */ |
4a2777a8 | 938 | if (port) |
27ab92d9 | 939 | len += TCPOLEN_MPTCP_PORT_LEN + TCPOLEN_MPTCP_PORT_ALIGN; |
2ec72fae GT |
940 | |
941 | return len; | |
1b1c7a0e PK |
942 | } |
943 | ||
6445e17a GT |
944 | static inline int mptcp_rm_addr_len(const struct mptcp_rm_list *rm_list) |
945 | { | |
946 | if (rm_list->nr == 0 || rm_list->nr > MPTCP_RM_IDS_MAX) | |
947 | return -EINVAL; | |
948 | ||
949 | return TCPOLEN_MPTCP_RM_ADDR_BASE + roundup(rm_list->nr - 1, 4) + 1; | |
950 | } | |
951 | ||
90d93088 | 952 | bool mptcp_pm_add_addr_signal(struct mptcp_sock *msk, const struct sk_buff *skb, |
1f5e9e2f | 953 | unsigned int opt_size, unsigned int remaining, |
f462a446 | 954 | struct mptcp_addr_info *addr, bool *echo, |
af7939f3 | 955 | bool *drop_other_suboptions); |
5cb104ae | 956 | bool mptcp_pm_rm_addr_signal(struct mptcp_sock *msk, unsigned int remaining, |
6445e17a | 957 | struct mptcp_rm_list *rm_list); |
1b1c7a0e | 958 | int mptcp_pm_get_local_id(struct mptcp_sock *msk, struct sock_common *skc); |
9bbec87e | 959 | int mptcp_pm_nl_get_local_id(struct mptcp_sock *msk, struct mptcp_addr_info *skc); |
8b201370 | 960 | int mptcp_userspace_pm_get_local_id(struct mptcp_sock *msk, struct mptcp_addr_info *skc); |
1b1c7a0e | 961 | |
d39dceca | 962 | void __init mptcp_pm_nl_init(void); |
e9801430 | 963 | void mptcp_pm_nl_work(struct mptcp_sock *msk); |
ddd14bb8 GT |
964 | void mptcp_pm_nl_rm_subflow_received(struct mptcp_sock *msk, |
965 | const struct mptcp_rm_list *rm_list); | |
90d93088 PA |
966 | unsigned int mptcp_pm_get_add_addr_signal_max(const struct mptcp_sock *msk); |
967 | unsigned int mptcp_pm_get_add_addr_accept_max(const struct mptcp_sock *msk); | |
968 | unsigned int mptcp_pm_get_subflows_max(const struct mptcp_sock *msk); | |
969 | unsigned int mptcp_pm_get_local_addr_max(const struct mptcp_sock *msk); | |
78962489 | 970 | |
95d68651 PA |
971 | /* called under PM lock */ |
972 | static inline void __mptcp_pm_close_subflow(struct mptcp_sock *msk) | |
973 | { | |
974 | if (--msk->pm.subflows < mptcp_pm_get_subflows_max(msk)) | |
975 | WRITE_ONCE(msk->pm.accept_subflow, true); | |
976 | } | |
977 | ||
978 | static inline void mptcp_pm_close_subflow(struct mptcp_sock *msk) | |
979 | { | |
980 | spin_lock_bh(&msk->pm.lock); | |
981 | __mptcp_pm_close_subflow(msk); | |
982 | spin_unlock_bh(&msk->pm.lock); | |
983 | } | |
984 | ||
78962489 | 985 | void mptcp_sockopt_sync(struct mptcp_sock *msk, struct sock *ssk); |
3e501490 | 986 | void mptcp_sockopt_sync_locked(struct mptcp_sock *msk, struct sock *ssk); |
01cacb00 | 987 | |
dc87efdb | 988 | static inline struct mptcp_ext *mptcp_get_ext(const struct sk_buff *skb) |
6d0060f6 MM |
989 | { |
990 | return (struct mptcp_ext *)skb_ext_find(skb, SKB_EXT_MPTCP); | |
991 | } | |
992 | ||
5147dfb5 DC |
993 | void mptcp_diag_subflow_init(struct tcp_ulp_ops *ops); |
994 | ||
d5f49190 | 995 | static inline bool __mptcp_check_fallback(const struct mptcp_sock *msk) |
e1ff9e82 DC |
996 | { |
997 | return test_bit(MPTCP_FALLBACK_DONE, &msk->flags); | |
998 | } | |
999 | ||
d5f49190 | 1000 | static inline bool mptcp_check_fallback(const struct sock *sk) |
e1ff9e82 DC |
1001 | { |
1002 | struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk); | |
1003 | struct mptcp_sock *msk = mptcp_sk(subflow->conn); | |
1004 | ||
1005 | return __mptcp_check_fallback(msk); | |
1006 | } | |
1007 | ||
1008 | static inline void __mptcp_do_fallback(struct mptcp_sock *msk) | |
1009 | { | |
1010 | if (test_bit(MPTCP_FALLBACK_DONE, &msk->flags)) { | |
1011 | pr_debug("TCP fallback already done (msk=%p)", msk); | |
1012 | return; | |
1013 | } | |
1014 | set_bit(MPTCP_FALLBACK_DONE, &msk->flags); | |
1015 | } | |
1016 | ||
d51991e2 | 1017 | static inline void mptcp_do_fallback(struct sock *ssk) |
e1ff9e82 | 1018 | { |
d51991e2 PA |
1019 | struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk); |
1020 | struct sock *sk = subflow->conn; | |
1021 | struct mptcp_sock *msk; | |
e1ff9e82 | 1022 | |
d51991e2 | 1023 | msk = mptcp_sk(sk); |
e1ff9e82 | 1024 | __mptcp_do_fallback(msk); |
d51991e2 PA |
1025 | if (READ_ONCE(msk->snd_data_fin_enable) && !(ssk->sk_shutdown & SEND_SHUTDOWN)) { |
1026 | gfp_t saved_allocation = ssk->sk_allocation; | |
1027 | ||
1028 | /* we are in a atomic (BH) scope, override ssk default for data | |
1029 | * fin allocation | |
1030 | */ | |
1031 | ssk->sk_allocation = GFP_ATOMIC; | |
1032 | ssk->sk_shutdown |= SEND_SHUTDOWN; | |
1033 | tcp_shutdown(ssk, SEND_SHUTDOWN); | |
1034 | ssk->sk_allocation = saved_allocation; | |
1035 | } | |
e1ff9e82 DC |
1036 | } |
1037 | ||
1038 | #define pr_fallback(a) pr_debug("%s:fallback to TCP (msk=%p)", __func__, a) | |
1039 | ||
1e39e5a3 GT |
1040 | static inline bool mptcp_check_infinite_map(struct sk_buff *skb) |
1041 | { | |
1042 | struct mptcp_ext *mpext; | |
1043 | ||
1044 | mpext = skb ? mptcp_get_ext(skb) : NULL; | |
1045 | if (mpext && mpext->infinite_map) | |
1046 | return true; | |
1047 | ||
1048 | return false; | |
1049 | } | |
1050 | ||
70c708e8 KM |
1051 | static inline bool is_active_ssk(struct mptcp_subflow_context *subflow) |
1052 | { | |
1053 | return (subflow->request_mptcp || subflow->request_join); | |
1054 | } | |
1055 | ||
8fd73804 DC |
1056 | static inline bool subflow_simultaneous_connect(struct sock *sk) |
1057 | { | |
1058 | struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk); | |
8fd73804 DC |
1059 | |
1060 | return sk->sk_state == TCP_ESTABLISHED && | |
70c708e8 | 1061 | is_active_ssk(subflow) && |
8fd73804 DC |
1062 | !subflow->conn_finished; |
1063 | } | |
1064 | ||
9466a1cc FW |
1065 | #ifdef CONFIG_SYN_COOKIES |
1066 | void subflow_init_req_cookie_join_save(const struct mptcp_subflow_request_sock *subflow_req, | |
1067 | struct sk_buff *skb); | |
1068 | bool mptcp_token_join_cookie_init_state(struct mptcp_subflow_request_sock *subflow_req, | |
1069 | struct sk_buff *skb); | |
1070 | void __init mptcp_join_cookie_init(void); | |
1071 | #else | |
1072 | static inline void | |
1073 | subflow_init_req_cookie_join_save(const struct mptcp_subflow_request_sock *subflow_req, | |
1074 | struct sk_buff *skb) {} | |
1075 | static inline bool | |
1076 | mptcp_token_join_cookie_init_state(struct mptcp_subflow_request_sock *subflow_req, | |
1077 | struct sk_buff *skb) | |
1078 | { | |
1079 | return false; | |
1080 | } | |
1081 | ||
1082 | static inline void mptcp_join_cookie_init(void) {} | |
1083 | #endif | |
1084 | ||
f870fa0b | 1085 | #endif /* __MPTCP_PROTOCOL_H */ |