]>
Commit | Line | Data |
---|---|---|
604326b4 DB |
1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* Copyright (c) 2017 - 2018 Covalent IO, Inc. http://covalent.io */ | |
3 | ||
4 | #include <linux/skmsg.h> | |
5 | #include <linux/filter.h> | |
6 | #include <linux/bpf.h> | |
7 | #include <linux/init.h> | |
8 | #include <linux/wait.h> | |
9 | ||
10 | #include <net/inet_common.h> | |
0608c69c | 11 | #include <net/tls.h> |
604326b4 | 12 | |
604326b4 | 13 | int __tcp_bpf_recvmsg(struct sock *sk, struct sk_psock *psock, |
02c558b2 | 14 | struct msghdr *msg, int len, int flags) |
604326b4 DB |
15 | { |
16 | struct iov_iter *iter = &msg->msg_iter; | |
02c558b2 | 17 | int peek = flags & MSG_PEEK; |
604326b4 | 18 | int i, ret, copied = 0; |
02c558b2 JF |
19 | struct sk_msg *msg_rx; |
20 | ||
21 | msg_rx = list_first_entry_or_null(&psock->ingress_msg, | |
22 | struct sk_msg, list); | |
604326b4 DB |
23 | |
24 | while (copied != len) { | |
25 | struct scatterlist *sge; | |
604326b4 | 26 | |
604326b4 DB |
27 | if (unlikely(!msg_rx)) |
28 | break; | |
29 | ||
30 | i = msg_rx->sg.start; | |
31 | do { | |
32 | struct page *page; | |
33 | int copy; | |
34 | ||
35 | sge = sk_msg_elem(msg_rx, i); | |
36 | copy = sge->length; | |
37 | page = sg_page(sge); | |
38 | if (copied + copy > len) | |
39 | copy = len - copied; | |
40 | ret = copy_page_to_iter(page, sge->offset, copy, iter); | |
41 | if (ret != copy) { | |
42 | msg_rx->sg.start = i; | |
43 | return -EFAULT; | |
44 | } | |
45 | ||
46 | copied += copy; | |
02c558b2 JF |
47 | if (likely(!peek)) { |
48 | sge->offset += copy; | |
49 | sge->length -= copy; | |
50 | sk_mem_uncharge(sk, copy); | |
51 | msg_rx->sg.size -= copy; | |
52 | ||
53 | if (!sge->length) { | |
54 | sk_msg_iter_var_next(i); | |
55 | if (!msg_rx->skb) | |
56 | put_page(page); | |
57 | } | |
58 | } else { | |
59 | sk_msg_iter_var_next(i); | |
604326b4 DB |
60 | } |
61 | ||
62 | if (copied == len) | |
63 | break; | |
64 | } while (i != msg_rx->sg.end); | |
65 | ||
02c558b2 JF |
66 | if (unlikely(peek)) { |
67 | msg_rx = list_next_entry(msg_rx, list); | |
68 | continue; | |
69 | } | |
70 | ||
604326b4 DB |
71 | msg_rx->sg.start = i; |
72 | if (!sge->length && msg_rx->sg.start == msg_rx->sg.end) { | |
73 | list_del(&msg_rx->list); | |
74 | if (msg_rx->skb) | |
75 | consume_skb(msg_rx->skb); | |
76 | kfree(msg_rx); | |
77 | } | |
02c558b2 JF |
78 | msg_rx = list_first_entry_or_null(&psock->ingress_msg, |
79 | struct sk_msg, list); | |
604326b4 DB |
80 | } |
81 | ||
82 | return copied; | |
83 | } | |
84 | EXPORT_SYMBOL_GPL(__tcp_bpf_recvmsg); | |
85 | ||
604326b4 DB |
86 | static int bpf_tcp_ingress(struct sock *sk, struct sk_psock *psock, |
87 | struct sk_msg *msg, u32 apply_bytes, int flags) | |
88 | { | |
89 | bool apply = apply_bytes; | |
90 | struct scatterlist *sge; | |
91 | u32 size, copied = 0; | |
92 | struct sk_msg *tmp; | |
93 | int i, ret = 0; | |
94 | ||
95 | tmp = kzalloc(sizeof(*tmp), __GFP_NOWARN | GFP_KERNEL); | |
96 | if (unlikely(!tmp)) | |
97 | return -ENOMEM; | |
98 | ||
99 | lock_sock(sk); | |
100 | tmp->sg.start = msg->sg.start; | |
101 | i = msg->sg.start; | |
102 | do { | |
103 | sge = sk_msg_elem(msg, i); | |
104 | size = (apply && apply_bytes < sge->length) ? | |
105 | apply_bytes : sge->length; | |
106 | if (!sk_wmem_schedule(sk, size)) { | |
107 | if (!copied) | |
108 | ret = -ENOMEM; | |
109 | break; | |
110 | } | |
111 | ||
112 | sk_mem_charge(sk, size); | |
113 | sk_msg_xfer(tmp, msg, i, size); | |
114 | copied += size; | |
115 | if (sge->length) | |
116 | get_page(sk_msg_page(tmp, i)); | |
117 | sk_msg_iter_var_next(i); | |
118 | tmp->sg.end = i; | |
119 | if (apply) { | |
120 | apply_bytes -= size; | |
121 | if (!apply_bytes) | |
122 | break; | |
123 | } | |
124 | } while (i != msg->sg.end); | |
125 | ||
126 | if (!ret) { | |
127 | msg->sg.start = i; | |
604326b4 | 128 | sk_psock_queue_msg(psock, tmp); |
552de910 | 129 | sk_psock_data_ready(sk, psock); |
604326b4 DB |
130 | } else { |
131 | sk_msg_free(sk, tmp); | |
132 | kfree(tmp); | |
133 | } | |
134 | ||
135 | release_sock(sk); | |
136 | return ret; | |
137 | } | |
138 | ||
139 | static int tcp_bpf_push(struct sock *sk, struct sk_msg *msg, u32 apply_bytes, | |
140 | int flags, bool uncharge) | |
141 | { | |
142 | bool apply = apply_bytes; | |
143 | struct scatterlist *sge; | |
144 | struct page *page; | |
145 | int size, ret = 0; | |
146 | u32 off; | |
147 | ||
148 | while (1) { | |
0608c69c JF |
149 | bool has_tx_ulp; |
150 | ||
604326b4 DB |
151 | sge = sk_msg_elem(msg, msg->sg.start); |
152 | size = (apply && apply_bytes < sge->length) ? | |
153 | apply_bytes : sge->length; | |
154 | off = sge->offset; | |
155 | page = sg_page(sge); | |
156 | ||
157 | tcp_rate_check_app_limited(sk); | |
158 | retry: | |
0608c69c JF |
159 | has_tx_ulp = tls_sw_has_ctx_tx(sk); |
160 | if (has_tx_ulp) { | |
161 | flags |= MSG_SENDPAGE_NOPOLICY; | |
162 | ret = kernel_sendpage_locked(sk, | |
163 | page, off, size, flags); | |
164 | } else { | |
165 | ret = do_tcp_sendpages(sk, page, off, size, flags); | |
166 | } | |
167 | ||
604326b4 DB |
168 | if (ret <= 0) |
169 | return ret; | |
170 | if (apply) | |
171 | apply_bytes -= ret; | |
172 | msg->sg.size -= ret; | |
173 | sge->offset += ret; | |
174 | sge->length -= ret; | |
175 | if (uncharge) | |
176 | sk_mem_uncharge(sk, ret); | |
177 | if (ret != size) { | |
178 | size -= ret; | |
179 | off += ret; | |
180 | goto retry; | |
181 | } | |
182 | if (!sge->length) { | |
183 | put_page(page); | |
184 | sk_msg_iter_next(msg, start); | |
185 | sg_init_table(sge, 1); | |
186 | if (msg->sg.start == msg->sg.end) | |
187 | break; | |
188 | } | |
189 | if (apply && !apply_bytes) | |
190 | break; | |
191 | } | |
192 | ||
193 | return 0; | |
194 | } | |
195 | ||
196 | static int tcp_bpf_push_locked(struct sock *sk, struct sk_msg *msg, | |
197 | u32 apply_bytes, int flags, bool uncharge) | |
198 | { | |
199 | int ret; | |
200 | ||
201 | lock_sock(sk); | |
202 | ret = tcp_bpf_push(sk, msg, apply_bytes, flags, uncharge); | |
203 | release_sock(sk); | |
204 | return ret; | |
205 | } | |
206 | ||
207 | int tcp_bpf_sendmsg_redir(struct sock *sk, struct sk_msg *msg, | |
208 | u32 bytes, int flags) | |
209 | { | |
210 | bool ingress = sk_msg_to_ingress(msg); | |
211 | struct sk_psock *psock = sk_psock_get(sk); | |
212 | int ret; | |
213 | ||
214 | if (unlikely(!psock)) { | |
215 | sk_msg_free(sk, msg); | |
216 | return 0; | |
217 | } | |
218 | ret = ingress ? bpf_tcp_ingress(sk, psock, msg, bytes, flags) : | |
219 | tcp_bpf_push_locked(sk, msg, bytes, flags, false); | |
220 | sk_psock_put(sk, psock); | |
221 | return ret; | |
222 | } | |
223 | EXPORT_SYMBOL_GPL(tcp_bpf_sendmsg_redir); | |
224 | ||
a2652798 Y |
225 | #ifdef CONFIG_BPF_STREAM_PARSER |
226 | static bool tcp_bpf_stream_read(const struct sock *sk) | |
227 | { | |
228 | struct sk_psock *psock; | |
229 | bool empty = true; | |
230 | ||
231 | rcu_read_lock(); | |
232 | psock = sk_psock(sk); | |
233 | if (likely(psock)) | |
234 | empty = list_empty(&psock->ingress_msg); | |
235 | rcu_read_unlock(); | |
236 | return !empty; | |
237 | } | |
238 | ||
c0fd336e Y |
239 | static int tcp_bpf_wait_data(struct sock *sk, struct sk_psock *psock, |
240 | int flags, long timeo, int *err) | |
241 | { | |
242 | DEFINE_WAIT_FUNC(wait, woken_wake_function); | |
243 | int ret = 0; | |
244 | ||
245 | if (!timeo) | |
246 | return ret; | |
247 | ||
248 | add_wait_queue(sk_sleep(sk), &wait); | |
249 | sk_set_bit(SOCKWQ_ASYNC_WAITDATA, sk); | |
250 | ret = sk_wait_event(sk, &timeo, | |
251 | !list_empty(&psock->ingress_msg) || | |
252 | !skb_queue_empty(&sk->sk_receive_queue), &wait); | |
253 | sk_clear_bit(SOCKWQ_ASYNC_WAITDATA, sk); | |
254 | remove_wait_queue(sk_sleep(sk), &wait); | |
255 | return ret; | |
256 | } | |
257 | ||
258 | static int tcp_bpf_recvmsg(struct sock *sk, struct msghdr *msg, size_t len, | |
259 | int nonblock, int flags, int *addr_len) | |
260 | { | |
261 | struct sk_psock *psock; | |
262 | int copied, ret; | |
263 | ||
18f02ad1 XY |
264 | if (unlikely(flags & MSG_ERRQUEUE)) |
265 | return inet_recv_error(sk, msg, len, addr_len); | |
266 | ||
c0fd336e Y |
267 | psock = sk_psock_get(sk); |
268 | if (unlikely(!psock)) | |
269 | return tcp_recvmsg(sk, msg, len, nonblock, flags, addr_len); | |
c0fd336e | 270 | if (!skb_queue_empty(&sk->sk_receive_queue) && |
18f02ad1 XY |
271 | sk_psock_queue_empty(psock)) { |
272 | sk_psock_put(sk, psock); | |
c0fd336e | 273 | return tcp_recvmsg(sk, msg, len, nonblock, flags, addr_len); |
18f02ad1 | 274 | } |
c0fd336e Y |
275 | lock_sock(sk); |
276 | msg_bytes_ready: | |
277 | copied = __tcp_bpf_recvmsg(sk, psock, msg, len, flags); | |
278 | if (!copied) { | |
279 | int data, err = 0; | |
280 | long timeo; | |
281 | ||
282 | timeo = sock_rcvtimeo(sk, nonblock); | |
283 | data = tcp_bpf_wait_data(sk, psock, flags, timeo, &err); | |
284 | if (data) { | |
285 | if (!sk_psock_queue_empty(psock)) | |
286 | goto msg_bytes_ready; | |
287 | release_sock(sk); | |
288 | sk_psock_put(sk, psock); | |
289 | return tcp_recvmsg(sk, msg, len, nonblock, flags, addr_len); | |
290 | } | |
291 | if (err) { | |
292 | ret = err; | |
293 | goto out; | |
294 | } | |
295 | copied = -EAGAIN; | |
296 | } | |
297 | ret = copied; | |
298 | out: | |
299 | release_sock(sk); | |
300 | sk_psock_put(sk, psock); | |
301 | return ret; | |
302 | } | |
303 | ||
604326b4 DB |
304 | static int tcp_bpf_send_verdict(struct sock *sk, struct sk_psock *psock, |
305 | struct sk_msg *msg, int *copied, int flags) | |
306 | { | |
031097d9 | 307 | bool cork = false, enospc = sk_msg_full(msg); |
604326b4 | 308 | struct sock *sk_redir; |
7246d8ed | 309 | u32 tosend, delta = 0; |
604326b4 DB |
310 | int ret; |
311 | ||
312 | more_data: | |
7246d8ed JF |
313 | if (psock->eval == __SK_NONE) { |
314 | /* Track delta in msg size to add/subtract it on SK_DROP from | |
315 | * returned to user copied size. This ensures user doesn't | |
316 | * get a positive return code with msg_cut_data and SK_DROP | |
317 | * verdict. | |
318 | */ | |
319 | delta = msg->sg.size; | |
604326b4 | 320 | psock->eval = sk_psock_msg_verdict(sk, psock, msg); |
7361d448 | 321 | delta -= msg->sg.size; |
7246d8ed | 322 | } |
604326b4 DB |
323 | |
324 | if (msg->cork_bytes && | |
325 | msg->cork_bytes > msg->sg.size && !enospc) { | |
326 | psock->cork_bytes = msg->cork_bytes - msg->sg.size; | |
327 | if (!psock->cork) { | |
328 | psock->cork = kzalloc(sizeof(*psock->cork), | |
329 | GFP_ATOMIC | __GFP_NOWARN); | |
330 | if (!psock->cork) | |
331 | return -ENOMEM; | |
332 | } | |
333 | memcpy(psock->cork, msg, sizeof(*msg)); | |
334 | return 0; | |
335 | } | |
336 | ||
337 | tosend = msg->sg.size; | |
338 | if (psock->apply_bytes && psock->apply_bytes < tosend) | |
339 | tosend = psock->apply_bytes; | |
340 | ||
341 | switch (psock->eval) { | |
342 | case __SK_PASS: | |
343 | ret = tcp_bpf_push(sk, msg, tosend, flags, true); | |
344 | if (unlikely(ret)) { | |
345 | *copied -= sk_msg_free(sk, msg); | |
346 | break; | |
347 | } | |
348 | sk_msg_apply_bytes(psock, tosend); | |
349 | break; | |
350 | case __SK_REDIRECT: | |
351 | sk_redir = psock->sk_redir; | |
352 | sk_msg_apply_bytes(psock, tosend); | |
353 | if (psock->cork) { | |
354 | cork = true; | |
355 | psock->cork = NULL; | |
356 | } | |
357 | sk_msg_return(sk, msg, tosend); | |
358 | release_sock(sk); | |
359 | ret = tcp_bpf_sendmsg_redir(sk_redir, msg, tosend, flags); | |
360 | lock_sock(sk); | |
361 | if (unlikely(ret < 0)) { | |
362 | int free = sk_msg_free_nocharge(sk, msg); | |
363 | ||
364 | if (!cork) | |
365 | *copied -= free; | |
366 | } | |
367 | if (cork) { | |
368 | sk_msg_free(sk, msg); | |
369 | kfree(msg); | |
370 | msg = NULL; | |
371 | ret = 0; | |
372 | } | |
373 | break; | |
374 | case __SK_DROP: | |
375 | default: | |
376 | sk_msg_free_partial(sk, msg, tosend); | |
377 | sk_msg_apply_bytes(psock, tosend); | |
7246d8ed | 378 | *copied -= (tosend + delta); |
604326b4 DB |
379 | return -EACCES; |
380 | } | |
381 | ||
382 | if (likely(!ret)) { | |
383 | if (!psock->apply_bytes) { | |
384 | psock->eval = __SK_NONE; | |
385 | if (psock->sk_redir) { | |
386 | sock_put(psock->sk_redir); | |
387 | psock->sk_redir = NULL; | |
388 | } | |
389 | } | |
390 | if (msg && | |
391 | msg->sg.data[msg->sg.start].page_link && | |
392 | msg->sg.data[msg->sg.start].length) | |
393 | goto more_data; | |
394 | } | |
395 | return ret; | |
396 | } | |
397 | ||
398 | static int tcp_bpf_sendmsg(struct sock *sk, struct msghdr *msg, size_t size) | |
399 | { | |
400 | struct sk_msg tmp, *msg_tx = NULL; | |
604326b4 DB |
401 | int copied = 0, err = 0; |
402 | struct sk_psock *psock; | |
403 | long timeo; | |
41477662 JK |
404 | int flags; |
405 | ||
406 | /* Don't let internal do_tcp_sendpages() flags through */ | |
407 | flags = (msg->msg_flags & ~MSG_SENDPAGE_DECRYPTED); | |
408 | flags |= MSG_NO_SHARED_FRAGS; | |
604326b4 DB |
409 | |
410 | psock = sk_psock_get(sk); | |
411 | if (unlikely(!psock)) | |
412 | return tcp_sendmsg(sk, msg, size); | |
413 | ||
414 | lock_sock(sk); | |
415 | timeo = sock_sndtimeo(sk, msg->msg_flags & MSG_DONTWAIT); | |
416 | while (msg_data_left(msg)) { | |
417 | bool enospc = false; | |
418 | u32 copy, osize; | |
419 | ||
420 | if (sk->sk_err) { | |
421 | err = -sk->sk_err; | |
422 | goto out_err; | |
423 | } | |
424 | ||
425 | copy = msg_data_left(msg); | |
426 | if (!sk_stream_memory_free(sk)) | |
427 | goto wait_for_sndbuf; | |
428 | if (psock->cork) { | |
429 | msg_tx = psock->cork; | |
430 | } else { | |
431 | msg_tx = &tmp; | |
432 | sk_msg_init(msg_tx); | |
433 | } | |
434 | ||
435 | osize = msg_tx->sg.size; | |
436 | err = sk_msg_alloc(sk, msg_tx, msg_tx->sg.size + copy, msg_tx->sg.end - 1); | |
437 | if (err) { | |
438 | if (err != -ENOSPC) | |
439 | goto wait_for_memory; | |
440 | enospc = true; | |
441 | copy = msg_tx->sg.size - osize; | |
442 | } | |
443 | ||
444 | err = sk_msg_memcopy_from_iter(sk, &msg->msg_iter, msg_tx, | |
445 | copy); | |
446 | if (err < 0) { | |
447 | sk_msg_trim(sk, msg_tx, osize); | |
448 | goto out_err; | |
449 | } | |
450 | ||
451 | copied += copy; | |
452 | if (psock->cork_bytes) { | |
453 | if (size > psock->cork_bytes) | |
454 | psock->cork_bytes = 0; | |
455 | else | |
456 | psock->cork_bytes -= size; | |
457 | if (psock->cork_bytes && !enospc) | |
458 | goto out_err; | |
459 | /* All cork bytes are accounted, rerun the prog. */ | |
460 | psock->eval = __SK_NONE; | |
461 | psock->cork_bytes = 0; | |
462 | } | |
463 | ||
464 | err = tcp_bpf_send_verdict(sk, psock, msg_tx, &copied, flags); | |
465 | if (unlikely(err < 0)) | |
466 | goto out_err; | |
467 | continue; | |
468 | wait_for_sndbuf: | |
469 | set_bit(SOCK_NOSPACE, &sk->sk_socket->flags); | |
470 | wait_for_memory: | |
471 | err = sk_stream_wait_memory(sk, &timeo); | |
472 | if (err) { | |
473 | if (msg_tx && msg_tx != psock->cork) | |
474 | sk_msg_free(sk, msg_tx); | |
475 | goto out_err; | |
476 | } | |
477 | } | |
478 | out_err: | |
479 | if (err < 0) | |
480 | err = sk_stream_error(sk, msg->msg_flags, err); | |
481 | release_sock(sk); | |
482 | sk_psock_put(sk, psock); | |
483 | return copied ? copied : err; | |
484 | } | |
485 | ||
486 | static int tcp_bpf_sendpage(struct sock *sk, struct page *page, int offset, | |
487 | size_t size, int flags) | |
488 | { | |
489 | struct sk_msg tmp, *msg = NULL; | |
490 | int err = 0, copied = 0; | |
491 | struct sk_psock *psock; | |
492 | bool enospc = false; | |
493 | ||
494 | psock = sk_psock_get(sk); | |
495 | if (unlikely(!psock)) | |
496 | return tcp_sendpage(sk, page, offset, size, flags); | |
497 | ||
498 | lock_sock(sk); | |
499 | if (psock->cork) { | |
500 | msg = psock->cork; | |
501 | } else { | |
502 | msg = &tmp; | |
503 | sk_msg_init(msg); | |
504 | } | |
505 | ||
506 | /* Catch case where ring is full and sendpage is stalled. */ | |
507 | if (unlikely(sk_msg_full(msg))) | |
508 | goto out_err; | |
509 | ||
510 | sk_msg_page_add(msg, page, size, offset); | |
511 | sk_mem_charge(sk, size); | |
512 | copied = size; | |
513 | if (sk_msg_full(msg)) | |
514 | enospc = true; | |
515 | if (psock->cork_bytes) { | |
516 | if (size > psock->cork_bytes) | |
517 | psock->cork_bytes = 0; | |
518 | else | |
519 | psock->cork_bytes -= size; | |
520 | if (psock->cork_bytes && !enospc) | |
521 | goto out_err; | |
522 | /* All cork bytes are accounted, rerun the prog. */ | |
523 | psock->eval = __SK_NONE; | |
524 | psock->cork_bytes = 0; | |
525 | } | |
526 | ||
527 | err = tcp_bpf_send_verdict(sk, psock, msg, &copied, flags); | |
528 | out_err: | |
529 | release_sock(sk); | |
530 | sk_psock_put(sk, psock); | |
531 | return copied ? copied : err; | |
532 | } | |
533 | ||
604326b4 DB |
534 | enum { |
535 | TCP_BPF_IPV4, | |
536 | TCP_BPF_IPV6, | |
537 | TCP_BPF_NUM_PROTS, | |
538 | }; | |
539 | ||
540 | enum { | |
541 | TCP_BPF_BASE, | |
542 | TCP_BPF_TX, | |
543 | TCP_BPF_NUM_CFGS, | |
544 | }; | |
545 | ||
546 | static struct proto *tcpv6_prot_saved __read_mostly; | |
547 | static DEFINE_SPINLOCK(tcpv6_prot_lock); | |
548 | static struct proto tcp_bpf_prots[TCP_BPF_NUM_PROTS][TCP_BPF_NUM_CFGS]; | |
549 | ||
550 | static void tcp_bpf_rebuild_protos(struct proto prot[TCP_BPF_NUM_CFGS], | |
551 | struct proto *base) | |
552 | { | |
553 | prot[TCP_BPF_BASE] = *base; | |
f747632b LB |
554 | prot[TCP_BPF_BASE].unhash = sock_map_unhash; |
555 | prot[TCP_BPF_BASE].close = sock_map_close; | |
604326b4 DB |
556 | prot[TCP_BPF_BASE].recvmsg = tcp_bpf_recvmsg; |
557 | prot[TCP_BPF_BASE].stream_memory_read = tcp_bpf_stream_read; | |
558 | ||
559 | prot[TCP_BPF_TX] = prot[TCP_BPF_BASE]; | |
560 | prot[TCP_BPF_TX].sendmsg = tcp_bpf_sendmsg; | |
561 | prot[TCP_BPF_TX].sendpage = tcp_bpf_sendpage; | |
562 | } | |
563 | ||
564 | static void tcp_bpf_check_v6_needs_rebuild(struct sock *sk, struct proto *ops) | |
565 | { | |
566 | if (sk->sk_family == AF_INET6 && | |
567 | unlikely(ops != smp_load_acquire(&tcpv6_prot_saved))) { | |
568 | spin_lock_bh(&tcpv6_prot_lock); | |
569 | if (likely(ops != tcpv6_prot_saved)) { | |
570 | tcp_bpf_rebuild_protos(tcp_bpf_prots[TCP_BPF_IPV6], ops); | |
571 | smp_store_release(&tcpv6_prot_saved, ops); | |
572 | } | |
573 | spin_unlock_bh(&tcpv6_prot_lock); | |
574 | } | |
575 | } | |
576 | ||
577 | static int __init tcp_bpf_v4_build_proto(void) | |
578 | { | |
579 | tcp_bpf_rebuild_protos(tcp_bpf_prots[TCP_BPF_IPV4], &tcp_prot); | |
580 | return 0; | |
581 | } | |
582 | core_initcall(tcp_bpf_v4_build_proto); | |
583 | ||
604326b4 DB |
584 | static int tcp_bpf_assert_proto_ops(struct proto *ops) |
585 | { | |
586 | /* In order to avoid retpoline, we make assumptions when we call | |
587 | * into ops if e.g. a psock is not present. Make sure they are | |
588 | * indeed valid assumptions. | |
589 | */ | |
590 | return ops->recvmsg == tcp_recvmsg && | |
591 | ops->sendmsg == tcp_sendmsg && | |
592 | ops->sendpage == tcp_sendpage ? 0 : -ENOTSUPP; | |
593 | } | |
594 | ||
f747632b | 595 | struct proto *tcp_bpf_get_proto(struct sock *sk, struct sk_psock *psock) |
604326b4 | 596 | { |
d19da360 LB |
597 | int family = sk->sk_family == AF_INET6 ? TCP_BPF_IPV6 : TCP_BPF_IPV4; |
598 | int config = psock->progs.msg_parser ? TCP_BPF_TX : TCP_BPF_BASE; | |
604326b4 | 599 | |
d19da360 LB |
600 | if (!psock->sk_proto) { |
601 | struct proto *ops = READ_ONCE(sk->sk_prot); | |
604326b4 | 602 | |
d19da360 LB |
603 | if (tcp_bpf_assert_proto_ops(ops)) |
604 | return ERR_PTR(-EINVAL); | |
605 | ||
606 | tcp_bpf_check_v6_needs_rebuild(sk, ops); | |
607 | } | |
608 | ||
609 | return &tcp_bpf_prots[family][config]; | |
604326b4 DB |
610 | } |
611 | ||
e8025155 JS |
612 | /* If a child got cloned from a listening socket that had tcp_bpf |
613 | * protocol callbacks installed, we need to restore the callbacks to | |
614 | * the default ones because the child does not inherit the psock state | |
615 | * that tcp_bpf callbacks expect. | |
616 | */ | |
617 | void tcp_bpf_clone(const struct sock *sk, struct sock *newsk) | |
618 | { | |
619 | int family = sk->sk_family == AF_INET6 ? TCP_BPF_IPV6 : TCP_BPF_IPV4; | |
620 | struct proto *prot = newsk->sk_prot; | |
621 | ||
622 | if (prot == &tcp_bpf_prots[family][TCP_BPF_BASE]) | |
623 | newsk->sk_prot = sk->sk_prot_creator; | |
624 | } | |
f747632b | 625 | #endif /* CONFIG_BPF_STREAM_PARSER */ |