1 // SPDX-License-Identifier: GPL-2.0-only
3 * Copyright 2018 Google Inc.
4 * Author: Eric Dumazet (edumazet@google.com)
6 * Reference program demonstrating tcp mmap() usage,
7 * and SO_RCVLOWAT hints for receiver.
9 * Note : NIC with header split is needed to use mmap() on TCP :
10 * Each incoming frame must be a multiple of PAGE_SIZE bytes of TCP payload.
12 * How to use on loopback interface :
14 * ifconfig lo mtu 61512 # 15*4096 + 40 (ipv6 header) + 32 (TCP with TS option header)
18 * Or leave default lo mtu, but use -M option to set TCP_MAXSEG option to (4096 + 12)
19 * (4096 : page size on x86, 12: TCP TS option length)
20 * tcp_mmap -s -z -M $((4096+12)) &
21 * tcp_mmap -H ::1 -z -M $((4096+12))
23 * Note: -z option on sender uses MSG_ZEROCOPY, which forces a copy when packets go through loopback interface.
24 * We might use sendfile() instead, but really this test program is about mmap(), for receivers ;)
26 * $ ./tcp_mmap -s & # Without mmap()
27 * $ for i in {1..4}; do ./tcp_mmap -H ::1 -z ; done
28 * received 32768 MB (0 % mmap'ed) in 14.1157 s, 19.4732 Gbit
29 * cpu usage user:0.057 sys:7.815, 240.234 usec per MB, 65531 c-switches
30 * received 32768 MB (0 % mmap'ed) in 14.6833 s, 18.7204 Gbit
31 * cpu usage user:0.043 sys:8.103, 248.596 usec per MB, 65524 c-switches
32 * received 32768 MB (0 % mmap'ed) in 11.143 s, 24.6682 Gbit
33 * cpu usage user:0.044 sys:6.576, 202.026 usec per MB, 65519 c-switches
34 * received 32768 MB (0 % mmap'ed) in 14.9056 s, 18.4413 Gbit
35 * cpu usage user:0.036 sys:8.193, 251.129 usec per MB, 65530 c-switches
36 * $ kill %1 # kill tcp_mmap server
38 * $ ./tcp_mmap -s -z & # With mmap()
39 * $ for i in {1..4}; do ./tcp_mmap -H ::1 -z ; done
40 * received 32768 MB (99.9939 % mmap'ed) in 6.73792 s, 40.7956 Gbit
41 * cpu usage user:0.045 sys:2.827, 87.6465 usec per MB, 65532 c-switches
42 * received 32768 MB (99.9939 % mmap'ed) in 7.26732 s, 37.8238 Gbit
43 * cpu usage user:0.037 sys:3.087, 95.3369 usec per MB, 65532 c-switches
44 * received 32768 MB (99.9939 % mmap'ed) in 7.61661 s, 36.0893 Gbit
45 * cpu usage user:0.046 sys:3.559, 110.016 usec per MB, 65529 c-switches
46 * received 32768 MB (99.9939 % mmap'ed) in 7.43764 s, 36.9577 Gbit
47 * cpu usage user:0.035 sys:3.467, 106.873 usec per MB, 65530 c-switches
51 #include <sys/types.h>
54 #include <sys/socket.h>
56 #include <sys/resource.h>
64 #include <netinet/in.h>
65 #include <arpa/inet.h>
67 #include <linux/tcp.h>
71 #define MSG_ZEROCOPY 0x4000000
74 #define FILE_SZ (1ULL << 35)
75 static int cfg_family
= AF_INET6
;
76 static socklen_t cfg_alen
= sizeof(struct sockaddr_in6
);
77 static int cfg_port
= 8787;
79 static int rcvbuf
; /* Default: autotuning. Can be set with -r <integer> option */
80 static int sndbuf
; /* Default: autotuning. Can be set with -w <integer> option */
81 static int zflg
; /* zero copy option. (MSG_ZEROCOPY for sender, mmap() for receiver */
82 static int xflg
; /* hash received data (simple xor) (-h option) */
83 static int keepflag
; /* -k option: receiver shall keep all received file in memory (no munmap() calls) */
85 static size_t chunk_size
= 512*1024;
87 static size_t map_align
;
91 static inline void prefetch(const void *x
)
93 #if defined(__x86_64__)
94 asm volatile("prefetcht0 %P0" : : "m" (*(const char *)x
));
98 void hash_zone(void *zone
, unsigned int length
)
100 unsigned long temp
= htotal
;
102 while (length
>= 8*sizeof(long)) {
103 prefetch(zone
+ 384);
104 temp
^= *(unsigned long *)zone
;
105 temp
^= *(unsigned long *)(zone
+ sizeof(long));
106 temp
^= *(unsigned long *)(zone
+ 2*sizeof(long));
107 temp
^= *(unsigned long *)(zone
+ 3*sizeof(long));
108 temp
^= *(unsigned long *)(zone
+ 4*sizeof(long));
109 temp
^= *(unsigned long *)(zone
+ 5*sizeof(long));
110 temp
^= *(unsigned long *)(zone
+ 6*sizeof(long));
111 temp
^= *(unsigned long *)(zone
+ 7*sizeof(long));
112 zone
+= 8*sizeof(long);
113 length
-= 8*sizeof(long);
115 while (length
>= 1) {
116 temp
^= *(unsigned char *)zone
;
123 #define ALIGN_UP(x, align_to) (((x) + ((align_to)-1)) & ~((align_to)-1))
124 #define ALIGN_PTR_UP(p, ptr_align_to) ((typeof(p))ALIGN_UP((unsigned long)(p), ptr_align_to))
126 void *child_thread(void *arg
)
128 unsigned long total_mmap
= 0, total
= 0;
129 struct tcp_zerocopy_receive zc
;
130 unsigned long delta_usec
;
131 int flags
= MAP_SHARED
;
132 struct timeval t0
, t1
;
140 fd
= (int)(unsigned long)arg
;
142 gettimeofday(&t0
, NULL
);
144 fcntl(fd
, F_SETFL
, O_NDELAY
);
145 buffer
= malloc(chunk_size
);
151 raddr
= mmap(NULL
, chunk_size
+ map_align
, PROT_READ
, flags
, fd
, 0);
152 if (raddr
== (void *)-1) {
156 addr
= ALIGN_PTR_UP(raddr
, map_align
);
160 struct pollfd pfd
= { .fd
= fd
, .events
= POLLIN
, };
163 poll(&pfd
, 1, 10000);
165 socklen_t zc_len
= sizeof(zc
);
168 memset(&zc
, 0, sizeof(zc
));
169 zc
.address
= (__u64
)((unsigned long)addr
);
170 zc
.length
= chunk_size
;
172 res
= getsockopt(fd
, IPPROTO_TCP
, TCP_ZEROCOPY_RECEIVE
,
178 assert(zc
.length
<= chunk_size
);
179 total_mmap
+= zc
.length
;
181 hash_zone(addr
, zc
.length
);
184 if (zc
.recv_skip_hint
) {
185 assert(zc
.recv_skip_hint
<= chunk_size
);
186 lu
= read(fd
, buffer
, zc
.recv_skip_hint
);
189 hash_zone(buffer
, lu
);
196 while (sub
< chunk_size
) {
197 lu
= read(fd
, buffer
+ sub
, chunk_size
- sub
);
203 hash_zone(buffer
+ sub
, lu
);
209 gettimeofday(&t1
, NULL
);
210 delta_usec
= (t1
.tv_sec
- t0
.tv_sec
) * 1000000 + t1
.tv_usec
- t0
.tv_usec
;
214 throughput
= total
* 8.0 / (double)delta_usec
/ 1000.0;
215 getrusage(RUSAGE_THREAD
, &ru
);
216 if (total
> 1024*1024) {
217 unsigned long total_usec
;
218 unsigned long mb
= total
>> 20;
219 total_usec
= 1000000*ru
.ru_utime
.tv_sec
+ ru
.ru_utime
.tv_usec
+
220 1000000*ru
.ru_stime
.tv_sec
+ ru
.ru_stime
.tv_usec
;
221 printf("received %lg MB (%lg %% mmap'ed) in %lg s, %lg Gbit\n"
222 " cpu usage user:%lg sys:%lg, %lg usec per MB, %lu c-switches\n",
223 total
/ (1024.0 * 1024.0),
224 100.0*total_mmap
/total
,
225 (double)delta_usec
/ 1000000.0,
227 (double)ru
.ru_utime
.tv_sec
+ (double)ru
.ru_utime
.tv_usec
/ 1000000.0,
228 (double)ru
.ru_stime
.tv_sec
+ (double)ru
.ru_stime
.tv_usec
/ 1000000.0,
229 (double)total_usec
/mb
,
236 munmap(raddr
, chunk_size
+ map_align
);
240 static void apply_rcvsnd_buf(int fd
)
242 if (rcvbuf
&& setsockopt(fd
, SOL_SOCKET
,
243 SO_RCVBUF
, &rcvbuf
, sizeof(rcvbuf
)) == -1) {
244 perror("setsockopt SO_RCVBUF");
247 if (sndbuf
&& setsockopt(fd
, SOL_SOCKET
,
248 SO_SNDBUF
, &sndbuf
, sizeof(sndbuf
)) == -1) {
249 perror("setsockopt SO_SNDBUF");
254 static void setup_sockaddr(int domain
, const char *str_addr
,
255 struct sockaddr_storage
*sockaddr
)
257 struct sockaddr_in6
*addr6
= (void *) sockaddr
;
258 struct sockaddr_in
*addr4
= (void *) sockaddr
;
262 memset(addr4
, 0, sizeof(*addr4
));
263 addr4
->sin_family
= AF_INET
;
264 addr4
->sin_port
= htons(cfg_port
);
266 inet_pton(AF_INET
, str_addr
, &(addr4
->sin_addr
)) != 1)
267 error(1, 0, "ipv4 parse error: %s", str_addr
);
270 memset(addr6
, 0, sizeof(*addr6
));
271 addr6
->sin6_family
= AF_INET6
;
272 addr6
->sin6_port
= htons(cfg_port
);
274 inet_pton(AF_INET6
, str_addr
, &(addr6
->sin6_addr
)) != 1)
275 error(1, 0, "ipv6 parse error: %s", str_addr
);
278 error(1, 0, "illegal domain");
282 static void do_accept(int fdlisten
)
287 pthread_attr_init(&attr
);
288 pthread_attr_setdetachstate(&attr
, PTHREAD_CREATE_DETACHED
);
290 rcvlowat
= chunk_size
;
291 if (setsockopt(fdlisten
, SOL_SOCKET
, SO_RCVLOWAT
,
292 &rcvlowat
, sizeof(rcvlowat
)) == -1) {
293 perror("setsockopt SO_RCVLOWAT");
296 apply_rcvsnd_buf(fdlisten
);
299 struct sockaddr_in addr
;
300 socklen_t addrlen
= sizeof(addr
);
304 fd
= accept(fdlisten
, (struct sockaddr
*)&addr
, &addrlen
);
309 res
= pthread_create(&th
, &attr
, child_thread
,
310 (void *)(unsigned long)fd
);
313 perror("pthread_create");
319 /* Each thread should reserve a big enough vma to avoid
320 * spinlock collisions in ptl locks.
321 * This size is 2MB on x86_64, and is exported in /proc/meminfo.
323 static unsigned long default_huge_page_size(void)
325 FILE *f
= fopen("/proc/meminfo", "r");
326 unsigned long hps
= 0;
332 while (getline(&line
, &linelen
, f
) > 0) {
333 if (sscanf(line
, "Hugepagesize: %lu kB", &hps
) == 1) {
343 int main(int argc
, char *argv
[])
345 struct sockaddr_storage listenaddr
, addr
;
346 unsigned int max_pacing_rate
= 0;
354 while ((c
= getopt(argc
, argv
, "46p:svr:w:H:zxkP:M:C:a:")) != -1) {
357 cfg_family
= PF_INET
;
358 cfg_alen
= sizeof(struct sockaddr_in
);
361 cfg_family
= PF_INET6
;
362 cfg_alen
= sizeof(struct sockaddr_in6
);
365 cfg_port
= atoi(optarg
);
370 case 's': /* server : listen for incoming connections */
374 rcvbuf
= atoi(optarg
);
377 sndbuf
= atoi(optarg
);
392 max_pacing_rate
= atoi(optarg
) ;
395 chunk_size
= atol(optarg
);
398 map_align
= atol(optarg
);
405 map_align
= default_huge_page_size();
406 /* if really /proc/meminfo is not helping,
407 * we use the default x86_64 hugepagesize.
410 map_align
= 2*1024*1024;
413 int fdlisten
= socket(cfg_family
, SOCK_STREAM
, 0);
415 if (fdlisten
== -1) {
419 apply_rcvsnd_buf(fdlisten
);
420 setsockopt(fdlisten
, SOL_SOCKET
, SO_REUSEADDR
, &on
, sizeof(on
));
422 setup_sockaddr(cfg_family
, host
, &listenaddr
);
425 setsockopt(fdlisten
, IPPROTO_TCP
, TCP_MAXSEG
,
426 &mss
, sizeof(mss
)) == -1) {
427 perror("setsockopt TCP_MAXSEG");
430 if (bind(fdlisten
, (const struct sockaddr
*)&listenaddr
, cfg_alen
) == -1) {
434 if (listen(fdlisten
, 128) == -1) {
440 buffer
= mmap(NULL
, chunk_size
, PROT_READ
| PROT_WRITE
,
441 MAP_PRIVATE
| MAP_ANONYMOUS
, -1, 0);
442 if (buffer
== (char *)-1) {
447 fd
= socket(cfg_family
, SOCK_STREAM
, 0);
452 apply_rcvsnd_buf(fd
);
454 setup_sockaddr(cfg_family
, host
, &addr
);
457 setsockopt(fd
, IPPROTO_TCP
, TCP_MAXSEG
, &mss
, sizeof(mss
)) == -1) {
458 perror("setsockopt TCP_MAXSEG");
461 if (connect(fd
, (const struct sockaddr
*)&addr
, cfg_alen
) == -1) {
465 if (max_pacing_rate
&&
466 setsockopt(fd
, SOL_SOCKET
, SO_MAX_PACING_RATE
,
467 &max_pacing_rate
, sizeof(max_pacing_rate
)) == -1)
468 perror("setsockopt SO_MAX_PACING_RATE");
470 if (zflg
&& setsockopt(fd
, SOL_SOCKET
, SO_ZEROCOPY
,
471 &on
, sizeof(on
)) == -1) {
472 perror("setsockopt SO_ZEROCOPY, (-z option disabled)");
475 while (total
< FILE_SZ
) {
476 ssize_t wr
= FILE_SZ
- total
;
480 /* Note : we just want to fill the pipe with 0 bytes */
481 wr
= send(fd
, buffer
, wr
, zflg
? MSG_ZEROCOPY
: 0);
487 munmap(buffer
, chunk_size
);