1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Copyright 2018 Google Inc.
4 * Author: Eric Dumazet (edumazet@google.com)
5 *
6 * Reference program demonstrating tcp mmap() usage,
7 * and SO_RCVLOWAT hints for receiver.
8 *
9 * Note : NIC with header split is needed to use mmap() on TCP :
10 * Each incoming frame must be a multiple of PAGE_SIZE bytes of TCP payload.
11 *
12 * How to use on loopback interface :
13 *
14 * ifconfig lo mtu 61512 # 15*4096 + 40 (ipv6 header) + 32 (TCP with TS option header)
15 * tcp_mmap -s -z &
16 * tcp_mmap -H ::1 -z
17 *
18 * Or leave default lo mtu, but use -M option to set TCP_MAXSEG option to (4096 + 12)
19 * (4096 : page size on x86, 12: TCP TS option length)
20 * tcp_mmap -s -z -M $((4096+12)) &
21 * tcp_mmap -H ::1 -z -M $((4096+12))
22 *
23 * Note: -z option on sender uses MSG_ZEROCOPY, which forces a copy when packets go through loopback interface.
24 * We might use sendfile() instead, but really this test program is about mmap(), for receivers ;)
25 *
26 * $ ./tcp_mmap -s & # Without mmap()
27 * $ for i in {1..4}; do ./tcp_mmap -H ::1 -z ; done
28 * received 32768 MB (0 % mmap'ed) in 14.1157 s, 19.4732 Gbit
29 * cpu usage user:0.057 sys:7.815, 240.234 usec per MB, 65531 c-switches
30 * received 32768 MB (0 % mmap'ed) in 14.6833 s, 18.7204 Gbit
31 * cpu usage user:0.043 sys:8.103, 248.596 usec per MB, 65524 c-switches
32 * received 32768 MB (0 % mmap'ed) in 11.143 s, 24.6682 Gbit
33 * cpu usage user:0.044 sys:6.576, 202.026 usec per MB, 65519 c-switches
34 * received 32768 MB (0 % mmap'ed) in 14.9056 s, 18.4413 Gbit
35 * cpu usage user:0.036 sys:8.193, 251.129 usec per MB, 65530 c-switches
36 * $ kill %1 # kill tcp_mmap server
37 *
38 * $ ./tcp_mmap -s -z & # With mmap()
39 * $ for i in {1..4}; do ./tcp_mmap -H ::1 -z ; done
40 * received 32768 MB (99.9939 % mmap'ed) in 6.73792 s, 40.7956 Gbit
41 * cpu usage user:0.045 sys:2.827, 87.6465 usec per MB, 65532 c-switches
42 * received 32768 MB (99.9939 % mmap'ed) in 7.26732 s, 37.8238 Gbit
43 * cpu usage user:0.037 sys:3.087, 95.3369 usec per MB, 65532 c-switches
44 * received 32768 MB (99.9939 % mmap'ed) in 7.61661 s, 36.0893 Gbit
45 * cpu usage user:0.046 sys:3.559, 110.016 usec per MB, 65529 c-switches
46 * received 32768 MB (99.9939 % mmap'ed) in 7.43764 s, 36.9577 Gbit
47 * cpu usage user:0.035 sys:3.467, 106.873 usec per MB, 65530 c-switches
48 */
49 #define _GNU_SOURCE
50 #include <pthread.h>
51 #include <sys/types.h>
52 #include <fcntl.h>
53 #include <error.h>
54 #include <sys/socket.h>
55 #include <sys/mman.h>
56 #include <sys/resource.h>
57 #include <unistd.h>
58 #include <string.h>
59 #include <stdlib.h>
60 #include <stdio.h>
61 #include <errno.h>
62 #include <time.h>
63 #include <sys/time.h>
64 #include <netinet/in.h>
65 #include <arpa/inet.h>
66 #include <poll.h>
67 #include <linux/tcp.h>
68 #include <assert.h>
69
70 #ifndef MSG_ZEROCOPY
71 #define MSG_ZEROCOPY 0x4000000
72 #endif
73
74 #define FILE_SZ (1ULL << 35)
75 static int cfg_family = AF_INET6;
76 static socklen_t cfg_alen = sizeof(struct sockaddr_in6);
77 static int cfg_port = 8787;
78
79 static int rcvbuf; /* Default: autotuning. Can be set with -r <integer> option */
80 static int sndbuf; /* Default: autotuning. Can be set with -w <integer> option */
81 static int zflg; /* zero copy option. (MSG_ZEROCOPY for sender, mmap() for receiver */
82 static int xflg; /* hash received data (simple xor) (-h option) */
83 static int keepflag; /* -k option: receiver shall keep all received file in memory (no munmap() calls) */
84
85 static size_t chunk_size = 512*1024;
86
87 static size_t map_align;
88
89 unsigned long htotal;
90
prefetch(const void * x)91 static inline void prefetch(const void *x)
92 {
93 #if defined(__x86_64__)
94 asm volatile("prefetcht0 %P0" : : "m" (*(const char *)x));
95 #endif
96 }
97
hash_zone(void * zone,unsigned int length)98 void hash_zone(void *zone, unsigned int length)
99 {
100 unsigned long temp = htotal;
101
102 while (length >= 8*sizeof(long)) {
103 prefetch(zone + 384);
104 temp ^= *(unsigned long *)zone;
105 temp ^= *(unsigned long *)(zone + sizeof(long));
106 temp ^= *(unsigned long *)(zone + 2*sizeof(long));
107 temp ^= *(unsigned long *)(zone + 3*sizeof(long));
108 temp ^= *(unsigned long *)(zone + 4*sizeof(long));
109 temp ^= *(unsigned long *)(zone + 5*sizeof(long));
110 temp ^= *(unsigned long *)(zone + 6*sizeof(long));
111 temp ^= *(unsigned long *)(zone + 7*sizeof(long));
112 zone += 8*sizeof(long);
113 length -= 8*sizeof(long);
114 }
115 while (length >= 1) {
116 temp ^= *(unsigned char *)zone;
117 zone += 1;
118 length--;
119 }
120 htotal = temp;
121 }
122
123 #define ALIGN_UP(x, align_to) (((x) + ((align_to)-1)) & ~((align_to)-1))
124 #define ALIGN_PTR_UP(p, ptr_align_to) ((typeof(p))ALIGN_UP((unsigned long)(p), ptr_align_to))
125
126
mmap_large_buffer(size_t need,size_t * allocated)127 static void *mmap_large_buffer(size_t need, size_t *allocated)
128 {
129 void *buffer;
130 size_t sz;
131
132 /* Attempt to use huge pages if possible. */
133 sz = ALIGN_UP(need, map_align);
134 buffer = mmap(NULL, sz, PROT_READ | PROT_WRITE,
135 MAP_PRIVATE | MAP_ANONYMOUS | MAP_HUGETLB, -1, 0);
136
137 if (buffer == (void *)-1) {
138 sz = need;
139 buffer = mmap(NULL, sz, PROT_READ | PROT_WRITE,
140 MAP_PRIVATE | MAP_ANONYMOUS | MAP_POPULATE,
141 -1, 0);
142 if (buffer != (void *)-1)
143 fprintf(stderr, "MAP_HUGETLB attempt failed, look at /sys/kernel/mm/hugepages for optimal performance\n");
144 }
145 *allocated = sz;
146 return buffer;
147 }
148
child_thread(void * arg)149 void *child_thread(void *arg)
150 {
151 unsigned long total_mmap = 0, total = 0;
152 struct tcp_zerocopy_receive zc;
153 unsigned long delta_usec;
154 int flags = MAP_SHARED;
155 struct timeval t0, t1;
156 char *buffer = NULL;
157 void *raddr = NULL;
158 void *addr = NULL;
159 double throughput;
160 struct rusage ru;
161 size_t buffer_sz;
162 int lu, fd;
163
164 fd = (int)(unsigned long)arg;
165
166 gettimeofday(&t0, NULL);
167
168 fcntl(fd, F_SETFL, O_NDELAY);
169 buffer = mmap_large_buffer(chunk_size, &buffer_sz);
170 if (buffer == (void *)-1) {
171 perror("mmap");
172 goto error;
173 }
174 if (zflg) {
175 raddr = mmap(NULL, chunk_size + map_align, PROT_READ, flags, fd, 0);
176 if (raddr == (void *)-1) {
177 perror("mmap");
178 zflg = 0;
179 } else {
180 addr = ALIGN_PTR_UP(raddr, map_align);
181 }
182 }
183 while (1) {
184 struct pollfd pfd = { .fd = fd, .events = POLLIN, };
185 int sub;
186
187 poll(&pfd, 1, 10000);
188 if (zflg) {
189 socklen_t zc_len = sizeof(zc);
190 int res;
191
192 memset(&zc, 0, sizeof(zc));
193 zc.address = (__u64)((unsigned long)addr);
194 zc.length = chunk_size;
195
196 res = getsockopt(fd, IPPROTO_TCP, TCP_ZEROCOPY_RECEIVE,
197 &zc, &zc_len);
198 if (res == -1)
199 break;
200
201 if (zc.length) {
202 assert(zc.length <= chunk_size);
203 total_mmap += zc.length;
204 if (xflg)
205 hash_zone(addr, zc.length);
206 /* It is more efficient to unmap the pages right now,
207 * instead of doing this in next TCP_ZEROCOPY_RECEIVE.
208 */
209 madvise(addr, zc.length, MADV_DONTNEED);
210 total += zc.length;
211 }
212 if (zc.recv_skip_hint) {
213 assert(zc.recv_skip_hint <= chunk_size);
214 lu = read(fd, buffer, zc.recv_skip_hint);
215 if (lu > 0) {
216 if (xflg)
217 hash_zone(buffer, lu);
218 total += lu;
219 }
220 }
221 continue;
222 }
223 sub = 0;
224 while (sub < chunk_size) {
225 lu = read(fd, buffer + sub, chunk_size - sub);
226 if (lu == 0)
227 goto end;
228 if (lu < 0)
229 break;
230 if (xflg)
231 hash_zone(buffer + sub, lu);
232 total += lu;
233 sub += lu;
234 }
235 }
236 end:
237 gettimeofday(&t1, NULL);
238 delta_usec = (t1.tv_sec - t0.tv_sec) * 1000000 + t1.tv_usec - t0.tv_usec;
239
240 throughput = 0;
241 if (delta_usec)
242 throughput = total * 8.0 / (double)delta_usec / 1000.0;
243 getrusage(RUSAGE_THREAD, &ru);
244 if (total > 1024*1024) {
245 unsigned long total_usec;
246 unsigned long mb = total >> 20;
247 total_usec = 1000000*ru.ru_utime.tv_sec + ru.ru_utime.tv_usec +
248 1000000*ru.ru_stime.tv_sec + ru.ru_stime.tv_usec;
249 printf("received %lg MB (%lg %% mmap'ed) in %lg s, %lg Gbit\n"
250 " cpu usage user:%lg sys:%lg, %lg usec per MB, %lu c-switches\n",
251 total / (1024.0 * 1024.0),
252 100.0*total_mmap/total,
253 (double)delta_usec / 1000000.0,
254 throughput,
255 (double)ru.ru_utime.tv_sec + (double)ru.ru_utime.tv_usec / 1000000.0,
256 (double)ru.ru_stime.tv_sec + (double)ru.ru_stime.tv_usec / 1000000.0,
257 (double)total_usec/mb,
258 ru.ru_nvcsw);
259 }
260 error:
261 munmap(buffer, buffer_sz);
262 close(fd);
263 if (zflg)
264 munmap(raddr, chunk_size + map_align);
265 pthread_exit(0);
266 }
267
apply_rcvsnd_buf(int fd)268 static void apply_rcvsnd_buf(int fd)
269 {
270 if (rcvbuf && setsockopt(fd, SOL_SOCKET,
271 SO_RCVBUF, &rcvbuf, sizeof(rcvbuf)) == -1) {
272 perror("setsockopt SO_RCVBUF");
273 }
274
275 if (sndbuf && setsockopt(fd, SOL_SOCKET,
276 SO_SNDBUF, &sndbuf, sizeof(sndbuf)) == -1) {
277 perror("setsockopt SO_SNDBUF");
278 }
279 }
280
281
setup_sockaddr(int domain,const char * str_addr,struct sockaddr_storage * sockaddr)282 static void setup_sockaddr(int domain, const char *str_addr,
283 struct sockaddr_storage *sockaddr)
284 {
285 struct sockaddr_in6 *addr6 = (void *) sockaddr;
286 struct sockaddr_in *addr4 = (void *) sockaddr;
287
288 switch (domain) {
289 case PF_INET:
290 memset(addr4, 0, sizeof(*addr4));
291 addr4->sin_family = AF_INET;
292 addr4->sin_port = htons(cfg_port);
293 if (str_addr &&
294 inet_pton(AF_INET, str_addr, &(addr4->sin_addr)) != 1)
295 error(1, 0, "ipv4 parse error: %s", str_addr);
296 break;
297 case PF_INET6:
298 memset(addr6, 0, sizeof(*addr6));
299 addr6->sin6_family = AF_INET6;
300 addr6->sin6_port = htons(cfg_port);
301 if (str_addr &&
302 inet_pton(AF_INET6, str_addr, &(addr6->sin6_addr)) != 1)
303 error(1, 0, "ipv6 parse error: %s", str_addr);
304 break;
305 default:
306 error(1, 0, "illegal domain");
307 }
308 }
309
do_accept(int fdlisten)310 static void do_accept(int fdlisten)
311 {
312 pthread_attr_t attr;
313 int rcvlowat;
314
315 pthread_attr_init(&attr);
316 pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED);
317
318 rcvlowat = chunk_size;
319 if (setsockopt(fdlisten, SOL_SOCKET, SO_RCVLOWAT,
320 &rcvlowat, sizeof(rcvlowat)) == -1) {
321 perror("setsockopt SO_RCVLOWAT");
322 }
323
324 apply_rcvsnd_buf(fdlisten);
325
326 while (1) {
327 struct sockaddr_in addr;
328 socklen_t addrlen = sizeof(addr);
329 pthread_t th;
330 int fd, res;
331
332 fd = accept(fdlisten, (struct sockaddr *)&addr, &addrlen);
333 if (fd == -1) {
334 perror("accept");
335 continue;
336 }
337 res = pthread_create(&th, &attr, child_thread,
338 (void *)(unsigned long)fd);
339 if (res) {
340 errno = res;
341 perror("pthread_create");
342 close(fd);
343 }
344 }
345 }
346
347 /* Each thread should reserve a big enough vma to avoid
348 * spinlock collisions in ptl locks.
349 * This size is 2MB on x86_64, and is exported in /proc/meminfo.
350 */
default_huge_page_size(void)351 static unsigned long default_huge_page_size(void)
352 {
353 FILE *f = fopen("/proc/meminfo", "r");
354 unsigned long hps = 0;
355 size_t linelen = 0;
356 char *line = NULL;
357
358 if (!f)
359 return 0;
360 while (getline(&line, &linelen, f) > 0) {
361 if (sscanf(line, "Hugepagesize: %lu kB", &hps) == 1) {
362 hps <<= 10;
363 break;
364 }
365 }
366 free(line);
367 fclose(f);
368 return hps;
369 }
370
main(int argc,char * argv[])371 int main(int argc, char *argv[])
372 {
373 struct sockaddr_storage listenaddr, addr;
374 unsigned int max_pacing_rate = 0;
375 uint64_t total = 0;
376 char *host = NULL;
377 int fd, c, on = 1;
378 size_t buffer_sz;
379 char *buffer;
380 int sflg = 0;
381 int mss = 0;
382
383 while ((c = getopt(argc, argv, "46p:svr:w:H:zxkP:M:C:a:")) != -1) {
384 switch (c) {
385 case '4':
386 cfg_family = PF_INET;
387 cfg_alen = sizeof(struct sockaddr_in);
388 break;
389 case '6':
390 cfg_family = PF_INET6;
391 cfg_alen = sizeof(struct sockaddr_in6);
392 break;
393 case 'p':
394 cfg_port = atoi(optarg);
395 break;
396 case 'H':
397 host = optarg;
398 break;
399 case 's': /* server : listen for incoming connections */
400 sflg++;
401 break;
402 case 'r':
403 rcvbuf = atoi(optarg);
404 break;
405 case 'w':
406 sndbuf = atoi(optarg);
407 break;
408 case 'z':
409 zflg = 1;
410 break;
411 case 'M':
412 mss = atoi(optarg);
413 break;
414 case 'x':
415 xflg = 1;
416 break;
417 case 'k':
418 keepflag = 1;
419 break;
420 case 'P':
421 max_pacing_rate = atoi(optarg) ;
422 break;
423 case 'C':
424 chunk_size = atol(optarg);
425 break;
426 case 'a':
427 map_align = atol(optarg);
428 break;
429 default:
430 exit(1);
431 }
432 }
433 if (!map_align) {
434 map_align = default_huge_page_size();
435 /* if really /proc/meminfo is not helping,
436 * we use the default x86_64 hugepagesize.
437 */
438 if (!map_align)
439 map_align = 2*1024*1024;
440 }
441 if (sflg) {
442 int fdlisten = socket(cfg_family, SOCK_STREAM, 0);
443
444 if (fdlisten == -1) {
445 perror("socket");
446 exit(1);
447 }
448 apply_rcvsnd_buf(fdlisten);
449 setsockopt(fdlisten, SOL_SOCKET, SO_REUSEADDR, &on, sizeof(on));
450
451 setup_sockaddr(cfg_family, host, &listenaddr);
452
453 if (mss &&
454 setsockopt(fdlisten, IPPROTO_TCP, TCP_MAXSEG,
455 &mss, sizeof(mss)) == -1) {
456 perror("setsockopt TCP_MAXSEG");
457 exit(1);
458 }
459 if (bind(fdlisten, (const struct sockaddr *)&listenaddr, cfg_alen) == -1) {
460 perror("bind");
461 exit(1);
462 }
463 if (listen(fdlisten, 128) == -1) {
464 perror("listen");
465 exit(1);
466 }
467 do_accept(fdlisten);
468 }
469
470 buffer = mmap_large_buffer(chunk_size, &buffer_sz);
471 if (buffer == (char *)-1) {
472 perror("mmap");
473 exit(1);
474 }
475
476 fd = socket(cfg_family, SOCK_STREAM, 0);
477 if (fd == -1) {
478 perror("socket");
479 exit(1);
480 }
481 apply_rcvsnd_buf(fd);
482
483 setup_sockaddr(cfg_family, host, &addr);
484
485 if (mss &&
486 setsockopt(fd, IPPROTO_TCP, TCP_MAXSEG, &mss, sizeof(mss)) == -1) {
487 perror("setsockopt TCP_MAXSEG");
488 exit(1);
489 }
490 if (connect(fd, (const struct sockaddr *)&addr, cfg_alen) == -1) {
491 perror("connect");
492 exit(1);
493 }
494 if (max_pacing_rate &&
495 setsockopt(fd, SOL_SOCKET, SO_MAX_PACING_RATE,
496 &max_pacing_rate, sizeof(max_pacing_rate)) == -1)
497 perror("setsockopt SO_MAX_PACING_RATE");
498
499 if (zflg && setsockopt(fd, SOL_SOCKET, SO_ZEROCOPY,
500 &on, sizeof(on)) == -1) {
501 perror("setsockopt SO_ZEROCOPY, (-z option disabled)");
502 zflg = 0;
503 }
504 while (total < FILE_SZ) {
505 int64_t wr = FILE_SZ - total;
506
507 if (wr > chunk_size)
508 wr = chunk_size;
509 /* Note : we just want to fill the pipe with 0 bytes */
510 wr = send(fd, buffer, (size_t)wr, zflg ? MSG_ZEROCOPY : 0);
511 if (wr <= 0)
512 break;
513 total += wr;
514 }
515 close(fd);
516 munmap(buffer, buffer_sz);
517 return 0;
518 }
519