From 748c7b9720593e20ce547e9d4cda52acdd1b4bc6 Mon Sep 17 00:00:00 2001 From: Dust Li Date: Tue, 30 Sep 2025 17:10:34 +0800 Subject: [PATCH] add udp-dont-frag option to support force setting DF bit MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When measuring maximum packets-per-second (PPS) on Linux using multiple sockperf processes to stress UDP throughput, the kernel’s per-packet atomic operation for generating IP ID becomes a bottleneck. Setting the Don’t Fragment (DF) bit in IP headers allows the kernel to skip this atomic IP ID assignment (as DF packets can use a constant or zero ID), reducing contention and improving scalability under high load. - Testing: ``` for((i=0;i<64;i++)); do taskset -c $i sockperf tp -i 200.16.10.2 -t 100 --udp-dont-frag done ``` - perf top without udp-dont-frag 56.56% [kernel] [k] ip_idents_reserve 5.98% [kernel] [k] __dev_xmit_skb 3.73% [kernel] [k] skb_set_owner_w 2.37% [kernel] [k] mlx5e_xmit 2.08% [kernel] [k] mlx5e_consume_skb 1.63% [kernel] [k] mlx5e_poll_tx_cq 1.43% [kernel] [k] mlx5e_handle_tx_dim 1.28% [kernel] [k] sock_wfree 1.12% [kernel] [k] __slab_free 0.64% [kernel] [k] build_detached_freelist 0.61% [kernel] [k] fib_table_lookup 0.59% sockperf [.] client_handler 0.57% [kernel] [k] pfifo_fast_dequeue 0.51% [kernel] [k] udp_sendmsg 0.50% libpthread-2.32.so [.] __libc_sendto 0.50% [kernel] [k] __ip_append_data - Without this option: 05:23:49 PM IFACE rxpck/s txpck/s rxkB/s txkB/s rxcmp/s txcmp/s rxmcst/s %ifutil 05:23:50 PM lo 22.00 22.00 8.61 8.61 0.00 0.00 0.00 0.00 05:23:50 PM eth0 2.00 7958649.00 0.20 466327.09 0.00 0.00 1.00 1.91 05:23:50 PM eth1 1.00 6588679.00 0.12 386055.41 0.00 0.00 1.00 1.58 05:23:50 PM bond0 3.00 14547313.00 0.32 852381.62 0.00 0.00 2.00 1.75 - With this option: Average: IFACE rxpck/s txpck/s rxkB/s txkB/s rxcmp/s txcmp/s rxmcst/s %ifutil Average: lo 0.00 0.00 0.00 0.00 0.00 0.00 0.00 0.00 Average: eth0 1.00 12339901.50 0.10 723041.10 0.00 0.00 0.50 2.96 Average: eth1 1.00 13850533.00 0.10 811554.67 0.00 0.00 0.50 3.32 Average: bond0 2.00 26190604.50 0.20 1534605.73 0.00 0.00 1.00 3.14 Signed-off-by: Dust Li --- src/defs.h | 4 +++- src/sockperf.cpp | 35 +++++++++++++++++++++++++++++++++-- 2 files changed, 36 insertions(+), 3 deletions(-) diff --git a/src/defs.h b/src/defs.h index 2d80a95..2531933 100644 --- a/src/defs.h +++ b/src/defs.h @@ -292,8 +292,9 @@ enum { OPT_LOAD_XLIO, // 47 OPT_TCP_NB_CONN_TIMEOUT_MS, // 48 #if defined(DEFINED_TLS) - OPT_TLS + OPT_TLS, // 49 #endif /* DEFINED_TLS */ + OPT_UDP_DONTFRAG // 50 }; static const char *const round_trip_str[] = { "latency", "rtt" }; @@ -805,6 +806,7 @@ struct user_params_t { #if defined(DEFINED_TLS) bool tls = false; #endif /* DEFINED_TLS */ + int udp_dontfrag = 0; user_params_t() { memset(&client_bind_info, 0, sizeof(client_bind_info)); diff --git a/src/sockperf.cpp b/src/sockperf.cpp index 6649b9b..6044b75 100644 --- a/src/sockperf.cpp +++ b/src/sockperf.cpp @@ -294,6 +294,11 @@ static const AOPT_DESC common_opt_desc[] = { aopt_set_literal(0), aopt_set_string("set-sock-accl"), "Set socket acceleration before run (available for some of Mellanox systems)" }, + { OPT_UDP_DONTFRAG, + AOPT_NOARG, + aopt_set_literal(0), + aopt_set_string("udp-dont-frag"), + "Force setting the DF bit for the outgoing UDP packets" }, #if defined(DEFINED_TLS) { OPT_TLS, AOPT_OPTARG, aopt_set_literal(0), aopt_set_string("tls"), "Use TLSv1.2 (default " TLS_CHIPER_DEFAULT ")." }, @@ -2233,6 +2238,16 @@ static int parse_common_opt(const AOPT_OBJECT *common_obj) { } } #endif /* DEFINED_TLS */ + + if (!rc && aopt_check(common_obj, OPT_UDP_DONTFRAG)) { + if (!aopt_check(common_obj, OPT_TCP) && !aopt_check(common_obj, OPT_LOAD_VMA) && + !aopt_check(common_obj, OPT_LOAD_XLIO)) { + s_user_params.udp_dontfrag = true; + } else { + log_msg("--udp-dont-frag conflicts with --tcp/--load-vma and --load-xlio options"); + rc = SOCKPERF_ERR_BAD_ARGUMENT; + } + } } // resolve address: -i, -p and --tcp options must be processed before @@ -2805,6 +2820,17 @@ int sock_set_tos(int fd) { return rc; } +int sock_set_udpdf(int fd) { + int rc = 0; + int v = IP_PMTUDISC_DO; + if (setsockopt(fd, IPPROTO_IP, IP_MTU_DISCOVER, &v, sizeof(v))) { + log_err("setsockopt(IP_MTU_DISCOVER), set failed. It could be that this option is not supported " + "in your system"); + rc = SOCKPERF_ERR_SOCKET; + } + return rc; +} + static int sock_join_multicast_v4(int fd, struct fds_data *p_data, const sockaddr_in *p_addr) { if (s_user_params.rx_mc_if_addr.family() != AF_UNSPEC && @@ -3085,6 +3111,10 @@ int prepare_socket(int fd, struct fds_data *p_data) rc = sock_set_tos(fd); } + if (!rc && (s_user_params.udp_dontfrag)) { + rc = sock_set_udpdf(fd); + } + #ifdef USING_EXTRA_API #ifdef ST_TEST if (!stTest) @@ -3902,7 +3932,8 @@ b_stream = %d \n\t\ daemonize = %d \n\t\ feedfile_name = %s \n\t\ tos = %d \n\t\ -packet pace limit = %d", +packet pace limit = %d \n\t\ +udp_dontfrag = %d", s_user_params.mode, s_user_params.measurement, s_user_params.withsock_accl, s_user_params.msg_size, s_user_params.msg_size_range, s_user_params.sec_test_duration, s_user_params.number_test_target, s_user_params.data_integrity, @@ -3921,7 +3952,7 @@ packet pace limit = %d", (strlen(s_user_params.receiver_affinity) ? s_user_params.receiver_affinity : ""), s_user_params.b_stream, s_user_params.daemonize, (strlen(s_user_params.feedfile_name) ? s_user_params.feedfile_name : ""), - s_user_params.tos, s_user_params.rate_limit); + s_user_params.tos, s_user_params.rate_limit, s_user_params.udp_dontfrag); // Display application version log_msg(MAGNETA "== version #%s == " ENDCOLOR, VERSION);