From 5d6c65c40686e411411cdfe81a4a50cefb82b260 Mon Sep 17 00:00:00 2001 From: "Arrobo, Gabriel" Date: Wed, 7 Oct 2026 10:38:41 -0700 Subject: [PATCH 1/4] Add dl_rate/ul_rate config and uplink seq-update helpers Signed-off-by: Arrobo, Gabriel --- conf/parser.py | 20 +++++++++++++++++++- conf/sim.py | 30 +++++++++++++++++++++++++++++- 2 files changed, 48 insertions(+), 2 deletions(-) diff --git a/conf/parser.py b/conf/parser.py index cf9e76153..d2c9aac97 100755 --- a/conf/parser.py +++ b/conf/parser.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # SPDX-License-Identifier: Apache-2.0 -# Copyright 2019 Intel Corporation +# Copyright (c) 2019-Present Intel Corporation # for get_env from conf.utils import * @@ -22,6 +22,8 @@ class Parser: def __init__(self, fname): self.name = get_env("CONF_FILE", fname) self.conf = get_json_conf(self.name, False) + self.dl_rate = 0 + self.ul_rate = 0 self.max_ip_defrag_flows = None self.ip_frag_with_eth_mtu = None self.hwcksum = False @@ -59,6 +61,22 @@ def parse(self, ifaces): if self.conf is None: print("Error loading configuration file.") + # Number of packets per second to be created in DL + try: + self.dl_rate = int(self.conf["dl_rate"]) + except ValueError: + print("Invalid value for dl_rate. Disabling traffic generation in DL.") + except KeyError: + print("dl_rate value not set. Ignoring parameter.") + + # Number of packets per second to be created in UL + try: + self.ul_rate = int(self.conf["ul_rate"]) + except ValueError: + print("Invalid value for ul_rate. Disabling traffic generation in UL.") + except KeyError: + print("ul_rate value not set. Ignoring parameter.") + # Maximum number of flows to manage ip4 frags for re-assembly try: self.max_ip_defrag_flows = int(self.conf["max_ip_defrag_flows"]) diff --git a/conf/sim.py b/conf/sim.py index 4ef3775b0..d21b396bb 100755 --- a/conf/sim.py +++ b/conf/sim.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # SPDX-License-Identifier: Apache-2.0 -# Copyright 2019 Intel Corporation +# Copyright 2019-Present Intel Corporation from scapy.all import * from scapy.contrib.gtp import * @@ -42,6 +42,20 @@ def gen_inet_sequpdate_args(max_session, start_ue_ip): return kwargs +def gen_inet_sequpdate_ul_args(max_session, start_ue_ip): + kwargs = { + "fields": [ + { + "offset": 26, + "size": 4, + "min": ip2long(start_ue_ip), + "max": ip2long(start_ue_ip) + max_session - 1, + } + ] + } + return kwargs + + def gen_gtpu_packet( size, src_mac, @@ -95,3 +109,17 @@ def gen_gtpu_sequpdate_args(max_session, start_ue_ip, ue_ip_offset, start_teid): ] } return kwargs + + +def gen_gtpu_sequpdate_ul_args(max_session, start_teid): + kwargs = { + "fields": [ + { + "offset": 46, + "size": 4, + "min": start_teid, + "max": start_teid + max_session - 1, + } + ] + } + return kwargs From ca637c28cbb39938ee02704eec3adac306b86afc Mon Sep 17 00:00:00 2001 From: "Arrobo, Gabriel" Date: Wed, 7 Oct 2026 10:40:06 -0700 Subject: [PATCH 2/4] Config-driven bidirectional external-datapath generator Signed-off-by: Arrobo, Gabriel --- conf/pktgen.bess | 251 ++++++++++++++++++++++++++++++---------- conf/pktgen.jsonc | 68 +++++++++++ conf/pktgen_sim.bess | 78 +++++++++++++ scripts/pktgen_setup.sh | 183 +++++++++++++++++++++++++++++ 4 files changed, 522 insertions(+), 58 deletions(-) create mode 100644 conf/pktgen.jsonc create mode 100644 conf/pktgen_sim.bess create mode 100755 scripts/pktgen_setup.sh diff --git a/conf/pktgen.bess b/conf/pktgen.bess index 2799cfd54..16db18366 100644 --- a/conf/pktgen.bess +++ b/conf/pktgen.bess @@ -1,78 +1,213 @@ +# vim: syntax=py +# -*- mode: python -*- # SPDX-License-Identifier: Apache-2.0 -# Copyright 2021 Intel Corporation - -""" -docker run --name pktgen -td --restart unless-stopped \ - --cpuset-cpus=2-5 --ulimit memlock=-1 --cap-add IPC_LOCK \ - -v /dev/hugepages:/dev/hugepages -v "$PWD/conf":/opt/bess/bessctl/conf \ - --device=/dev/vfio/vfio --device=/dev/vfio/176 \ - upf-bess:"$( parser.workers: + nonworkers = cores[parser.workers:] +else: + nonworkers = cores + +set_process_affinity_all(nonworkers) +for wid in range(parser.workers): bess.add_worker(wid=wid, core=int(workers[wid % len(workers)])) -num_q = len(workers) -kwargs = {'size_inc_q': 512, - 'size_out_q': 512, - 'num_inc_q': num_q, - 'num_out_q': num_q} -p = PMDPort(port_id=0, **kwargs) -n3seq_kwargs = sim.gen_gtpu_sequpdate_args(flows, "16.0.0.1", n3seq_offset, 0x30000000) -n9seq_kwargs = sim.gen_gtpu_sequpdate_args(flows, "16.0.0.1", n9seq_offset, 0x90000000) -n6seq_kwargs = sim.gen_inet_sequpdate_args(flows, "16.0.0.1") +# ==================================================== +# Port Setup +# ==================================================== + +ports = {} -# 25 Gbps each to saturate 100 Gbps -bess.add_tc('39_limit', wid=0, policy='rate_limit', resource='bit', limit={'bit': 25000000000}) -bess.add_tc('36_limit', wid=0, policy='rate_limit', resource='bit', limit={'bit': 25000000000}) -bess.add_tc('6_limit', wid=1, policy='rate_limit', resource='bit', limit={'bit': 25000000000}) -bess.add_tc('9_limit', wid=1, policy='rate_limit', resource='bit', limit={'bit': 25000000000}) +for idx, iface in enumerate(interfaces): + # check if source natting for a given port is required + try: + ext_addrs = parser.interfaces[iface]["ip_masquerade"] + except KeyError: + ext_addrs = None + except TypeError: + ext_addrs = None -src39::Source(pkt_size=pkt_size) -> Rewrite(templates=n39_pkts) -> n39update::SequentialUpdate(**n3seq_kwargs) -> L4Checksum() -> IPChecksum() -> QueueOut(port=p.name, qid=0) -src36::Source(pkt_size=pkt_size) -> Rewrite(templates=n36_pkts) -> n36update::SequentialUpdate(**n3seq_kwargs) -> L4Checksum() -> IPChecksum() -> QueueOut(port=p.name, qid=1) + p = port.Port(parser.interfaces[iface]["ifname"], parser.hwcksum, ext_addrs) + if p.name in ports: + continue -src9::Source(pkt_size=pkt_size) -> Rewrite(templates=n9_pkts) -> n9update::SequentialUpdate(**n9seq_kwargs) -> L4Checksum() -> IPChecksum() -> QueueOut(port=p.name, qid=2) -src6::Source(pkt_size=pkt_size) -> Rewrite(templates=n6_pkts) -> n6update::SequentialUpdate(**n6seq_kwargs) -> L4Checksum() -> IPChecksum() -> QueueOut(port=p.name, qid=3) + if parser.ddp: + p.configure_flow_profiles(iface) -src39.attach_task(parent='39_limit') -src36.attach_task(parent='36_limit') + # initialize port with the configured driver + p.workers = [i for i in range(len(workers))] + p.init_port(idx, parser.mode) -src9.attach_task(parent='9_limit') -src6.attach_task(parent='6_limit') + # setup port module with auxiliary modules + p.setup_port(parser.ip_frag_with_eth_mtu, parser.max_ip_defrag_flows, parser.measure_upf) + + # Finally add entry to ports list + ports[p.name] = p + + +# ==================================================== +# Downlink packet generation +# ==================================================== + +n6_pkts = [sim.gen_inet_packet(parser.sim_pkt_size, core_mac, core_dst_mac, + parser.sim_n6_app_ip, parser.sim_start_ue_ip)] +n6_seq = sim.gen_inet_sequpdate_args(parser.sim_total_flows, parser.sim_start_ue_ip) + +# dl_rate == 0 disables downlink generation (see pktgen.jsonc) +if parser.dl_rate > 0: + n6TxSrc::Source() -> n6TxGen::Rewrite(templates=n6_pkts) \ + -> n6TxSeq::SequentialUpdate(**n6_seq) -> n6TxL4Cksum::L4Checksum() \ + -> n6TxIPCksum::IPChecksum() -> ports[parser.core_ifname].rtr + n6TxSrc.set_burst(burst=1) + + bess.add_tc('schedule_limit_dl', + policy='rate_limit', + resource='count', + limit={'count': parser.dl_rate}, + wid=0) + n6TxSrc.attach_task(parent='schedule_limit_dl') +else: + print("dl_rate is 0: downlink traffic generation disabled") + + +# ==================================================== +# Downlink packet reception +# ==================================================== + +n3RxFastBPF = ports[parser.access_ifname].bpf +GTPUGate = 0 + +n3RxFastBPF:GTPUGate \ + -> n3RxIPCksum::IPChecksum(verify=True, hw=parser.hwcksum) \ + -> n3RxL4Cksum::L4Checksum(verify=True, hw=parser.hwcksum) \ + -> n3RxSink::Sink() + +# Add Access filter rules, i.e.: +# setting filter to detect gtpu traffic +# and dst host 11.1.1.1 # check S/PGWU IP +# and udp dst port 2152 # check GTPU port +check_ip = "ip" +check_spgwu_ip = " and dst host " + \ + " or ".join(str(x) for x in access_ip) +check_gtpu_port = " and udp dst port 2152" + +# PDU rule +uplink_filter = {"priority": -GTPUGate, "filter": check_ip + + check_spgwu_ip + check_gtpu_port, "gate": GTPUGate} +n3RxFastBPF.add(filters=[uplink_filter]) + + +# ==================================================== +# Uplink packet generation +# ==================================================== + +n3_pkts = [sim.gen_inet_packet(parser.sim_pkt_size, access_mac, access_dst_mac, + parser.sim_start_ue_ip, parser.sim_n6_app_ip)] +n3_seq = sim.gen_inet_sequpdate_ul_args(parser.sim_total_flows, parser.sim_start_ue_ip) +gtpu_seq = sim.gen_gtpu_sequpdate_ul_args(parser.sim_total_flows, parser.sim_start_n3_teid) + +# ul_rate == 0 disables uplink generation (see pktgen.jsonc) +if parser.ul_rate > 0: + n3TxSrc::Source() -> n3TxGen::Rewrite(templates=n3_pkts) \ + -> n3TxSeq::SequentialUpdate(**n3_seq) -> n3TxL4Cksum::L4Checksum() \ + -> n3TxIPCksum::IPChecksum() -> gtpuEncap::GtpuEncap(add_psc=parser.gtppsc):1 \ + -> Update(fields=[{'offset': 26, 'size': 4, 'value': ip2long(parser.sim_start_enb_ip)}, \ + {'offset': 30, 'size': 4, 'value': ip2long(access_dst_ip)}, \ + {'offset': 34, 'size': 2, 'value': 2152}, \ + {'offset': 36, 'size': 2, 'value': 2152}]) \ + -> gtpuTxSeq::SequentialUpdate(**gtpu_seq) -> gtpuL4Cksum::L4Checksum() \ + -> gtpuIPCksum::IPChecksum() -> ports[parser.access_ifname].rtr + n3TxSrc.set_burst(burst=1) + + bess.add_tc('schedule_limit_ul', + policy='rate_limit', + resource='count', + limit={'count': parser.ul_rate}, + wid=0) + n3TxSrc.attach_task(parent='schedule_limit_ul') +else: + print("ul_rate is 0: uplink traffic generation disabled") + + +# ==================================================== +# Uplink packet reception +# ==================================================== + +n6RxFastBPF = ports[parser.core_ifname].bpf +UEGate = 0 + +n6RxFastBPF:UEGate \ + -> n6RxIPCksum::IPChecksum(verify=True, hw=parser.hwcksum) \ + -> n6RxL4Cksum::L4Checksum(verify=True, hw=parser.hwcksum) \ + -> n6RxSink::Sink() + +# Add Core filter rules, i.e.: +# setting filter to detect gtpu traffic +# and dst host x.x.x.x # check S/PGWU IP +# and udp dst port 2152 # check GTPU port +check_ip = "ip" +check_spgwu_ip = " and dst host " + \ + " or ".join(str(x) for x in core_ip) +check_gtpu_port = " and udp dst port 2152" +GTPUGate = 0 #ports[parser.core_ifname].bpf_gate() +downlink_filter = {"priority": -GTPUGate, "filter": check_ip + + check_spgwu_ip + check_gtpu_port, "gate": GTPUGate} +n6RxFastBPF.add(filters=[downlink_filter]) + +ports[parser.access_ifname].rtr.delete(prefix='0.0.0.0', prefix_len=0) +ports[parser.access_ifname].rtr.add(prefix='0.0.0.0', prefix_len=0, gate=0) + + +# ==================================================== +# Route Control +# ==================================================== +# Finally send SIGHUP to route_control daemon on reload +# TODO: behavior is unspecified if route_control.py pid is not found +route_control_pid = getpythonpid('route_control.py') +if route_control_pid: + os.kill(route_control_pid, signal.SIGHUP) diff --git a/conf/pktgen.jsonc b/conf/pktgen.jsonc new file mode 100644 index 000000000..d71304f15 --- /dev/null +++ b/conf/pktgen.jsonc @@ -0,0 +1,68 @@ +// SPDX-FileCopyrightText: 2020-present Open Networking Foundation +// SPDX-License-Identifier: Apache-2.0 +{ + // UPF support various configuration modes: + // "af_packet"` to enable AF_PACKET mode, + // "af_xdp" to enable AF_XDP mode, + // "dpdk" to enable DPDK mode, + // "sim" to generate synthetic traffic from BESS's Source module, + // "" when running with UP4 + "mode": "dpdk", + + // [Optional] Set the log level to one of \"panic\", \"fatal\", \"error\", \"warning\", \"info\", \"debug\", \"trace\" + "log_level": "info", + + // Use the sim block to enable simulation using either Source module or via il_trafficgen + "sim": { + // At this point we can simulate either N3/N6 or N3/N9 traffic, so choose n6 or n9 below + "core": "n6", + "max_sessions": 50000, + "start_ue_ip": "16.0.0.1", + "start_enb_ip": "11.1.1.129", + "start_aupf_ip": "13.1.1.199", + "n6_app_ip": "6.6.6.6", + "n9_app_ip": "9.9.9.9", + "start_n3_teid": "0x30000000", + "start_n9_teid": "0x90000000", + "pkt_size": 128, + "total_flows": 5000 + }, + + // DL Data rate for traffic generation (in pps). '0' means dl traffic is disabled + "dl_rate": 0, + + // UL Data rate for traffic generation (in pps). '0' means ul traffic is disabled + "ul_rate": 0, + + // Max IP frag table entries (for IPv4 reassembly). Uncomment line below to enable + // "max_ip_defrag_flows": "1000", + + // Uncomment line below to enable + // "ip_frag_with_eth_mtu": "1518", + + // Enable hardware offload of checksum. Might disable vector PMD + "hwcksum": false, + + // Enable PDU Session Container extension + "gtppsc": false, + + // Enable Intel Dynamic Device Personalization (DDP) + "ddp": false, + + // N3 interface + "access": { + // "cndp_jsonc_file": "conf/cndp_upf_1worker.jsonc", + "ifname": "ens785f0" + }, + + // N6 or N9 interface (depending on the UPF's deployment [PSA-UPF or I-UPF]) + "core": { + // "cndp_jsonc_file": "conf/cndp_upf_1worker.jsonc", + // Uncomment line below to enable UE IP natting. It could be a single IP or multiple IPs + // "ip_masquerade": "18.0.0.1 or 18.0.0.2 or 18.0.0.3", + "ifname": "ens785f1" + }, + + // Number of worker threads. Default: 1 + "workers": 1 +} diff --git a/conf/pktgen_sim.bess b/conf/pktgen_sim.bess new file mode 100644 index 000000000..3236a2c80 --- /dev/null +++ b/conf/pktgen_sim.bess @@ -0,0 +1,78 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright 2021 Intel Corporation + +""" +docker run --name pktgen_sim -td --restart unless-stopped \ + --cpuset-cpus=2-5 --ulimit memlock=-1 --cap-add IPC_LOCK \ + -v /dev/hugepages:/dev/hugepages -v "$PWD/conf":/opt/bess/bessctl/conf \ + --device=/dev/vfio/vfio --device=/dev/vfio/176 \ + upf-bess:"$( Rewrite(templates=n39_pkts) -> n39update::SequentialUpdate(**n3seq_kwargs) -> L4Checksum() -> IPChecksum() -> QueueOut(port=p.name, qid=0) +src36::Source(pkt_size=pkt_size) -> Rewrite(templates=n36_pkts) -> n36update::SequentialUpdate(**n3seq_kwargs) -> L4Checksum() -> IPChecksum() -> QueueOut(port=p.name, qid=1) + +src9::Source(pkt_size=pkt_size) -> Rewrite(templates=n9_pkts) -> n9update::SequentialUpdate(**n9seq_kwargs) -> L4Checksum() -> IPChecksum() -> QueueOut(port=p.name, qid=2) +src6::Source(pkt_size=pkt_size) -> Rewrite(templates=n6_pkts) -> n6update::SequentialUpdate(**n6seq_kwargs) -> L4Checksum() -> IPChecksum() -> QueueOut(port=p.name, qid=3) + +src39.attach_task(parent='39_limit') +src36.attach_task(parent='36_limit') + +src9.attach_task(parent='9_limit') +src6.attach_task(parent='6_limit') diff --git a/scripts/pktgen_setup.sh b/scripts/pktgen_setup.sh new file mode 100755 index 000000000..a18f54b64 --- /dev/null +++ b/scripts/pktgen_setup.sh @@ -0,0 +1,183 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2023-Present Intel Corporation + +set -e +# TCP port of bess/web monitor +gui_port=8000 +bessd_port=10514 +metrics_port=8080 + +# Driver options. Choose any one of the supported modes. +# +# "dpdk" set as default +# "af_xdp" uses AF_XDP sockets via DPDK's vdev for pkt I/O. This version is non-zc version. ZC version still needs to be evaluated. +# "af_packet" uses AF_PACKET sockets via DPDK's vdev for pkt I/O. +mode="dpdk" + +# Gateway interface(s) +# +# In the order of ("s1u/n3" "sgi/n6") +ifaces=("ens785f0" "ens785f1") + +# Static IP addresses of gateway interface(s) in cidr format +# +# In the order of (s1u/n3 sgi/n6) +ipaddrs=(198.18.0.2/30 198.19.0.2/30) + +# MAC addresses of gateway interface(s) +# +# In the order of (s1u/n3 sgi/n6) +macaddrs=(b4:96:91:b4:44:b0 b4:96:91:b4:44:b1) + +# Static IP addresses of the neighbors of gateway interface(s) +# +# In the order of (n-s1u/n3 n-sgi/n6) +nhipaddrs=(198.18.0.1 198.19.0.1) + +# Static MAC addresses of the neighbors of gateway interface(s) +# +# In the order of (n-s1u/n3 n-sgi/n6) +nhmacaddrs=(b4:96:91:b4:47:b8 b4:96:91:b4:47:b9) + +# IPv4 route table entries in cidr format per port +# +# In the order of ("{r-s1u/n3}" "{r-sgi/n6}") +routes=("21.1.1.128/25" "0.0.0.0/0") + +num_ifaces=${#ifaces[@]} +num_ipaddrs=${#ipaddrs[@]} + +# Set up static route and neighbor table entries of the SPGW/UPF +function setup_trafficgen_routes() { + for ((i = 0; i < num_ipaddrs; i++)); do + sudo ip netns exec pause ip neighbor add "${nhipaddrs[$i]}" lladdr "${nhmacaddrs[$i]}" dev "${ifaces[$i % num_ifaces]}" + routelist=${routes[$i]} + for route in $routelist; do + sudo ip netns exec pause ip route add "$route" via "${nhipaddrs[$i]}" metric 100 + done + done +} + +# Assign IP address(es) of gateway interface(s) within the network namespace +function setup_addrs() { + for ((i = 0; i < num_ipaddrs; i++)); do + sudo ip netns exec pause ip addr add "${ipaddrs[$i]}" dev "${ifaces[$i % $num_ifaces]}" + done +} + +# Set up mirror links to communicate with the kernel +# +# These vdev interfaces are used for ARP + ICMP updates. +# ARP/ICMP requests are sent via the vdev interface to the kernel. +# ARP/ICMP responses are captured and relayed out of the dpdk ports. +function setup_mirror_links() { + for ((i = 0; i < num_ifaces; i++)); do + sudo ip netns exec pause ip link add "${ifaces[$i]}" type veth peer name "${ifaces[$i]}"-vdev + sudo ip netns exec pause ip link set "${ifaces[$i]}" up + sudo ip netns exec pause ip link set "${ifaces[$i]}-vdev" up + sudo ip netns exec pause ip link set dev "${ifaces[$i]}" address "${macaddrs[$i]}" + done + setup_addrs +} + +# Set up interfaces in the network namespace. For non-"dpdk" mode(s) +function move_ifaces() { + for ((i = 0; i < num_ifaces; i++)); do + sudo ip link set "${ifaces[$i]}" netns pause up + sudo ip netns exec pause ip link set "${ifaces[$i]}" promisc off + sudo ip netns exec pause ip link set "${ifaces[$i]}" xdp off + if [ "$mode" == 'af_xdp' ]; then + sudo ip netns exec pause ethtool --features "${ifaces[$i]}" ntuple off + sudo ip netns exec pause ethtool --features "${ifaces[$i]}" ntuple on + sudo ip netns exec pause ethtool -N "${ifaces[$i]}" flow-type udp4 action 0 + sudo ip netns exec pause ethtool -N "${ifaces[$i]}" flow-type tcp4 action 0 + sudo ip netns exec pause ethtool -u "${ifaces[$i]}" + fi + done + setup_addrs +} + +# Stop previous instances of bess* before restarting +docker stop pause bess bess-routectl bess-web || true +docker rm -f pause bess bess-routectl bess-web || true +sudo rm -rf /var/run/netns/pause + +# Build +make docker-build + +if [ "$mode" == 'dpdk' ]; then + DEVICES=${DEVICES:-'--device=/dev/vfio/184 --device=/dev/vfio/185 --device=/dev/vfio/vfio'} + PRIVS='--cap-add IPC_LOCK' + +elif [ "$mode" == 'af_xdp' ]; then + PRIVS='--privileged' + +elif [ "$mode" == 'af_packet' ]; then + PRIVS='--cap-add IPC_LOCK' +fi + +# Run pause +docker run --name pause -td --restart unless-stopped \ + -p $bessd_port:$bessd_port \ + -p $gui_port:$gui_port \ + -p $metrics_port:$metrics_port \ + --hostname $(hostname) \ + k8s.gcr.io/pause + +# Emulate CNI + init container +sudo mkdir -p /var/run/netns +sandbox=$(docker inspect --format='{{.NetworkSettings.SandboxKey}}' pause) +sudo ln -s "$sandbox" /var/run/netns/pause + +case $mode in +"dpdk" | "sim") setup_mirror_links ;; +"af_xdp" | "af_packet") + move_ifaces + # Make sure that kernel does not send back icmp dest unreachable msg(s) + sudo ip netns exec pause iptables -I OUTPUT -p icmp --icmp-type port-unreachable -j DROP + ;; +*) ;; + +esac + +# Setup trafficgen routes +if [ "$mode" != 'sim' ]; then + setup_trafficgen_routes +fi + +# Run bessd +docker run --name bess -td --restart unless-stopped \ + -v /lib/firmware/intel/ice/ddp:/lib/firmware/intel/ice/ddp \ + --cpuset-cpus=54-71 \ + --ulimit memlock=-1 -v /dev/hugepages:/dev/hugepages \ + -v "$PWD/conf":/opt/bess/bessctl/conf \ + --net container:pause \ + $PRIVS \ + $DEVICES \ + upf-bess:"$( Date: Wed, 7 Oct 2026 10:40:54 -0700 Subject: [PATCH 3/4] Document external-datapath and loopback workflows Signed-off-by: Arrobo, Gabriel --- docs/INSTALL.md | 48 ++++++++++++++++------- docs/images/ubench-pktgen-sim.svg | 3 ++ docs/images/ubench-pktgen-sim.svg.license | 3 ++ 3 files changed, 39 insertions(+), 15 deletions(-) create mode 100755 docs/images/ubench-pktgen-sim.svg create mode 100644 docs/images/ubench-pktgen-sim.svg.license diff --git a/docs/INSTALL.md b/docs/INSTALL.md index ee3abb764..7669c4706 100644 --- a/docs/INSTALL.md +++ b/docs/INSTALL.md @@ -407,32 +407,50 @@ are provided in the [Configuration: Simulation mode](#configuration-simulation-m ### [Pktgen](../conf/pktgen.bess) -Pktgen allows us to test the upf pipeline with external datapath interfaces. -This can be done either using a single machine or two machines. +Pktgen lets us test the UPF pipeline with an external datapath, driving traffic +from a separate machine (or an extra VF on the same machine) into the UPF under +test. > Make sure the mode is not set to sim in scripts/docker_setup.sh and upf.jsonc ![ubench-pktgen](images/ubench-pktgen.svg) -1. On the same machine using an extra VF or from a different machine run pktgen instance +1. Customize [conf/pktgen.jsonc](../conf/pktgen.jsonc): set the `access`/`core` + interface names, the number of `workers`, and the `dl_rate`/`ul_rate` packet + rates in pps (`0` disables generation in that direction). Then set the + destination MAC/IP addresses of the UPF ports marked `TODO` in + [conf/pktgen.bess](../conf/pktgen.bess). + +2. Customize [scripts/pktgen_setup.sh](../scripts/pktgen_setup.sh) to match your + deployment: interface names, IP/MAC addresses, routes, and the VFIO + `--device` entries. + +3. Start pktgen. The script builds the image, sets up the datapath, and runs + [conf/pktgen.bess](../conf/pktgen.bess): ```bash - docker run --name pktgen -td --restart unless-stopped \ - --cpuset-cpus=2-5 --ulimit memlock=-1 --cap-add IPC_LOCK \ - -v /dev/hugepages:/dev/hugepages -v "$PWD/conf":/opt/bess/bessctl/conf \ - --device=/dev/vfio/vfio --device=/dev/vfio/176 \ - upf-bess:"$( + +
up4.bess/BESS
mode: dpdk
up4.bess/BESS...
pfcpiface -simulate
pfcpiface -simulate
access
access
core
core
pktgen_sim.bess/BESS
pktgen_sim.bess/BESS
NIC (same NIC, extra VF loopback)
NIC (same NIC, extra VF loopback)
Install sessions via gRPC
Install se...Viewer does not support full SVG 1.1 diff --git a/docs/images/ubench-pktgen-sim.svg.license b/docs/images/ubench-pktgen-sim.svg.license new file mode 100644 index 000000000..cced96315 --- /dev/null +++ b/docs/images/ubench-pktgen-sim.svg.license @@ -0,0 +1,3 @@ +Copyright (c) 2026 Intel Corporation + +SPDX-License-Identifier: Apache-2.0 From 426fee613a8bcfc11092082ff5a9bc6f730b1e52 Mon Sep 17 00:00:00 2001 From: "Arrobo, Gabriel" Date: Wed, 7 Oct 2026 11:49:56 -0700 Subject: [PATCH 4/4] Address review comments Signed-off-by: Arrobo, Gabriel --- .wordlist.txt | 4 ++ conf/parser.py | 16 ++++++ conf/pktgen.bess | 20 +++++++- conf/pktgen.jsonc | 17 ++++--- docs/INSTALL.md | 21 ++++++-- scripts/pktgen_setup.sh | 110 ++++++++++++++++++++++++++-------------- 6 files changed, 137 insertions(+), 51 deletions(-) diff --git a/.wordlist.txt b/.wordlist.txt index d6aea179d..a45ff3ebe 100644 --- a/.wordlist.txt +++ b/.wordlist.txt @@ -75,11 +75,14 @@ kubernetes linerate linux localhost +loopback lports MAKEFLAGS md memlock Microbenchmarks +namespace +namespaces Natting natting netdev @@ -103,6 +106,7 @@ pfcp pfcpsim PFDs pktgen +pps protobuf PTF PWD diff --git a/conf/parser.py b/conf/parser.py index d2c9aac97..21d2f5bd4 100755 --- a/conf/parser.py +++ b/conf/parser.py @@ -42,6 +42,7 @@ def __init__(self, fname): self.sim_start_n9_teid = None self.sim_pkt_size = None self.sim_total_flows = None + self.sim_qfi = 9 self.workers = 1 self.access_ifname = None self.core_ifname = None @@ -157,6 +158,21 @@ def parse(self, ifaces): except KeyError: print("Sim mode not selected.") + # QFI written into the PDU Session Container for uplink GTP-U + # encapsulation (pktgen). Parsed separately so an absent key keeps + # the default without discarding the sim block above. The PSC encodes + # QFI in 6 bits (see bess/core/utils/gtp.h), so reject anything outside + # 0-63: higher values would be silently truncated by GtpuEncap or break + # the one-byte SetMetadata module at init. + try: + qfi = int(self.conf["sim"]["qfi"]) + if 0 <= qfi <= 63: + self.sim_qfi = qfi + else: + print(f"sim qfi {qfi} out of range (0-63). Using default QFI 9.") + except (KeyError, ValueError, TypeError): + print("sim qfi not set. Using default QFI 9.") + # Parse workers try: self.workers = int(self.conf["workers"]) diff --git a/conf/pktgen.bess b/conf/pktgen.bess index 16db18366..6580f93df 100644 --- a/conf/pktgen.bess +++ b/conf/pktgen.bess @@ -146,7 +146,20 @@ n3RxFastBPF.add(filters=[uplink_filter]) # Uplink packet generation # ==================================================== -n3_pkts = [sim.gen_inet_packet(parser.sim_pkt_size, access_mac, access_dst_mac, +# GtpuEncap prepends an outer IPv4/UDP/GTP-U header (36 bytes), plus an +# 8-byte PDU Session Container when gtppsc is enabled (44 bytes total). +# Build the inner frame smaller by that overhead so the encapsulated uplink +# frame on the wire matches sim_pkt_size (and the downlink frames). +gtpu_encap_overhead = 44 if parser.gtppsc else 36 +n3_inner_size = parser.sim_pkt_size - gtpu_encap_overhead + +# PDU Session Container metadata consumed by GtpuEncap: the "action" +# attribute sets the PSC PDU type and "qfi" sets the PSC QFI. Uplink +# traffic uses PDU type 1 (UL PDU SESSION INFORMATION, 3GPP TS 38.415); +# QFI is configurable via pktgen.jsonc (sim.qfi). +ul_pdu_session_type = 1 + +n3_pkts = [sim.gen_inet_packet(n3_inner_size, access_mac, access_dst_mac, parser.sim_start_ue_ip, parser.sim_n6_app_ip)] n3_seq = sim.gen_inet_sequpdate_ul_args(parser.sim_total_flows, parser.sim_start_ue_ip) gtpu_seq = sim.gen_gtpu_sequpdate_ul_args(parser.sim_total_flows, parser.sim_start_n3_teid) @@ -155,7 +168,10 @@ gtpu_seq = sim.gen_gtpu_sequpdate_ul_args(parser.sim_total_flows, parser.sim_sta if parser.ul_rate > 0: n3TxSrc::Source() -> n3TxGen::Rewrite(templates=n3_pkts) \ -> n3TxSeq::SequentialUpdate(**n3_seq) -> n3TxL4Cksum::L4Checksum() \ - -> n3TxIPCksum::IPChecksum() -> gtpuEncap::GtpuEncap(add_psc=parser.gtppsc):1 \ + -> n3TxIPCksum::IPChecksum() \ + -> n3TxPscMeta::SetMetadata(attrs=[{'name': 'action', 'size': 1, 'value_int': ul_pdu_session_type}, \ + {'name': 'qfi', 'size': 1, 'value_int': parser.sim_qfi}]) \ + -> gtpuEncap::GtpuEncap(add_psc=parser.gtppsc):1 \ -> Update(fields=[{'offset': 26, 'size': 4, 'value': ip2long(parser.sim_start_enb_ip)}, \ {'offset': 30, 'size': 4, 'value': ip2long(access_dst_ip)}, \ {'offset': 34, 'size': 2, 'value': 2152}, \ diff --git a/conf/pktgen.jsonc b/conf/pktgen.jsonc index d71304f15..a653c70af 100644 --- a/conf/pktgen.jsonc +++ b/conf/pktgen.jsonc @@ -1,12 +1,14 @@ // SPDX-FileCopyrightText: 2020-present Open Networking Foundation // SPDX-License-Identifier: Apache-2.0 { - // UPF support various configuration modes: - // "af_packet"` to enable AF_PACKET mode, - // "af_xdp" to enable AF_XDP mode, + // Driver mode for the pktgen external datapath. Supported values: // "dpdk" to enable DPDK mode, - // "sim" to generate synthetic traffic from BESS's Source module, - // "" when running with UP4 + // "af_xdp" to enable AF_XDP mode, + // "af_packet" to enable AF_PACKET mode. + // "sim" is NOT supported here: pktgen.bess exits in sim mode. For + // single-machine loopback simulation use conf/pktgen_sim.bess instead. + // This value is the single source of truth: scripts/pktgen_setup.sh + // reads it to build the matching namespace topology. "mode": "dpdk", // [Optional] Set the log level to one of \"panic\", \"fatal\", \"error\", \"warning\", \"info\", \"debug\", \"trace\" @@ -25,7 +27,10 @@ "start_n3_teid": "0x30000000", "start_n9_teid": "0x90000000", "pkt_size": 128, - "total_flows": 5000 + "total_flows": 5000, + // QFI written into the PDU Session Container for uplink traffic + // (only used when "gtppsc" is enabled below) + "qfi": 9 }, // DL Data rate for traffic generation (in pps). '0' means dl traffic is disabled diff --git a/docs/INSTALL.md b/docs/INSTALL.md index 7669c4706..315f72d01 100644 --- a/docs/INSTALL.md +++ b/docs/INSTALL.md @@ -432,11 +432,22 @@ test. ./scripts/pktgen_setup.sh ``` -#### Single-machine (loopback) alternative - -To generate traffic on a single machine without external interfaces, use -[conf/pktgen_sim.bess](../conf/pktgen_sim.bess) instead. Customize it to match -[conf/upf.jsonc](../conf/upf.jsonc) from the + The generator namespaces its containers (`pktgen-pause`, `pktgen-bess`, + `pktgen-web`, `pktgen-routectl`), its network namespace (`pktgen`), and its + published host ports (BESS gRPC on `10515`, web GUI on `8001`, metrics on + `8081`) so it can coexist with a UPF deployment + ([scripts/docker_setup.sh](../scripts/docker_setup.sh)) on the same host + without tearing down the UPF under test. Observe the pipeline in the GUI at + `http://[hostip]:8001`. + +#### Single-machine (extra-VF loopback) alternative + +To generate traffic on a single machine without a second host, use +[conf/pktgen_sim.bess](../conf/pktgen_sim.bess) instead. This is not a +device-free setup: the pipeline still binds a `PMDPort`, so it requires an extra +NIC virtual function (VF) looped back to the UPF under test (bound to VFIO and +passed with `--device`, as in the command below). Customize the pipeline to +match [conf/upf.jsonc](../conf/upf.jsonc) from the [Configuration: Simulation mode](#configuration-simulation-mode) section, then run a pktgen instance and load the pipeline: diff --git a/scripts/pktgen_setup.sh b/scripts/pktgen_setup.sh index a18f54b64..419d88453 100755 --- a/scripts/pktgen_setup.sh +++ b/scripts/pktgen_setup.sh @@ -3,17 +3,51 @@ # Copyright (c) 2023-Present Intel Corporation set -e -# TCP port of bess/web monitor + +# Namespaced pktgen resources. +# +# The traffic generator reuses the same BESS image and pipeline as the UPF, +# so its containers, network namespace, and host ports are prefixed/offset to +# avoid colliding with a UPF deployment (scripts/docker_setup.sh) when both run +# on the same host (e.g. the extra-VF-on-the-same-machine setup). Without this, +# the "docker stop/rm" below and the published ports would tear down the UPF +# under test. +pause_name=pktgen-pause +bess_name=pktgen-bess +web_name=pktgen-web +routectl_name=pktgen-routectl +netns=pktgen + +# Container-internal ports. Kept at the BESS defaults so bessctl and +# route_control connect without extra flags. gui_port=8000 bessd_port=10514 metrics_port=8080 -# Driver options. Choose any one of the supported modes. +# Host-published ports. Offset from the BESS defaults so the published ports +# do not clash with a UPF deployment on the same host. +host_gui_port=8001 +host_bessd_port=10515 +host_metrics_port=8081 + +# Path to the pktgen config consumed by conf/pktgen.bess. +conf_file="${CONF_FILE:-conf/pktgen.jsonc}" + +# Driver mode. Read from pktgen.jsonc so the namespace topology built here +# always matches the port driver selected by the pipeline (conf/pktgen.bess): # -# "dpdk" set as default +# "dpdk" (default) sets up DPDK mirror veths. # "af_xdp" uses AF_XDP sockets via DPDK's vdev for pkt I/O. This version is non-zc version. ZC version still needs to be evaluated. # "af_packet" uses AF_PACKET sockets via DPDK's vdev for pkt I/O. -mode="dpdk" +# +# Editing the single "mode" value in pktgen.jsonc keeps setup and pipeline in +# sync; a mismatch would create the wrong topology (e.g. mirror veths while the +# pipeline expects a moved NIC) and the generator could not reach the datapath. +mode=$(grep -v '^[[:space:]]*//' "$conf_file" \ + | sed -n 's/.*"mode"[[:space:]]*:[[:space:]]*"\([^"]*\)".*/\1/p' \ + | head -1) +mode="${mode:-dpdk}" +echo "pktgen driver mode (from $conf_file): $mode" # Gateway interface(s) # @@ -51,10 +85,10 @@ num_ipaddrs=${#ipaddrs[@]} # Set up static route and neighbor table entries of the SPGW/UPF function setup_trafficgen_routes() { for ((i = 0; i < num_ipaddrs; i++)); do - sudo ip netns exec pause ip neighbor add "${nhipaddrs[$i]}" lladdr "${nhmacaddrs[$i]}" dev "${ifaces[$i % num_ifaces]}" + sudo ip netns exec "$netns" ip neighbor add "${nhipaddrs[$i]}" lladdr "${nhmacaddrs[$i]}" dev "${ifaces[$i % num_ifaces]}" routelist=${routes[$i]} for route in $routelist; do - sudo ip netns exec pause ip route add "$route" via "${nhipaddrs[$i]}" metric 100 + sudo ip netns exec "$netns" ip route add "$route" via "${nhipaddrs[$i]}" metric 100 done done } @@ -62,7 +96,7 @@ function setup_trafficgen_routes() { # Assign IP address(es) of gateway interface(s) within the network namespace function setup_addrs() { for ((i = 0; i < num_ipaddrs; i++)); do - sudo ip netns exec pause ip addr add "${ipaddrs[$i]}" dev "${ifaces[$i % $num_ifaces]}" + sudo ip netns exec "$netns" ip addr add "${ipaddrs[$i]}" dev "${ifaces[$i % $num_ifaces]}" done } @@ -73,10 +107,10 @@ function setup_addrs() { # ARP/ICMP responses are captured and relayed out of the dpdk ports. function setup_mirror_links() { for ((i = 0; i < num_ifaces; i++)); do - sudo ip netns exec pause ip link add "${ifaces[$i]}" type veth peer name "${ifaces[$i]}"-vdev - sudo ip netns exec pause ip link set "${ifaces[$i]}" up - sudo ip netns exec pause ip link set "${ifaces[$i]}-vdev" up - sudo ip netns exec pause ip link set dev "${ifaces[$i]}" address "${macaddrs[$i]}" + sudo ip netns exec "$netns" ip link add "${ifaces[$i]}" type veth peer name "${ifaces[$i]}"-vdev + sudo ip netns exec "$netns" ip link set "${ifaces[$i]}" up + sudo ip netns exec "$netns" ip link set "${ifaces[$i]}-vdev" up + sudo ip netns exec "$netns" ip link set dev "${ifaces[$i]}" address "${macaddrs[$i]}" done setup_addrs } @@ -84,24 +118,24 @@ function setup_mirror_links() { # Set up interfaces in the network namespace. For non-"dpdk" mode(s) function move_ifaces() { for ((i = 0; i < num_ifaces; i++)); do - sudo ip link set "${ifaces[$i]}" netns pause up - sudo ip netns exec pause ip link set "${ifaces[$i]}" promisc off - sudo ip netns exec pause ip link set "${ifaces[$i]}" xdp off + sudo ip link set "${ifaces[$i]}" netns "$netns" up + sudo ip netns exec "$netns" ip link set "${ifaces[$i]}" promisc off + sudo ip netns exec "$netns" ip link set "${ifaces[$i]}" xdp off if [ "$mode" == 'af_xdp' ]; then - sudo ip netns exec pause ethtool --features "${ifaces[$i]}" ntuple off - sudo ip netns exec pause ethtool --features "${ifaces[$i]}" ntuple on - sudo ip netns exec pause ethtool -N "${ifaces[$i]}" flow-type udp4 action 0 - sudo ip netns exec pause ethtool -N "${ifaces[$i]}" flow-type tcp4 action 0 - sudo ip netns exec pause ethtool -u "${ifaces[$i]}" + sudo ip netns exec "$netns" ethtool --features "${ifaces[$i]}" ntuple off + sudo ip netns exec "$netns" ethtool --features "${ifaces[$i]}" ntuple on + sudo ip netns exec "$netns" ethtool -N "${ifaces[$i]}" flow-type udp4 action 0 + sudo ip netns exec "$netns" ethtool -N "${ifaces[$i]}" flow-type tcp4 action 0 + sudo ip netns exec "$netns" ethtool -u "${ifaces[$i]}" fi done setup_addrs } -# Stop previous instances of bess* before restarting -docker stop pause bess bess-routectl bess-web || true -docker rm -f pause bess bess-routectl bess-web || true -sudo rm -rf /var/run/netns/pause +# Stop previous instances of the pktgen containers before restarting +docker stop "$pause_name" "$bess_name" "$routectl_name" "$web_name" || true +docker rm -f "$pause_name" "$bess_name" "$routectl_name" "$web_name" || true +sudo rm -rf /var/run/netns/"$netns" # Build make docker-build @@ -118,24 +152,24 @@ elif [ "$mode" == 'af_packet' ]; then fi # Run pause -docker run --name pause -td --restart unless-stopped \ - -p $bessd_port:$bessd_port \ - -p $gui_port:$gui_port \ - -p $metrics_port:$metrics_port \ +docker run --name "$pause_name" -td --restart unless-stopped \ + -p $host_bessd_port:$bessd_port \ + -p $host_gui_port:$gui_port \ + -p $host_metrics_port:$metrics_port \ --hostname $(hostname) \ k8s.gcr.io/pause # Emulate CNI + init container sudo mkdir -p /var/run/netns -sandbox=$(docker inspect --format='{{.NetworkSettings.SandboxKey}}' pause) -sudo ln -s "$sandbox" /var/run/netns/pause +sandbox=$(docker inspect --format='{{.NetworkSettings.SandboxKey}}' "$pause_name") +sudo ln -s "$sandbox" /var/run/netns/"$netns" case $mode in "dpdk" | "sim") setup_mirror_links ;; "af_xdp" | "af_packet") move_ifaces # Make sure that kernel does not send back icmp dest unreachable msg(s) - sudo ip netns exec pause iptables -I OUTPUT -p icmp --icmp-type port-unreachable -j DROP + sudo ip netns exec "$netns" iptables -I OUTPUT -p icmp --icmp-type port-unreachable -j DROP ;; *) ;; @@ -147,26 +181,26 @@ if [ "$mode" != 'sim' ]; then fi # Run bessd -docker run --name bess -td --restart unless-stopped \ +docker run --name "$bess_name" -td --restart unless-stopped \ -v /lib/firmware/intel/ice/ddp:/lib/firmware/intel/ice/ddp \ --cpuset-cpus=54-71 \ --ulimit memlock=-1 -v /dev/hugepages:/dev/hugepages \ -v "$PWD/conf":/opt/bess/bessctl/conf \ - --net container:pause \ + --net container:"$pause_name" \ $PRIVS \ $DEVICES \ upf-bess:"$(