From 15bac1860c338fd60ea415d6ebbcfa66fa960052 Mon Sep 17 00:00:00 2001 From: toffee <24222058@bjtu.edu.cn> Date: Tue, 26 May 2026 12:35:57 +0800 Subject: [PATCH 1/3] Warn if Linux automatic NUMA balancing is enabled at buffer init --- README.md | 13 +++++++++++++ deep_ep/__init__.py | 26 ++++++++++++++++++++++++++ 2 files changed, 39 insertions(+) diff --git a/README.md b/README.md index 507e2ba2d..b41414076 100644 --- a/README.md +++ b/README.md @@ -335,6 +335,7 @@ The library provides some environment variables, which may be useful: - General - `EP_BUFFER_DEBUG`: `0` or `1`, print buffer initialization, SM approximation, and backend debugging information, `0` by default - `EP_SUPPRESS_NCCL_CHECK`: `0` or `1`, suppress NCCL version mismatch checking, `0` by default + - `EP_SUPPRESS_NUMA_CHECK`: `0` or `1`, suppress the Linux automatic NUMA balancing warning at import (see [Performance gotchas](#performance-gotchas)), `0` by default - `EP_AVOID_RECORD_STREAM`: `0` or `1`, avoid `record_stream` on output tensors, `0` by default - `EP_NUM_TOPK_IDX_BITS`: integer, override the number of bits for top-k index encoding, `0` (auto) by default - Networking @@ -399,6 +400,18 @@ If the hardware supports it, we recommend using the following command to set the sudo mlxconfig -y -d mlx5_$i set PCI_ATOMIC_MODE=4 ``` +## Performance gotchas + +DeepEP's communication kernels run on the critical path of every MoE step, so host-side interference is directly visible in dispatch/combine latency: + +- **Linux automatic NUMA balancing** (`kernel.numa_balancing != 0`, enabled by default on most distros). The `task_numa_work` kernel routine rewrites PTEs at the user/kernel boundary and can add up to 16+ ms to affected `internode_dispatch` invocations (issue #624 trace: 139 invocations over 60 s on a 5.15 kernel running SGLang, avg 5 ms with 2 samples in the 16-31 ms bucket). We recommend disabling it on EP serving nodes: + + ```bash + sudo sysctl -w kernel.numa_balancing=0 + ``` + + DeepEP emits a warning at module init if it is still enabled. Silence it with `EP_SUPPRESS_NUMA_CHECK=1` if you cannot toggle the sysctl (e.g. inside a container). + ## Experimental branches - [Zero-copy](https://github.com/deepseek-ai/DeepEP/pull/453) diff --git a/deep_ep/__init__.py b/deep_ep/__init__.py index 39854394e..511dc42db 100644 --- a/deep_ep/__init__.py +++ b/deep_ep/__init__.py @@ -68,6 +68,31 @@ def check_nccl_so(): f'please contact Chenggang or Shangyan to upgrade PyTorch NCCL version') +def check_numa_balancing(): + """ + Warn if Linux automatic NUMA balancing is enabled, which can add tail latency to internode dispatch. + """ + if int(os.environ.get('EP_SUPPRESS_NUMA_CHECK', 0)): + return + + # Non-Linux systems or unreadable procfs (e.g. containers): skip silently + try: + with open('/proc/sys/kernel/numa_balancing', 'r') as f: + value = f.read().strip() + except OSError: + return + + # `kernel.numa_balancing` is a bitmask; any non-zero value runs `task_numa_work` + if value and value != '0': + import warnings + warnings.warn( + f'Automatic NUMA balancing is enabled (kernel.numa_balancing={value}), which can add ' + 'up to 16+ ms tail latency to DeepEP internode dispatch. Disable with ' + '`sudo sysctl -w kernel.numa_balancing=0`, or set EP_SUPPRESS_NUMA_CHECK=1 to silence. ' + 'See https://github.com/deepseek-ai/DeepEP/issues/624.', + RuntimeWarning, stacklevel=2) + + def init_jit(): """ Initialize the JIT compilation runtime. Sets up CUDA and NCCL root paths for the JIT compiler. @@ -81,6 +106,7 @@ def init_jit(): # Run initialization check_nccl_so() +check_numa_balancing() init_jit() From f26be9f1ee35cdc3ec576c858b605e713c3ec04a Mon Sep 17 00:00:00 2001 From: toffee <260035245+toffee-desuwa@users.noreply.github.com> Date: Mon, 1 Jun 2026 22:18:20 +0800 Subject: [PATCH 2/3] Soften NUMA balancing warning wording based on counter-evidence A contributor tested with frequent DeepEP internode dispatch on a 5.10 kernel (no NUMA pinning, numa_balancing=1) and observed negligible overhead (4 hits/60s vs 139 in issue #624 on 5.15). - Replace "can add up to 16+ ms" with "can add milliseconds of tail latency on some kernel versions and workloads" - Change "Disable with" to "If you observe unexpected latency spikes, consider:" - Add the counter-example to README for context Trigger condition (numa_balancing != 0) and warning level (RuntimeWarning) unchanged. Co-Authored-By: Claude Opus 4.6 (1M context) --- README.md | 2 +- deep_ep/__init__.py | 12 ++++++++---- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index b41414076..9e89f56dc 100644 --- a/README.md +++ b/README.md @@ -404,7 +404,7 @@ sudo mlxconfig -y -d mlx5_$i set PCI_ATOMIC_MODE=4 DeepEP's communication kernels run on the critical path of every MoE step, so host-side interference is directly visible in dispatch/combine latency: -- **Linux automatic NUMA balancing** (`kernel.numa_balancing != 0`, enabled by default on most distros). The `task_numa_work` kernel routine rewrites PTEs at the user/kernel boundary and can add up to 16+ ms to affected `internode_dispatch` invocations (issue #624 trace: 139 invocations over 60 s on a 5.15 kernel running SGLang, avg 5 ms with 2 samples in the 16-31 ms bucket). We recommend disabling it on EP serving nodes: +- **Linux automatic NUMA balancing** (`kernel.numa_balancing != 0`, enabled by default on most distros). The `task_numa_work` kernel routine rewrites PTEs at the user/kernel boundary and can add milliseconds of tail latency to `internode_dispatch` calls on some configurations (issue #624 trace on a 5.15 kernel running SGLang: 139 invocations over 60 s, avg 5 ms; a separate test on a 5.10 kernel with frequent internode dispatch showed negligible overhead). If you observe unexpected latency spikes on EP serving nodes, consider disabling it: ```bash sudo sysctl -w kernel.numa_balancing=0 diff --git a/deep_ep/__init__.py b/deep_ep/__init__.py index 511dc42db..fe12e2d2e 100644 --- a/deep_ep/__init__.py +++ b/deep_ep/__init__.py @@ -86,10 +86,14 @@ def check_numa_balancing(): if value and value != '0': import warnings warnings.warn( - f'Automatic NUMA balancing is enabled (kernel.numa_balancing={value}), which can add ' - 'up to 16+ ms tail latency to DeepEP internode dispatch. Disable with ' - '`sudo sysctl -w kernel.numa_balancing=0`, or set EP_SUPPRESS_NUMA_CHECK=1 to silence. ' - 'See https://github.com/deepseek-ai/DeepEP/issues/624.', + f'Automatic NUMA balancing is enabled (kernel.numa_balancing={value}). ' + 'On some kernel versions and workloads, the page scanner can add ' + 'milliseconds of tail latency to RDMA-based internode dispatch ' + '(see issue #624 for a 5.15-kernel example). ' + 'If you observe unexpected latency spikes, consider: ' + '`sudo sysctl -w kernel.numa_balancing=0`. ' + 'Set EP_SUPPRESS_NUMA_CHECK=1 to silence this warning. ' + 'Details: https://github.com/deepseek-ai/DeepEP/issues/624', RuntimeWarning, stacklevel=2) From d389e22166516dd4d68c7179e3bd497ea49ddd54 Mon Sep 17 00:00:00 2001 From: toffee <260035245+toffee-desuwa@users.noreply.github.com> Date: Fri, 12 Jun 2026 16:59:56 +0800 Subject: [PATCH 3/3] Explain the actual NUMA balancing cost mechanism in warning and README Per wenjianhn's root-cause explanation in #624: the kernel hands task_numa_work to busy threads, and the per-scan cost grows with process resident memory. A dispatch thread at 100% CPU inside a process holding hundreds of GB is the worst case. Replace the earlier kernel-version framing (a red herring) with this mechanism in both the warning text and the README. Co-Authored-By: Claude --- README.md | 2 +- deep_ep/__init__.py | 9 +++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 9e89f56dc..0f1f4b103 100644 --- a/README.md +++ b/README.md @@ -404,7 +404,7 @@ sudo mlxconfig -y -d mlx5_$i set PCI_ATOMIC_MODE=4 DeepEP's communication kernels run on the critical path of every MoE step, so host-side interference is directly visible in dispatch/combine latency: -- **Linux automatic NUMA balancing** (`kernel.numa_balancing != 0`, enabled by default on most distros). The `task_numa_work` kernel routine rewrites PTEs at the user/kernel boundary and can add milliseconds of tail latency to `internode_dispatch` calls on some configurations (issue #624 trace on a 5.15 kernel running SGLang: 139 invocations over 60 s, avg 5 ms; a separate test on a 5.10 kernel with frequent internode dispatch showed negligible overhead). If you observe unexpected latency spikes on EP serving nodes, consider disabling it: +- **Linux automatic NUMA balancing** (`kernel.numa_balancing != 0`, enabled by default on most distros). The kernel hands NUMA page scanning (`task_numa_work`) to busy threads at the user/kernel boundary, and the per-scan cost grows with the process's resident memory. An EP serving process with large memory and a busy dispatch thread is the worst case: the trace in issue #624 (hundreds of GB resident, dispatch thread near 100% CPU) hit 139 invocations over 60 s at 5 ms average, while a smaller-footprint setup reported negligible overhead. If you observe unexpected latency spikes on EP serving nodes, consider disabling it: ```bash sudo sysctl -w kernel.numa_balancing=0 diff --git a/deep_ep/__init__.py b/deep_ep/__init__.py index fe12e2d2e..8f6d6cde2 100644 --- a/deep_ep/__init__.py +++ b/deep_ep/__init__.py @@ -87,10 +87,11 @@ def check_numa_balancing(): import warnings warnings.warn( f'Automatic NUMA balancing is enabled (kernel.numa_balancing={value}). ' - 'On some kernel versions and workloads, the page scanner can add ' - 'milliseconds of tail latency to RDMA-based internode dispatch ' - '(see issue #624 for a 5.15-kernel example). ' - 'If you observe unexpected latency spikes, consider: ' + 'The kernel hands NUMA page scanning (task_numa_work) to busy threads, ' + 'and the per-scan cost grows with process memory. For EP serving ' + 'processes with large resident memory, this can add milliseconds of ' + 'tail latency to internode dispatch (see issue #624). ' + 'If you observe latency spikes, consider: ' '`sudo sysctl -w kernel.numa_balancing=0`. ' 'Set EP_SUPPRESS_NUMA_CHECK=1 to silence this warning. ' 'Details: https://github.com/deepseek-ai/DeepEP/issues/624',