From 95504970c760ccd7e0a25d14277da77e2d659893 Mon Sep 17 00:00:00 2001 From: Pawel Beza Date: Fri, 28 Aug 2026 17:46:59 +0200 Subject: [PATCH] Add option to disable gpu metrics collection MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Eslam Elnikety Signed-off-by: Eslam Elnikety Signed-off-by: Paweł Bęza Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- python/ray/_private/node.py | 4 ++++ python/ray/_private/services.py | 6 ++++++ python/ray/dashboard/agent.py | 8 ++++++++ .../ray/dashboard/modules/reporter/gpu_providers.py | 8 ++++++-- .../ray/dashboard/modules/reporter/reporter_agent.py | 4 +++- .../modules/reporter/tests/test_gpu_providers.py | 12 ++++++++++++ src/ray/common/ray_config_def.h | 4 ++++ 7 files changed, 43 insertions(+), 3 deletions(-) diff --git a/python/ray/_private/node.py b/python/ray/_private/node.py index b8f8bbca1d59..bdb1aaf73d8e 100644 --- a/python/ray/_private/node.py +++ b/python/ray/_private/node.py @@ -1533,6 +1533,10 @@ def start_raylet( node_name=self._ray_params.node_name, webui=self._webui_url, resource_isolation_config=self.resource_isolation_config, + enable_gpu_metrics_collection=self._config.get( + "enable_gpu_metrics_collection", + ray_constants.env_bool("RAY_enable_gpu_metrics_collection", True), + ), ) assert ray_constants.PROCESS_TYPE_RAYLET not in self.all_processes self.all_processes[ray_constants.PROCESS_TYPE_RAYLET] = [process_info] diff --git a/python/ray/_private/services.py b/python/ray/_private/services.py index 120a20582cb2..55cecd0976c8 100644 --- a/python/ray/_private/services.py +++ b/python/ray/_private/services.py @@ -1658,6 +1658,7 @@ def start_raylet( env_updates: Optional[dict] = None, node_name: Optional[str] = None, webui: Optional[str] = None, + enable_gpu_metrics_collection: bool = True, ): """Start a raylet, which is a combined local scheduler and object manager. @@ -1741,6 +1742,8 @@ def start_raylet( env_updates: Environment variable overrides. node_name: The name of the node. webui: The url of the UI. + enable_gpu_metrics_collection: Whether the dashboard agent should collect + GPU metrics. Returns: ProcessInfo for the process that was started. """ @@ -1915,6 +1918,9 @@ def start_raylet( if is_head_node: dashboard_agent_command.append("--head") + if not enable_gpu_metrics_collection: + dashboard_agent_command.append("--disable-gpu-metrics") + runtime_env_agent_command = [ *_build_python_executable_command_memory_profileable( ray_constants.PROCESS_TYPE_RUNTIME_ENV_AGENT, session_dir diff --git a/python/ray/dashboard/agent.py b/python/ray/dashboard/agent.py index 9955bc7bc221..7a66cfe80220 100644 --- a/python/ray/dashboard/agent.py +++ b/python/ray/dashboard/agent.py @@ -49,6 +49,7 @@ def __init__( events_export_addr=None, listen_port=ray_constants.DEFAULT_DASHBOARD_AGENT_LISTEN_PORT, disable_metrics_collection: bool = False, + disable_gpu_metrics: bool = False, is_head: bool = False, *, # the following are required kwargs object_store_name: str, @@ -80,6 +81,7 @@ def __init__( self.raylet_name = raylet_name self.node_id = os.environ["RAY_NODE_ID"] self.metrics_collection_disabled = disable_metrics_collection + self.gpu_metrics_enabled = not disable_gpu_metrics self.session_name = session_name # grpc server is None in mininal. @@ -446,6 +448,11 @@ async def wait_forever(): action="store_true", help=("If this arg is set, metrics report won't be enabled from the agent."), ) + parser.add_argument( + "--disable-gpu-metrics", + action="store_true", + help="Disable GPU metric collection in the dashboard agent.", + ) parser.add_argument( "--head", action="store_true", @@ -520,6 +527,7 @@ async def wait_forever(): object_store_name=args.object_store_name, raylet_name=args.raylet_name, disable_metrics_collection=args.disable_metrics_collection, + disable_gpu_metrics=args.disable_gpu_metrics, is_head=args.head, session_name=args.session_name, ) diff --git a/python/ray/dashboard/modules/reporter/gpu_providers.py b/python/ray/dashboard/modules/reporter/gpu_providers.py index 65be58af0d06..9a5567f469f2 100644 --- a/python/ray/dashboard/modules/reporter/gpu_providers.py +++ b/python/ray/dashboard/modules/reporter/gpu_providers.py @@ -526,14 +526,18 @@ def get_gpu_utilization(self) -> List[GpuUtilizationInfo]: class GpuMetricProvider: """Provider class for GPU metrics collection.""" - def __init__(self): + def __init__(self, enable_metric_report: bool = True): self._provider: Optional[GpuProvider] = None - self._enable_metric_report = True + self._enable_metric_report = enable_metric_report self._providers = [NvidiaGpuProvider(), AmdGpuProvider()] self._initialized = False def initialize(self) -> bool: """Initialize the GPU metric provider by detecting available GPU providers.""" + if not self._enable_metric_report: + self._initialized = True + return False + if self._initialized: return True diff --git a/python/ray/dashboard/modules/reporter/reporter_agent.py b/python/ray/dashboard/modules/reporter/reporter_agent.py index bd8840bc8bb0..a6b6856bbb8a 100644 --- a/python/ray/dashboard/modules/reporter/reporter_agent.py +++ b/python/ray/dashboard/modules/reporter/reporter_agent.py @@ -578,7 +578,9 @@ def __init__(self, dashboard_agent, raylet_client=None): ) # Create GPU metric provider instance - self._gpu_metric_provider = GpuMetricProvider() + self._gpu_metric_provider = GpuMetricProvider( + enable_metric_report=dashboard_agent.gpu_metrics_enabled + ) if raylet_client: self._raylet_client = raylet_client diff --git a/python/ray/dashboard/modules/reporter/tests/test_gpu_providers.py b/python/ray/dashboard/modules/reporter/tests/test_gpu_providers.py index 61c262a7e0fa..853d8b875ab8 100644 --- a/python/ray/dashboard/modules/reporter/tests/test_gpu_providers.py +++ b/python/ray/dashboard/modules/reporter/tests/test_gpu_providers.py @@ -485,6 +485,18 @@ def test_init(self): self.assertEqual(len(self.provider._providers), 2) self.assertFalse(self.provider._initialized) + @patch.object(GpuMetricProvider, "_detect_gpu_provider") + def test_initialize_disabled(self, mock_detect): + """Test disabled GPU metrics do not probe GPU providers.""" + provider = GpuMetricProvider(enable_metric_report=False) + + self.assertFalse(provider.initialize()) + self.assertFalse(provider.initialize()) + self.assertEqual(provider.get_gpu_usage(), []) + self.assertTrue(provider._initialized) + self.assertFalse(provider.is_metric_report_enabled()) + mock_detect.assert_not_called() + @patch.object(NvidiaGpuProvider, "is_available", return_value=True) @patch.object(AmdGpuProvider, "is_available", return_value=False) def test_detect_gpu_provider_nvidia( diff --git a/src/ray/common/ray_config_def.h b/src/ray/common/ray_config_def.h index 60d82a1fe13d..fb0cc1ff554b 100644 --- a/src/ray/common/ray_config_def.h +++ b/src/ray/common/ray_config_def.h @@ -647,6 +647,10 @@ RAY_CONFIG(uint64_t, gcs_mark_task_failed_on_worker_dead_delay_ms, /* 1 secs */ /// Whether or not we enable metrics collection. RAY_CONFIG(bool, enable_metrics_collection, true) +/// Whether the dashboard agent collects GPU metrics. Disabling this avoids +/// polling GPU management libraries such as NVML. +RAY_CONFIG(bool, enable_gpu_metrics_collection, true) + /// Determine if the high cardinality labels such as WorkerId, task and actor Name /// should be used in the metrics. For the complete definition, see /// RAY_METRIC_CARDINALITY_LEVEL in ray_constants.py