From 30b0d3eba79f5801b28699a2705e06228aac5e35 Mon Sep 17 00:00:00 2001 From: Laurie Bax <47383333+lauriebax@users.noreply.github.com> Date: Tue, 28 Oct 2025 10:23:30 +0000 Subject: [PATCH 01/15] add python3.12 to supported pythons, add test for python 3.12 --- .github/workflows/pr_stage_test.yml | 2 +- pyproject.toml | 4 +++- uv.lock | 2 +- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pr_stage_test.yml b/.github/workflows/pr_stage_test.yml index b91c9ee3e5..59c268bbc0 100644 --- a/.github/workflows/pr_stage_test.yml +++ b/.github/workflows/pr_stage_test.yml @@ -98,7 +98,7 @@ jobs: torch: '2.4.1' cuda: 'cu118' image: 'nvidia/cuda:11.8.0-cudnn8-runtime-ubuntu22.04' - - python-version: '3.10' + - python-version: '3.12' torch: '2.8.0' cuda: 'cu129' image: 'nvidia/cuda:12.9.1-cudnn-runtime-ubuntu22.04' diff --git a/pyproject.toml b/pyproject.toml index 43f92a1ad2..ac7b43843d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "onedl-mmengine" -version = "0.10.8" +version = "0.10.9" description = "Engine of VBTI projects" readme = "README.md" authors = [ @@ -14,6 +14,8 @@ classifiers = [ "Operating System :: OS Independent", "Programming Language :: Python :: 3", "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", "Topic :: Utilities" ] dependencies = [ diff --git a/uv.lock b/uv.lock index fb2cd91297..7b4002f0d6 100644 --- a/uv.lock +++ b/uv.lock @@ -2802,7 +2802,7 @@ wheels = [ [[package]] name = "onedl-mmengine" -version = "0.10.8" +version = "0.10.9" source = { editable = "." } dependencies = [ { name = "addict" }, From f4a8be09c49749c6f578294889b4d92e954fd8d1 Mon Sep 17 00:00:00 2001 From: Laurie Bax <47383333+lauriebax@users.noreply.github.com> Date: Tue, 28 Oct 2025 12:14:08 +0000 Subject: [PATCH 02/15] add uv lock as pre-commit hook --- .pre-commit-config.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index dc5234058f..ba8038d668 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -65,3 +65,8 @@ repos: | ^docs ) additional_dependencies: ["types-setuptools", "types-requests", "types-PyYAML"] + - repo: https://github.com/astral-sh/uv-pre-commit + # uv version. + rev: 0.9.5 + hooks: + - id: uv-lock From a4e817532c6509c36e99084c6892b1b54ce7097c Mon Sep 17 00:00:00 2001 From: Laurie Bax <47383333+lauriebax@users.noreply.github.com> Date: Tue, 28 Oct 2025 12:14:42 +0000 Subject: [PATCH 03/15] relax constraints on aim for python 3.12 support --- pyproject.toml | 4 ++-- uv.lock | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index ac7b43843d..1e04481027 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,7 +31,7 @@ dependencies = [ [dependency-groups] test = [ - "aim<=3.17.5; sys_platform != 'win32'", + "aim; sys_platform != 'win32'", "bitsandbytes", "clearml", "coverage", @@ -79,7 +79,7 @@ all = [ "termcolor", "yapf", "regex; sys_platform == 'win32'", - "aim<=3.17.5; sys_platform != 'win32'", + "aim; sys_platform != 'win32'", "bitsandbytes", "clearml", "coverage", diff --git a/uv.lock b/uv.lock index 7b4002f0d6..47bd3168f9 100644 --- a/uv.lock +++ b/uv.lock @@ -2887,7 +2887,7 @@ test-lite = [ requires-dist = [ { name = "addict" }, { name = "addict", marker = "extra == 'all'" }, - { name = "aim", marker = "sys_platform != 'win32' and extra == 'all'", specifier = "<=3.17.5" }, + { name = "aim", marker = "sys_platform != 'win32' and extra == 'all'" }, { name = "bitsandbytes", marker = "extra == 'all'" }, { name = "clearml", marker = "extra == 'all'" }, { name = "coverage", marker = "extra == 'all'" }, @@ -2935,7 +2935,7 @@ docs = [ { name = "urllib3" }, ] test = [ - { name = "aim", marker = "sys_platform != 'win32'", specifier = "<=3.17.5" }, + { name = "aim", marker = "sys_platform != 'win32'" }, { name = "bitsandbytes" }, { name = "clearml" }, { name = "coverage" }, From 04eb931c7ff7437a06d231354752f5e40810f1da Mon Sep 17 00:00:00 2001 From: Laurie Bax <47383333+lauriebax@users.noreply.github.com> Date: Tue, 28 Oct 2025 14:23:45 +0000 Subject: [PATCH 04/15] make compatible with numpy 2 --- mmengine/config/utils.py | 3 +++ .../config/lazy_module_config/test_ast_transform.py | 2 +- tests/test_config/test_lazy.py | 10 +++++----- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/mmengine/config/utils.py b/mmengine/config/utils.py index 001a2267ac..81e53aa637 100644 --- a/mmengine/config/utils.py +++ b/mmengine/config/utils.py @@ -177,6 +177,9 @@ def _is_builtin_module(module_name: str) -> bool: origin_path = getattr(spec, 'origin', None) if origin_path is None: return False + # Handle frozen modules (e.g., os module in some Python distributions) + if origin_path == 'frozen': + return True origin_path = osp.abspath(origin_path) if ('site-package' in origin_path or 'dist-package' in origin_path or not origin_path.startswith( diff --git a/tests/data/config/lazy_module_config/test_ast_transform.py b/tests/data/config/lazy_module_config/test_ast_transform.py index a8803dde24..6f0ada1736 100644 --- a/tests/data/config/lazy_module_config/test_ast_transform.py +++ b/tests/data/config/lazy_module_config/test_ast_transform.py @@ -3,7 +3,7 @@ from importlib.util import find_spec as find_module import numpy -import numpy.compat +import numpy.fft import numpy.linalg as linalg from mmengine.config import Config diff --git a/tests/test_config/test_lazy.py b/tests/test_config/test_lazy.py index d69822814b..78676ed4c5 100644 --- a/tests/test_config/test_lazy.py +++ b/tests/test_config/test_lazy.py @@ -8,7 +8,7 @@ from unittest import TestCase import numpy -import numpy.compat +import numpy.fft import numpy.linalg as linalg from rich.progress import Progress @@ -56,17 +56,17 @@ def test_lazy_module(self): # 1.2 getattr as LazyAttr self.assertIsInstance(lazy_numpy.linalg, LazyAttr) - self.assertIsInstance(lazy_numpy.compat, LazyAttr) + self.assertIsInstance(lazy_numpy.fft, LazyAttr) # 1.3 Build module from LazyObject. amp and functional can be accessed imported_numpy = lazy_numpy.build() self.assertIs(imported_numpy.linalg, linalg) - self.assertIs(imported_numpy.compat, numpy.compat) + self.assertIs(imported_numpy.fft, numpy.fft) # 1.4.1 Build module from LazyAttr imported_linalg = lazy_numpy.linalg.build() - imported_compat = lazy_numpy.compat.build() - self.assertIs(imported_compat, numpy.compat) + imported_compat = lazy_numpy.fft.build() + self.assertIs(imported_compat, numpy.fft) self.assertIs(imported_linalg, linalg) # 1.4.2 build class method from LazyAttr From 91fff7b5ff6f58ae0d84d435b6b0e98ed123de77 Mon Sep 17 00:00:00 2001 From: Laurie Bax <47383333+lauriebax@users.noreply.github.com> Date: Tue, 28 Oct 2025 14:25:10 +0000 Subject: [PATCH 05/15] remove code coverage report post from pr pipeline --- .github/workflows/pr_stage_test.yml | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/.github/workflows/pr_stage_test.yml b/.github/workflows/pr_stage_test.yml index 59c268bbc0..382d878eed 100644 --- a/.github/workflows/pr_stage_test.yml +++ b/.github/workflows/pr_stage_test.yml @@ -155,12 +155,7 @@ jobs: format: markdown output: both hide_branch_rate: false - - name: Add Coverage PR Comment - uses: marocchino/sticky-pull-request-comment@v2 - if: github.event_name == 'pull_request' - with: - recreate: true - path: code-coverage-results.md + # build_windows: # runs-on: windows-2022 From b7bee242f4c0e8db8a3e874532ff176ef2e36533 Mon Sep 17 00:00:00 2001 From: Laurie Bax <47383333+lauriebax@users.noreply.github.com> Date: Tue, 28 Oct 2025 14:52:30 +0000 Subject: [PATCH 06/15] optimize AimVisBackend tests to prevent database locks. --- tests/test_visualizer/test_vis_backend.py | 31 +++++++++++------------ 1 file changed, 15 insertions(+), 16 deletions(-) diff --git a/tests/test_visualizer/test_vis_backend.py b/tests/test_visualizer/test_vis_backend.py index c991462ef9..0e090583e2 100644 --- a/tests/test_visualizer/test_vis_backend.py +++ b/tests/test_visualizer/test_vis_backend.py @@ -462,37 +462,36 @@ def test_close(self): reason='Aim does not support Windows for now.') class TestAimVisBackend: + @classmethod + def setup_class(cls): + """Setup AimVisBackend instance once for all tests in this class.""" + cls.aim_vis_backend = AimVisBackend() + def test_init(self): AimVisBackend() VISBACKENDS.build(dict(type='AimVisBackend')) def test_experiment(self): - aim_vis_backend = AimVisBackend() - assert aim_vis_backend.experiment == aim_vis_backend._aim_run + assert self.aim_vis_backend.experiment == self.aim_vis_backend._aim_run def test_add_config(self): cfg = Config(dict(a=1, b=dict(b1=[0, 1]))) - aim_vis_backend = AimVisBackend() - aim_vis_backend.add_config(cfg) + self.aim_vis_backend.add_config(cfg) def test_add_image(self): image = np.random.randint(0, 256, size=(10, 10, 3)).astype(np.uint8) - aim_vis_backend = AimVisBackend() - aim_vis_backend.add_image('img', image) - aim_vis_backend.add_image('img', image, step=1) + self.aim_vis_backend.add_image('img', image) + self.aim_vis_backend.add_image('img', image, step=1) def test_add_scalar(self): - aim_vis_backend = AimVisBackend() - aim_vis_backend.add_scalar('map', 0.9) - aim_vis_backend.add_scalar('map', 0.9, step=1) - aim_vis_backend.add_scalar('map', 0.95, step=2) + self.aim_vis_backend.add_scalar('map', 0.9) + self.aim_vis_backend.add_scalar('map', 0.9, step=1) + self.aim_vis_backend.add_scalar('map', 0.95, step=2) def test_add_scalars(self): - aim_vis_backend = AimVisBackend() input_dict = {'map': 0.7, 'acc': 0.9} - aim_vis_backend.add_scalars(input_dict) + self.aim_vis_backend.add_scalars(input_dict) def test_close(self): - aim_vis_backend = AimVisBackend() - aim_vis_backend._init_env() - aim_vis_backend.close() + self.aim_vis_backend._init_env() + self.aim_vis_backend.close() From 1a491bdbf0ab221b9015a386d8c0073e0903546c Mon Sep 17 00:00:00 2001 From: Laurie Bax <47383333+lauriebax@users.noreply.github.com> Date: Tue, 28 Oct 2025 15:12:04 +0000 Subject: [PATCH 07/15] fix mocking to work with python 3.12 --- .../test_backends/test_petrel_backend.py | 38 ++++++++++++++----- 1 file changed, 28 insertions(+), 10 deletions(-) diff --git a/tests/test_fileio/test_backends/test_petrel_backend.py b/tests/test_fileio/test_backends/test_petrel_backend.py index 6f379c3f23..bfebd18305 100644 --- a/tests/test_fileio/test_backends/test_petrel_backend.py +++ b/tests/test_fileio/test_backends/test_petrel_backend.py @@ -58,8 +58,6 @@ def build_temporary_directory(): # without mock. import petrel_client # noqa: F401 except ImportError: - sys.modules['petrel_client'] = MagicMock() - sys.modules['petrel_client.client'] = MagicMock() class MockPetrelClient: @@ -95,16 +93,36 @@ def list(self, dir_path): elif osp.isdir(entry.path): yield entry.name + '/' + # Create mock modules that properly expose the MockPetrelClient + mock_petrel_client = MagicMock() + mock_client_module = MagicMock() + mock_client_module.Client = MockPetrelClient + mock_petrel_client.client = mock_client_module + + sys.modules['petrel_client'] = mock_petrel_client + sys.modules['petrel_client.client'] = mock_client_module + @contextmanager def delete_and_reset_method(obj, method): - method_obj = deepcopy(getattr(type(obj), method)) - try: - delattr(type(obj), method) - yield - finally: - setattr(type(obj), method, method_obj) - - @patch('petrel_client.client.Client', MockPetrelClient) + # Handle MagicMock objects differently in Python 3.12+ + if hasattr(obj, '_mock_methods') or str( + type(obj).__name__) == 'MagicMock': + # For MagicMock objects, work with the instance directly + method_obj = deepcopy(getattr(obj, method)) + try: + delattr(obj, method) + yield + finally: + setattr(obj, method, method_obj) + else: + # Original behavior for non-mock objects + method_obj = deepcopy(getattr(type(obj), method)) + try: + delattr(type(obj), method) + yield + finally: + setattr(type(obj), method, method_obj) + class TestPetrelBackend(TestCase): @classmethod From 20bfd6d4cb23ba4fd2ee936fc36ade191c394abe Mon Sep 17 00:00:00 2001 From: Laurie Bax <47383333+lauriebax@users.noreply.github.com> Date: Tue, 28 Oct 2025 15:17:38 +0000 Subject: [PATCH 08/15] fix petrel backend test --- tests/test_fileio/test_backends/test_petrel_backend.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_fileio/test_backends/test_petrel_backend.py b/tests/test_fileio/test_backends/test_petrel_backend.py index bfebd18305..c48a164625 100644 --- a/tests/test_fileio/test_backends/test_petrel_backend.py +++ b/tests/test_fileio/test_backends/test_petrel_backend.py @@ -57,7 +57,7 @@ def build_temporary_directory(): # If petrel_client is imported successfully, we can test PetrelBackend # without mock. import petrel_client # noqa: F401 -except ImportError: +except (ImportError, ModuleNotFoundError): class MockPetrelClient: From a8287761d3241486c6aeca7add86f77d4db5d741 Mon Sep 17 00:00:00 2001 From: Laurie Bax <47383333+lauriebax@users.noreply.github.com> Date: Tue, 28 Oct 2025 15:25:26 +0000 Subject: [PATCH 09/15] revert delete_and_reset_method in test --- .../test_backends/test_petrel_backend.py | 24 +++++-------------- 1 file changed, 6 insertions(+), 18 deletions(-) diff --git a/tests/test_fileio/test_backends/test_petrel_backend.py b/tests/test_fileio/test_backends/test_petrel_backend.py index c48a164625..5c539fc495 100644 --- a/tests/test_fileio/test_backends/test_petrel_backend.py +++ b/tests/test_fileio/test_backends/test_petrel_backend.py @@ -104,24 +104,12 @@ def list(self, dir_path): @contextmanager def delete_and_reset_method(obj, method): - # Handle MagicMock objects differently in Python 3.12+ - if hasattr(obj, '_mock_methods') or str( - type(obj).__name__) == 'MagicMock': - # For MagicMock objects, work with the instance directly - method_obj = deepcopy(getattr(obj, method)) - try: - delattr(obj, method) - yield - finally: - setattr(obj, method, method_obj) - else: - # Original behavior for non-mock objects - method_obj = deepcopy(getattr(type(obj), method)) - try: - delattr(type(obj), method) - yield - finally: - setattr(type(obj), method, method_obj) + method_obj = deepcopy(getattr(type(obj), method)) + try: + delattr(type(obj), method) + yield + finally: + setattr(type(obj), method, method_obj) class TestPetrelBackend(TestCase): From b7fea9b8439829ea7437734a051d6a24a9525d01 Mon Sep 17 00:00:00 2001 From: Laurie Bax <47383333+lauriebax@users.noreply.github.com> Date: Wed, 29 Oct 2025 08:12:30 +0000 Subject: [PATCH 10/15] revert reversion --- .../test_backends/test_petrel_backend.py | 21 +++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/tests/test_fileio/test_backends/test_petrel_backend.py b/tests/test_fileio/test_backends/test_petrel_backend.py index 5c539fc495..0cdad753b9 100644 --- a/tests/test_fileio/test_backends/test_petrel_backend.py +++ b/tests/test_fileio/test_backends/test_petrel_backend.py @@ -104,12 +104,21 @@ def list(self, dir_path): @contextmanager def delete_and_reset_method(obj, method): - method_obj = deepcopy(getattr(type(obj), method)) - try: - delattr(type(obj), method) - yield - finally: - setattr(type(obj), method, method_obj) + if hasattr(obj, '_mock_methods') or str( + type(obj).__name__) == 'MagicMock': + method_obj = deepcopy(getattr(obj, method)) + try: + delattr(obj, method) + yield + finally: + setattr(obj, method, method_obj) + else: + method_obj = deepcopy(getattr(type(obj), method)) + try: + delattr(type(obj), method) + yield + finally: + setattr(type(obj), method, method_obj) class TestPetrelBackend(TestCase): From 540d8926e16022d7b57255d4e6bc3cfa91117ac5 Mon Sep 17 00:00:00 2001 From: Laurie Bax <47383333+lauriebax@users.noreply.github.com> Date: Wed, 29 Oct 2025 08:27:25 +0000 Subject: [PATCH 11/15] fix test --- tests/test_fileio/test_backends/test_petrel_backend.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/test_fileio/test_backends/test_petrel_backend.py b/tests/test_fileio/test_backends/test_petrel_backend.py index 0cdad753b9..6258dd5c5f 100644 --- a/tests/test_fileio/test_backends/test_petrel_backend.py +++ b/tests/test_fileio/test_backends/test_petrel_backend.py @@ -88,9 +88,12 @@ def isdir(self): def list(self, dir_path): for entry in os.scandir(dir_path): - if not entry.name.startswith('.') and entry.is_file(): + print(f"{entry=}") + if entry.name.startswith('.'): + continue + if entry.is_file(): yield entry.name - elif osp.isdir(entry.path): + elif entry.is_dir(): yield entry.name + '/' # Create mock modules that properly expose the MockPetrelClient From 0719e1b52ad3d5d40c51c30f88650b9e92e8d10e Mon Sep 17 00:00:00 2001 From: Laurie Bax <47383333+lauriebax@users.noreply.github.com> Date: Wed, 29 Oct 2025 08:49:26 +0000 Subject: [PATCH 12/15] add print statement for debugging --- tests/test_fileio/test_backends/test_petrel_backend.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_fileio/test_backends/test_petrel_backend.py b/tests/test_fileio/test_backends/test_petrel_backend.py index 6258dd5c5f..eabd43237d 100644 --- a/tests/test_fileio/test_backends/test_petrel_backend.py +++ b/tests/test_fileio/test_backends/test_petrel_backend.py @@ -87,6 +87,7 @@ def isdir(self): pass def list(self, dir_path): + print('Using mock list') for entry in os.scandir(dir_path): print(f"{entry=}") if entry.name.startswith('.'): From c6a902d768a5b7555e632111edc01eb7d580d530 Mon Sep 17 00:00:00 2001 From: Laurie Bax <47383333+lauriebax@users.noreply.github.com> Date: Wed, 29 Oct 2025 08:50:44 +0000 Subject: [PATCH 13/15] run tests in verbose mode --- .github/workflows/pr_stage_test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr_stage_test.yml b/.github/workflows/pr_stage_test.yml index 382d878eed..47ae1936e9 100644 --- a/.github/workflows/pr_stage_test.yml +++ b/.github/workflows/pr_stage_test.yml @@ -48,7 +48,7 @@ jobs: uv pip install -r pyproject.toml --group test-lite - name: Run unit tests with coverage run: | - uv run pytest tests/test_config tests/test_registry tests/test_fileio tests/test_logging tests/test_utils --ignore=tests/test_utils/test_dl_utils + uv run pytest -v tests/test_config tests/test_registry tests/test_fileio tests/test_logging tests/test_utils --ignore=tests/test_utils/test_dl_utils build_cpu: runs-on: ubuntu-22.04 From e52c2617de700695104b2ca8f0967fb9f4660cd7 Mon Sep 17 00:00:00 2001 From: Laurie Bax <47383333+lauriebax@users.noreply.github.com> Date: Wed, 29 Oct 2025 09:03:51 +0000 Subject: [PATCH 14/15] rework way petrel client is mocked for test --- .../test_backends/test_petrel_backend.py | 141 ++++++++++-------- 1 file changed, 77 insertions(+), 64 deletions(-) diff --git a/tests/test_fileio/test_backends/test_petrel_backend.py b/tests/test_fileio/test_backends/test_petrel_backend.py index eabd43237d..050c1d6d6e 100644 --- a/tests/test_fileio/test_backends/test_petrel_backend.py +++ b/tests/test_fileio/test_backends/test_petrel_backend.py @@ -1,7 +1,6 @@ # Copyright (c) OpenMMLab. All rights reserved. import os import os.path as osp -import sys import tempfile from contextlib import contextmanager from copy import deepcopy @@ -49,80 +48,75 @@ def build_temporary_directory(): yield tmp_dir +# Check if petrel_client is available +PETREL_CLIENT_AVAILABLE = False try: - # Other unit tests may mock these modules so we need to pop them first. - sys.modules.pop('petrel_client', None) - sys.modules.pop('petrel_client.client', None) - - # If petrel_client is imported successfully, we can test PetrelBackend - # without mock. import petrel_client # noqa: F401 + PETREL_CLIENT_AVAILABLE = True except (ImportError, ModuleNotFoundError): + PETREL_CLIENT_AVAILABLE = False - class MockPetrelClient: - def __init__(self, - enable_mc=True, - enable_multi_cluster=False, - conf_path=None): - self.enable_mc = enable_mc - self.enable_multi_cluster = enable_multi_cluster - self.conf_path = conf_path +class MockPetrelClient: + """Mock PetrelClient for testing when petrel_client is not available.""" - def Get(self, filepath): - with open(filepath, 'rb') as f: - content = f.read() - return content + def __init__(self, + enable_mc=True, + enable_multi_cluster=False, + conf_path=None): + self.enable_mc = enable_mc + self.enable_multi_cluster = enable_multi_cluster + self.conf_path = conf_path - def put(self): - pass + def Get(self, filepath): + with open(filepath, 'rb') as f: + content = f.read() + return content - def delete(self): - pass + def put(self, filepath, content): + pass - def contains(self): - pass + def delete(self, filepath): + pass - def isdir(self): - pass + def contains(self, filepath): + pass + + def isdir(self, filepath): + pass + + def list(self, dir_path): + print('Using mock list') + for entry in os.scandir(dir_path): + if entry.name.startswith('.'): + continue + if entry.is_file(): + yield entry.name + elif entry.is_dir(): + yield entry.name + '/' - def list(self, dir_path): - print('Using mock list') - for entry in os.scandir(dir_path): - print(f"{entry=}") - if entry.name.startswith('.'): - continue - if entry.is_file(): - yield entry.name - elif entry.is_dir(): - yield entry.name + '/' - - # Create mock modules that properly expose the MockPetrelClient - mock_petrel_client = MagicMock() - mock_client_module = MagicMock() - mock_client_module.Client = MockPetrelClient - mock_petrel_client.client = mock_client_module - - sys.modules['petrel_client'] = mock_petrel_client - sys.modules['petrel_client.client'] = mock_client_module - - @contextmanager - def delete_and_reset_method(obj, method): - if hasattr(obj, '_mock_methods') or str( - type(obj).__name__) == 'MagicMock': - method_obj = deepcopy(getattr(obj, method)) - try: - delattr(obj, method) - yield - finally: - setattr(obj, method, method_obj) - else: - method_obj = deepcopy(getattr(type(obj), method)) - try: - delattr(type(obj), method) - yield - finally: - setattr(type(obj), method, method_obj) + +@contextmanager +def delete_and_reset_method(obj, method): + if hasattr(obj, '_mock_methods') or str(type(obj).__name__) == 'MagicMock': + method_obj = deepcopy(getattr(obj, method)) + try: + delattr(obj, method) + yield + finally: + setattr(obj, method, method_obj) + else: + method_obj = deepcopy(getattr(type(obj), method)) + try: + delattr(type(obj), method) + yield + finally: + setattr(type(obj), method, method_obj) + + +if not PETREL_CLIENT_AVAILABLE: + # Define the test class that uses mocking when + # petrel_client is not available class TestPetrelBackend(TestCase): @@ -137,6 +131,24 @@ def setUpClass(cls): cls.expected_dir = 's3://user/data' cls.expected_path = f'{cls.expected_dir}/test.jpg' + def setUp(self): + # Mock petrel_client for each test + self.mock_petrel_client = MagicMock() + self.mock_client_module = MagicMock() + self.mock_client_module.Client = MockPetrelClient + self.mock_petrel_client.client = self.mock_client_module + + self.patcher_petrel = patch.dict( + 'sys.modules', { + 'petrel_client': self.mock_petrel_client, + 'petrel_client.client': self.mock_client_module + }) + self.patcher_petrel.start() + + def tearDown(self): + # Clean up the mock + self.patcher_petrel.stop() + def test_name(self): backend = PetrelBackend() self.assertEqual(backend.name, 'PetrelBackend') @@ -582,6 +594,7 @@ def test_generate_presigned_url(self): pass else: + # Define the test class that uses real petrel_client when available class TestPetrelBackend(TestCase): # type: ignore From 599adc888db30aa4e108919e215169ae8d76d971 Mon Sep 17 00:00:00 2001 From: Laurie Bax <47383333+lauriebax@users.noreply.github.com> Date: Wed, 29 Oct 2025 09:38:22 +0000 Subject: [PATCH 15/15] improve patching of petrel backend in other tests --- .github/workflows/pr_stage_test.yml | 4 +--- .../test_backends/test_petrel_backend.py | 1 - tests/test_fileio/test_fileclient.py | 21 ++++++++++++++++--- 3 files changed, 19 insertions(+), 7 deletions(-) diff --git a/.github/workflows/pr_stage_test.yml b/.github/workflows/pr_stage_test.yml index 47ae1936e9..021e892e2e 100644 --- a/.github/workflows/pr_stage_test.yml +++ b/.github/workflows/pr_stage_test.yml @@ -48,7 +48,7 @@ jobs: uv pip install -r pyproject.toml --group test-lite - name: Run unit tests with coverage run: | - uv run pytest -v tests/test_config tests/test_registry tests/test_fileio tests/test_logging tests/test_utils --ignore=tests/test_utils/test_dl_utils + uv run pytest tests/test_config tests/test_registry tests/test_fileio tests/test_logging tests/test_utils --ignore=tests/test_utils/test_dl_utils build_cpu: runs-on: ubuntu-22.04 @@ -104,8 +104,6 @@ jobs: image: 'nvidia/cuda:12.9.1-cudnn-runtime-ubuntu22.04' container: image: ${{ matrix.image }} - permissions: - pull-requests: write steps: - name: Show disk usage run: df -h diff --git a/tests/test_fileio/test_backends/test_petrel_backend.py b/tests/test_fileio/test_backends/test_petrel_backend.py index 050c1d6d6e..0dbad20853 100644 --- a/tests/test_fileio/test_backends/test_petrel_backend.py +++ b/tests/test_fileio/test_backends/test_petrel_backend.py @@ -86,7 +86,6 @@ def isdir(self, filepath): pass def list(self, dir_path): - print('Using mock list') for entry in os.scandir(dir_path): if entry.name.startswith('.'): continue diff --git a/tests/test_fileio/test_fileclient.py b/tests/test_fileio/test_fileclient.py index 5f71379c23..af7b7b47e5 100644 --- a/tests/test_fileio/test_fileclient.py +++ b/tests/test_fileio/test_fileclient.py @@ -16,8 +16,6 @@ from mmengine.utils import has_method sys.modules['ceph'] = MagicMock() -sys.modules['petrel_client'] = MagicMock() -sys.modules['petrel_client.client'] = MagicMock() sys.modules['mc'] = MagicMock() @@ -137,6 +135,24 @@ def setup_class(cls): cls.img_shape = (300, 400, 3) cls.text_path = cls.test_data_dir / 'filelist.txt' + def setup_method(self): + # Mock petrel_client for each test + self.mock_petrel_client = MagicMock() + self.mock_client_module = MagicMock() + self.mock_client_module.Client = MockPetrelClient + self.mock_petrel_client.client = self.mock_client_module + + self.patcher_petrel = patch.dict( + 'sys.modules', { + 'petrel_client': self.mock_petrel_client, + 'petrel_client.client': self.mock_client_module + }) + self.patcher_petrel.start() + + def teardown_method(self): + # Clean up the mock + self.patcher_petrel.stop() + def test_error(self): with pytest.raises(ValueError): FileClient('hadoop') @@ -289,7 +305,6 @@ def test_disk_backend(self): osp.join('dir2', 'img.jpg'), 'text1.txt', 'text2.txt' } - @patch('petrel_client.client.Client', MockPetrelClient) @pytest.mark.parametrize('backend,prefix', [('petrel', None), (None, 's3')]) def test_petrel_backend(self, backend, prefix):