From 0cb683a3ee675021f5ab432d09494b2500b4e05e Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Mon, 7 Aug 2023 14:06:19 +0200 Subject: [PATCH 1/5] infra --- CI/physmon/tests/conftest.py | 163 ++++++++++++++++++ .../tests/test_truth_tracking_kalman.py | 32 ++++ Examples/Python/tests/conftest.py | 42 +---- conftest.py | 44 +++++ pytest.ini | 2 + 5 files changed, 246 insertions(+), 37 deletions(-) create mode 100644 CI/physmon/tests/conftest.py create mode 100644 CI/physmon/tests/test_truth_tracking_kalman.py create mode 100644 conftest.py diff --git a/CI/physmon/tests/conftest.py b/CI/physmon/tests/conftest.py new file mode 100644 index 00000000000..7a9ad814204 --- /dev/null +++ b/CI/physmon/tests/conftest.py @@ -0,0 +1,163 @@ +from pathlib import Path +import collections +from typing import List, IO, Tuple +import threading +import csv +from datetime import datetime +import time + +import pytest +import psutil + +import acts +import acts.examples +from common import getOpenDataDetectorDirectory +from acts.examples.odd import getOpenDataDetector + + +@pytest.fixture() +def output_path(request): + path: Path = request.config.getoption("--physmon-output-path").resolve() + path.mkdir(parents=True, exist_ok=True) + + return path + + +PhysmonSetup = collections.namedtuple( + "Setup", + [ + "detector", + "trackingGeometry", + "decorators", + "field", + "digiConfig", + "geoSel", + ], +) + + +@pytest.fixture(scope="session") +def setup(): + u = acts.UnitConstants + srcdir = Path(__file__).resolve().parent.parent.parent.parent + + matDeco = acts.IMaterialDecorator.fromFile( + srcdir / "thirdparty/OpenDataDetector/data/odd-material-maps.root", + level=acts.logging.INFO, + ) + + detector, trackingGeometry, decorators = getOpenDataDetector( + getOpenDataDetectorDirectory(), matDeco + ) + setup = PhysmonSetup( + detector=detector, + trackingGeometry=trackingGeometry, + decorators=decorators, + digiConfig=srcdir + / "thirdparty/OpenDataDetector/config/odd-digi-smearing-config.json", + geoSel=srcdir / "thirdparty/OpenDataDetector/config/odd-seeding-config.json", + field=acts.ConstantBField(acts.Vector3(0, 0, 2 * u.T)), + ) + return setup + + +# @TODO: try using spyral directly + + +class Monitor: + interval: float + + max_rss: float = 0 + max_vms: float = 0 + + terminate: bool = False + + exception = None + + def __init__( + self, + output: IO[str], + interval: float = 0.5, + ): + self.interval = interval + self.writer = csv.writer(output) + self.writer.writerow(("time", "rss", "vms")) + + + self.time: List[float] = [0] + self.rss: List[float] = [0] + self.vms: List[float] = [0] + + @staticmethod + def _get_memory(p: psutil.Process) -> Tuple[float, float]: + rss = p.memory_info().rss + vms = p.memory_info().vms + for subp in p.children(recursive=True): + try: + rss += subp.memory_info().rss + vms += subp.memory_info().vms + except (psutil.NoSuchProcess, psutil.AccessDenied): + pass + return rss, vms + + def run(self, p: psutil.Process): + + rss_offset, vms_offset = self._get_memory(p) + self.writer.writerow((0, 0, 0)) + + try: + start = datetime.now() + while p.is_running() and p.status() in ( + psutil.STATUS_RUNNING, + psutil.STATUS_SLEEPING, + ): + if self.terminate: + return + + delta = (datetime.now() - start).total_seconds() + + rss, vms = self._get_memory(p) + rss -= rss_offset + vms -= vms_offset + + self.rss.append(rss / 1e6) + self.vms.append(vms / 1e6) + self.time.append(delta) + self.max_rss = max(rss, self.max_rss) + self.max_vms = max(vms, self.max_vms) + + self.writer.writerow((delta, rss, vms)) + + time.sleep(self.interval) + except (psutil.NoSuchProcess, psutil.AccessDenied): + return + except Exception as e: + self.exception = e + raise e + + +@pytest.fixture(autouse=True) +def monitor(output_path: Path, request, capsys): + import psutil + + p = psutil.Process() + + memory = output_path / "memory" + memory.mkdir(exist_ok=True) + with (memory / f"mem_{request.node.name}.csv").open("w") as fh: + mon = Monitor(output=fh, interval=0.1) + + t = threading.Thread(target=mon.run, args=(p,)) + t.start() + + yield + + mon.terminate = True + t.join() + + # @TODO: Add plotting + + with capsys.disabled(): + print("MONITORING") + + diff --git a/CI/physmon/tests/test_truth_tracking_kalman.py b/CI/physmon/tests/test_truth_tracking_kalman.py new file mode 100644 index 00000000000..a86f6ad0de5 --- /dev/null +++ b/CI/physmon/tests/test_truth_tracking_kalman.py @@ -0,0 +1,32 @@ +from pathlib import Path + +import acts.examples + +from truth_tracking_kalman import runTruthTrackingKalman + + +def test_truth_tracking_kalman(output_path: Path, tmp_path: Path, setup): + print(setup) + s = acts.examples.Sequencer( + events=100, + numThreads=-1, + logLevel=acts.logging.INFO, + fpeMasks=acts.examples.Sequencer.FpeMask.fromFile( + Path(__file__).parent.parent / "fpe_masks.yml" + ), + ) + + runTruthTrackingKalman( + setup.trackingGeometry, + setup.field, + digiConfigFile=setup.digiConfig, + outputDir=tmp_path, + s=s, + ) + + s.run() + del s + + perf_file = tmp_path / "performance_track_fitter.root" + assert perf_file.exists(), "Performance file not found" + # shutil.copy(perf_file, setup.outdir / "performance_truth_tracking.root") diff --git a/Examples/Python/tests/conftest.py b/Examples/Python/tests/conftest.py index 3daec9c6a30..31b04f61e5c 100644 --- a/Examples/Python/tests/conftest.py +++ b/Examples/Python/tests/conftest.py @@ -1,60 +1,28 @@ -import multiprocessing from pathlib import Path -import sys -import os -import tempfile -import shutil from typing import Dict -import warnings -import pytest_check as check +import shutil +import tempfile +import os +import multiprocessing from collections import namedtuple - sys.path = [ str(Path(__file__).parent.parent.parent.parent / "Examples/Scripts/Python/"), str(Path(__file__).parent), ] + sys.path +import pytest import helpers import helpers.hash_root -import pytest - import acts import acts.examples from acts.examples.odd import getOpenDataDetector, getOpenDataDetectorDirectory from acts.examples.simulation import addParticleGun, EtaConfig, ParticleConfig -try: - import ROOT - - ROOT.gSystem.ResetSignals() -except ImportError: - pass - -try: - if acts.logging.getFailureThreshold() != acts.logging.WARNING: - acts.logging.setFailureThreshold(acts.logging.WARNING) -except RuntimeError: - # Repackage with different error string - errtype = ( - "negative" - if acts.logging.getFailureThreshold() < acts.logging.WARNING - else "positive" - ) - warnings.warn( - "Runtime log failure threshold could not be set. " - "Compile-time value is probably set via CMake, i.e. " - f"`ACTS_LOG_FAILURE_THRESHOLD={acts.logging.getFailureThreshold().name}` is set, " - "or `ACTS_ENABLE_LOG_FAILURE_THRESHOLD=OFF`. " - f"The pytest test-suite can produce false-{errtype} results in this configuration" - ) - - u = acts.UnitConstants - class RootHashAssertionError(AssertionError): def __init__( self, file: Path, key: str, exp_hash: str, act_hash: str, *args, **kwargs diff --git a/conftest.py b/conftest.py new file mode 100644 index 00000000000..da66f7d3c4e --- /dev/null +++ b/conftest.py @@ -0,0 +1,44 @@ +from pathlib import Path +import sys +import warnings +import pytest_check as check + + +sys.path += [ + str(Path(__file__).parent / "Examples/Scripts/Python"), + str(Path(__file__).parent / "Examples/Python/tests" ), +] + + +import pytest + +import acts + +try: + import ROOT + + ROOT.gSystem.ResetSignals() +except ImportError: + pass + +try: + if acts.logging.getFailureThreshold() != acts.logging.WARNING: + acts.logging.setFailureThreshold(acts.logging.WARNING) +except RuntimeError: + # Repackage with different error string + errtype = ( + "negative" + if acts.logging.getFailureThreshold() < acts.logging.WARNING + else "positive" + ) + warnings.warn( + "Runtime log failure threshold could not be set. " + "Compile-time value is probably set via CMake, i.e. " + f"`ACTS_LOG_FAILURE_THRESHOLD={acts.logging.getFailureThreshold().name}` is set, " + "or `ACTS_ENABLE_LOG_FAILURE_THRESHOLD=OFF`. " + f"The pytest test-suite can produce false-{errtype} results in this configuration" + ) + + +def pytest_addoption(parser): + parser.addoption("--physmon-output-path", action="store", default=Path.cwd()/"physmon", type=Path) diff --git a/pytest.ini b/pytest.ini index 6e1657f482e..c581e301e68 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,6 +1,8 @@ [pytest] testpaths = Examples/Python/tests + CI/physmon/tests +addopts = --import-mode=importlib norecursedirs=Examples/Python/tests/helpers markers = csv From 6b45136fb2bd7a58688225b514e4ec4f545b2cf8 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Tue, 8 Aug 2023 13:55:13 +0200 Subject: [PATCH 2/5] wip --- CI/physmon/tests/conftest.py | 168 +++++++++++--- CI/physmon/tests/test_ckf_tracking.py | 210 ++++++++++++++++++ .../tests/test_truth_tracking_kalman.py | 20 +- conftest.py | 11 +- 4 files changed, 367 insertions(+), 42 deletions(-) create mode 100644 CI/physmon/tests/test_ckf_tracking.py diff --git a/CI/physmon/tests/conftest.py b/CI/physmon/tests/conftest.py index 7a9ad814204..ab8a8ccc0c9 100644 --- a/CI/physmon/tests/conftest.py +++ b/CI/physmon/tests/conftest.py @@ -1,13 +1,15 @@ from pathlib import Path -import collections -from typing import List, IO, Tuple +import re +from typing import List, IO, Tuple, Optional import threading import csv from datetime import datetime import time +import shutil import pytest import psutil +from pytest_check import check import acts import acts.examples @@ -15,7 +17,7 @@ from acts.examples.odd import getOpenDataDetector -@pytest.fixture() +@pytest.fixture(scope="session") def output_path(request): path: Path = request.config.getoption("--physmon-output-path").resolve() path.mkdir(parents=True, exist_ok=True) @@ -23,22 +25,119 @@ def output_path(request): return path -PhysmonSetup = collections.namedtuple( - "Setup", - [ - "detector", - "trackingGeometry", - "decorators", - "field", - "digiConfig", - "geoSel", - ], -) +class Physmon: + detector: "acts.examples.dd4hep.DD4hepDetector" + trackingGeometry: acts.TrackingGeometry + decorators: List[acts.IMaterialDecorator] + field: acts.MagneticFieldProvider + digiConfig: Path + geoSel: Path + output_path: Path + tmp_path: Path + name: str + + def __init__( + self, + detector, + trackingGeometry, + decorators, + field, + digiConfig, + geoSel, + output_path, + tmp_path, + name, + ): + self.detector = detector + self.trackingGeometry = trackingGeometry + self.decorators = decorators + self.field = field + self.digiConfig = digiConfig + self.geoSel = geoSel + self.output_path = output_path + self.tmp_path = tmp_path + self.name = name + + def add_output_file(self, filename: str, rename: Optional[str] = None): + __tracebackhide__ = True + tmp = self.tmp_path / filename + assert tmp.exists(), f"Output file {tmp} does not exist" + outname = rename if rename else filename + shutil.copy(tmp, self.output_path / outname) + + def histogram_comparison( + self, filename: Path, title: str, config_path: Optional[Path] = None + ): + __tracebackhide__ = True + monitored = self.output_path / filename + reference = Path(__file__).parent.parent / "reference" / filename + + assert monitored.exists(), f"Output file {monitored} does not exist" + assert reference.exists(), f"Reference file {reference} does not exist" + + from histcmp.console import Console + from histcmp.report import make_report + from histcmp.checks import Status + from histcmp.config import Config + from histcmp.github import is_github_actions, github_actions_marker + from histcmp.cli import print_summary + + from histcmp.compare import compare, Comparison + + from rich.panel import Panel + from rich.console import Group + from rich.pretty import Pretty + + import yaml + + console = Console() + + console.print( + Panel( + Group(f"Monitored: {monitored}", f"Reference: {reference}"), + title="Comparing files:", + ) + ) + + if config_path is None: + config = Config.default() + else: + with config_path.open() as fh: + config = Config(**yaml.safe_load(fh)) + + console.print(Panel(Pretty(config), title="Configuration")) + + # filter_path = Path(_filter) + # if filter_path.exists(): + # with filter_path.open() as fh: + # filters = fh.read().strip().split("\n") + # else: + # filters = [_filter] + filters = [] + comparison = compare( + config, monitored, reference, filters=filters, console=console + ) + + comparison.label_monitored = "monitored" + comparison.label_reference = "reference" + comparison.title = title + + status = print_summary(comparison, console) + + plots = self.output_path / "plots" + plots.mkdir(exist_ok=True, parents=True) + report_file = self.output_path / f"{self.name}.html" + make_report(comparison, report_file, console, plots, format="pdf") + + for item in comparison.items: + msg = f"{item.key} failures: " + ", ".join( + [c.name for c in item.checks if c.status == Status.FAILURE] + ) + check.equal(item.status, Status.SUCCESS, msg=msg) @pytest.fixture(scope="session") -def setup(): - u = acts.UnitConstants +def _physmon_prereqs(): srcdir = Path(__file__).resolve().parent.parent.parent.parent matDeco = acts.IMaterialDecorator.fromFile( @@ -49,7 +148,19 @@ def setup(): detector, trackingGeometry, decorators = getOpenDataDetector( getOpenDataDetectorDirectory(), matDeco ) - setup = PhysmonSetup( + + return srcdir, detector, trackingGeometry, decorators + + +@pytest.fixture() +def physmon(output_path, _physmon_prereqs, tmp_path, request): + u = acts.UnitConstants + + srcdir, detector, trackingGeometry, decorators = _physmon_prereqs + + name = re.sub(r"^test_", "", request.node.name) + + setup = Physmon( detector=detector, trackingGeometry=trackingGeometry, decorators=decorators, @@ -57,6 +168,9 @@ def setup(): / "thirdparty/OpenDataDetector/config/odd-digi-smearing-config.json", geoSel=srcdir / "thirdparty/OpenDataDetector/config/odd-seeding-config.json", field=acts.ConstantBField(acts.Vector3(0, 0, 2 * u.T)), + output_path=output_path, + tmp_path=tmp_path, + name=name, ) return setup @@ -83,7 +197,6 @@ def __init__( self.writer = csv.writer(output) self.writer.writerow(("time", "rss", "vms")) - self.time: List[float] = [0] self.rss: List[float] = [0] self.vms: List[float] = [0] @@ -101,10 +214,6 @@ def _get_memory(p: psutil.Process) -> Tuple[float, float]: return rss, vms def run(self, p: psutil.Process): - - rss_offset, vms_offset = self._get_memory(p) - self.writer.writerow((0, 0, 0)) - try: start = datetime.now() while p.is_running() and p.status() in ( @@ -117,8 +226,6 @@ def run(self, p: psutil.Process): delta = (datetime.now() - start).total_seconds() rss, vms = self._get_memory(p) - rss -= rss_offset - vms -= vms_offset self.rss.append(rss / 1e6) self.vms.append(vms / 1e6) @@ -137,14 +244,15 @@ def run(self, p: psutil.Process): @pytest.fixture(autouse=True) -def monitor(output_path: Path, request, capsys): +def monitor(physmon: Physmon, request, capsys): import psutil p = psutil.Process() - memory = output_path / "memory" + memory = physmon.output_path / "memory" memory.mkdir(exist_ok=True) - with (memory / f"mem_{request.node.name}.csv").open("w") as fh: + name = re.sub(r"^test_", "", request.node.name) + with (memory / f"mem_{name}.csv").open("w") as fh: mon = Monitor(output=fh, interval=0.1) t = threading.Thread(target=mon.run, args=(p,)) @@ -157,7 +265,5 @@ def monitor(output_path: Path, request, capsys): # @TODO: Add plotting - with capsys.disabled(): - print("MONITORING") - - + # with capsys.disabled(): + # print("MONITORING") diff --git a/CI/physmon/tests/test_ckf_tracking.py b/CI/physmon/tests/test_ckf_tracking.py new file mode 100644 index 00000000000..4cbcd68f845 --- /dev/null +++ b/CI/physmon/tests/test_ckf_tracking.py @@ -0,0 +1,210 @@ +from pathlib import Path + +import pytest + +import acts.examples + +from acts.examples.simulation import ( + addParticleGun, + MomentumConfig, + EtaConfig, + PhiConfig, + ParticleConfig, + addFatras, + addDigitization, +) + +from acts.examples.reconstruction import ( + addSeeding, + TruthSeedRanges, + ParticleSmearingSigmas, + SeedFinderConfigArg, + SeedFinderOptionsArg, + SeedingAlgorithm, + TruthEstimatedSeedingAlgorithmConfigArg, + addCKFTracks, + addAmbiguityResolution, + AmbiguityResolutionConfig, + addVertexFitting, + VertexFinder, + TrackSelectorConfig, +) + +from helpers import failure_threshold + +u = acts.UnitConstants + + +# def run_ckf_tracking(truthSmearedSeeded, truthEstimatedSeeded, label): +# with tempfile.TemporaryDirectory() as temp: + + +# for truthSmearedSeeded, truthEstimatedSeeded, label in [ +# (True, False, "truth_smeared"), # if first is true, second is ignored +# (False, True, "truth_estimated"), +# (False, False, "seeded"), +# (False, False, "orthogonal"), +# ]: +# run_ckf_tracking(truthSmearedSeeded, truthEstimatedSeeded, label) + + +@pytest.mark.parametrize( + "seeding_algorithm", + [ + SeedingAlgorithm.Default, + SeedingAlgorithm.TruthSmeared, + SeedingAlgorithm.TruthEstimated, + SeedingAlgorithm.Orthogonal, + ], +) +def test_ckf_tracking(seeding_algorithm, physmon: "Physmon"): + s = acts.examples.Sequencer( + events=10, + numThreads=-1, + logLevel=acts.logging.INFO, + fpeMasks=acts.examples.Sequencer.FpeMask.fromFile( + Path(__file__).parent.parent / "fpe_masks.yml" + ), + ) + + for d in physmon.decorators: + s.addContextDecorator(d) + + rnd = acts.examples.RandomNumbers(seed=42) + + addParticleGun( + s, + MomentumConfig(1.0 * u.GeV, 10.0 * u.GeV, transverse=True), + EtaConfig(-3.0, 3.0), + PhiConfig(0.0, 360.0 * u.degree), + ParticleConfig(4, acts.PdgParticle.eMuon, randomizeCharge=True), + vtxGen=acts.examples.GaussianVertexGenerator( + mean=acts.Vector4(0, 0, 0, 0), + stddev=acts.Vector4(0.0125 * u.mm, 0.0125 * u.mm, 55.5 * u.mm, 1.0 * u.ns), + ), + multiplicity=50, + rnd=rnd, + ) + + addFatras( + s, + physmon.trackingGeometry, + physmon.field, + enableInteractions=True, + rnd=rnd, + ) + + addDigitization( + s, + physmon.trackingGeometry, + physmon.field, + digiConfigFile=physmon.digiConfig, + rnd=rnd, + ) + + addSeeding( + s, + physmon.trackingGeometry, + physmon.field, + TruthSeedRanges(pt=(500 * u.MeV, None), nHits=(9, None)), + ParticleSmearingSigmas(pRel=0.01), # only used by SeedingAlgorithm.TruthSmeared + SeedFinderConfigArg( + r=(33 * u.mm, 200 * u.mm), + deltaR=(1 * u.mm, 60 * u.mm), + collisionRegion=(-250 * u.mm, 250 * u.mm), + z=(-2000 * u.mm, 2000 * u.mm), + maxSeedsPerSpM=1, + sigmaScattering=5, + radLengthPerSeed=0.1, + minPt=500 * u.MeV, + impactMax=3 * u.mm, + ), + SeedFinderOptionsArg(bFieldInZ=2 * u.T), + TruthEstimatedSeedingAlgorithmConfigArg(deltaR=(10.0 * u.mm, None)), + seedingAlgorithm=seeding_algorithm, + geoSelectionConfigFile=physmon.geoSel, + rnd=rnd, # only used by SeedingAlgorithm.TruthSmeared + outputDirRoot=physmon.tmp_path, + ) + + addCKFTracks( + s, + physmon.trackingGeometry, + physmon.field, + TrackSelectorConfig( + pt=(500 * u.MeV, None), + loc0=(-4.0 * u.mm, 4.0 * u.mm), + nMeasurementsMin=6, + ), + outputDirRoot=physmon.tmp_path, + ) + + stems = [ + "performance_ckf", + "tracksummary_ckf", + "performance_ivf", + "performance_amvf", + ] + + associatedParticles = "particles_input" + if seeding_algorithm in [SeedingAlgorithm.Default, SeedingAlgorithm.Orthogonal]: + addAmbiguityResolution( + s, + AmbiguityResolutionConfig(maximumSharedHits=3), + outputDirRoot=physmon.tmp_path, + ) + + associatedParticles = None + + stems += ["performance_ambi"] + + addVertexFitting( + s, + physmon.field, + associatedParticles=associatedParticles, + outputProtoVertices="ivf_protovertices", + outputVertices="ivf_fittedVertices", + vertexFinder=VertexFinder.Iterative, + outputDirRoot=physmon.tmp_path / "ivf", + ) + + addVertexFitting( + s, + physmon.field, + associatedParticles=associatedParticles, + outputProtoVertices="amvf_protovertices", + outputVertices="amvf_fittedVertices", + vertexFinder=VertexFinder.AMVF, + outputDirRoot=physmon.tmp_path / "amvf", + ) + + with failure_threshold(acts.logging.FATAL): + s.run() + del s + + for vertexing in ["ivf", "amvf"]: + physmon.add_output_file( + f"{vertexing}/performance_vertexing.root", + rename=f"performance_{vertexing}.root", + ) + + if seeding_algorithm in [ + SeedingAlgorithm.Default, + SeedingAlgorithm.Orthogonal, + SeedingAlgorithm.TruthEstimated, + ]: + stems += ["performance_seeding"] + + for stem in stems: + physmon.add_output_file(f"{stem}.root", rename=f"{stem}_{physmon.name}.root") + + # ] + ( + # ["performance_seeding", "performance_ambi"] + # if label in ["seeded", "orthogonal"] + # else ["performance_seeding"] + # if label == "truth_estimated" + # else [] + # ): + # perf_file = physmon.tmp_path / f"{stem}.root" + # assert perf_file.exists(), "Performance file not found" + # shutil.copy(perf_file, physmon.outdir / f"{stem}_{label}.root") diff --git a/CI/physmon/tests/test_truth_tracking_kalman.py b/CI/physmon/tests/test_truth_tracking_kalman.py index a86f6ad0de5..553fddb1616 100644 --- a/CI/physmon/tests/test_truth_tracking_kalman.py +++ b/CI/physmon/tests/test_truth_tracking_kalman.py @@ -5,8 +5,7 @@ from truth_tracking_kalman import runTruthTrackingKalman -def test_truth_tracking_kalman(output_path: Path, tmp_path: Path, setup): - print(setup) +def test_truth_tracking_kalman(physmon: "Physmon"): s = acts.examples.Sequencer( events=100, numThreads=-1, @@ -17,16 +16,19 @@ def test_truth_tracking_kalman(output_path: Path, tmp_path: Path, setup): ) runTruthTrackingKalman( - setup.trackingGeometry, - setup.field, - digiConfigFile=setup.digiConfig, - outputDir=tmp_path, + physmon.trackingGeometry, + physmon.field, + digiConfigFile=physmon.digiConfig, + outputDir=physmon.tmp_path, s=s, ) s.run() del s - perf_file = tmp_path / "performance_track_fitter.root" - assert perf_file.exists(), "Performance file not found" - # shutil.copy(perf_file, setup.outdir / "performance_truth_tracking.root") + physmon.add_output_file( + "performance_track_fitter.root", rename="performance_truth_tracking.root" + ) + physmon.histogram_comparison( + "performance_truth_tracking.root", title="Truth tracking KF" + ) diff --git a/conftest.py b/conftest.py index da66f7d3c4e..00764d2ad07 100644 --- a/conftest.py +++ b/conftest.py @@ -5,8 +5,9 @@ sys.path += [ + str(Path(__file__).parent / "Examples/Scripts"), str(Path(__file__).parent / "Examples/Scripts/Python"), - str(Path(__file__).parent / "Examples/Python/tests" ), + str(Path(__file__).parent / "Examples/Python/tests"), ] @@ -18,6 +19,7 @@ import ROOT ROOT.gSystem.ResetSignals() + ROOT.gROOT.SetBatch(ROOT.kTRUE) except ImportError: pass @@ -41,4 +43,9 @@ def pytest_addoption(parser): - parser.addoption("--physmon-output-path", action="store", default=Path.cwd()/"physmon", type=Path) + parser.addoption( + "--physmon-output-path", + action="store", + default=Path.cwd() / "physmon", + type=Path, + ) From ba43634a1270535103016250624afc8b8287d66b Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Wed, 4 Oct 2023 17:38:01 +0200 Subject: [PATCH 3/5] histogram comparisons working there's a non deterministic failure from inside histcmp annoyingly --- CI/physmon/tests/conftest.py | 56 ++++++++++++++++++++++----- CI/physmon/tests/test_ckf_tracking.py | 12 ++++-- conftest.py | 8 ++++ 3 files changed, 62 insertions(+), 14 deletions(-) diff --git a/CI/physmon/tests/conftest.py b/CI/physmon/tests/conftest.py index ab8a8ccc0c9..dfac9e44f53 100644 --- a/CI/physmon/tests/conftest.py +++ b/CI/physmon/tests/conftest.py @@ -25,6 +25,13 @@ def output_path(request): return path +@pytest.fixture(scope="session") +def reference_path(request): + path: Path = request.config.getoption("--physmon-reference-path").resolve() + + return path + + class Physmon: detector: "acts.examples.dd4hep.DD4hepDetector" trackingGeometry: acts.TrackingGeometry @@ -33,6 +40,8 @@ class Physmon: digiConfig: Path geoSel: Path output_path: Path + reference_path: Path + update_references: bool tmp_path: Path name: str @@ -45,6 +54,8 @@ def __init__( digiConfig, geoSel, output_path, + reference_path, + update_references, tmp_path, name, ): @@ -55,24 +66,40 @@ def __init__( self.digiConfig = digiConfig self.geoSel = geoSel self.output_path = output_path + self.reference_path = reference_path + self.update_references = update_references self.tmp_path = tmp_path self.name = name + self.test_output_path.mkdir(exist_ok=True, parents=True) + self.test_reference_path.mkdir(exist_ok=True, parents=True) + + @property + def test_output_path(self) -> Path: + return self.output_path / self.name + + @property + def test_reference_path(self) -> Path: + return self.reference_path / self.name + def add_output_file(self, filename: str, rename: Optional[str] = None): __tracebackhide__ = True tmp = self.tmp_path / filename assert tmp.exists(), f"Output file {tmp} does not exist" outname = rename if rename else filename - shutil.copy(tmp, self.output_path / outname) + shutil.copy(tmp, self.test_output_path / outname) def histogram_comparison( self, filename: Path, title: str, config_path: Optional[Path] = None ): __tracebackhide__ = True - monitored = self.output_path / filename - reference = Path(__file__).parent.parent / "reference" / filename + monitored = self.test_output_path / filename + reference = self.test_reference_path / filename assert monitored.exists(), f"Output file {monitored} does not exist" + + if self.update_references: + shutil.copy(monitored, reference) assert reference.exists(), f"Reference file {reference} does not exist" from histcmp.console import Console @@ -124,9 +151,9 @@ def histogram_comparison( status = print_summary(comparison, console) - plots = self.output_path / "plots" + plots = self.test_output_path / "plots" plots.mkdir(exist_ok=True, parents=True) - report_file = self.output_path / f"{self.name}.html" + report_file = self.test_output_path / f"{monitored.stem}.html" make_report(comparison, report_file, console, plots, format="pdf") for item in comparison.items: @@ -152,14 +179,21 @@ def _physmon_prereqs(): return srcdir, detector, trackingGeometry, decorators +def _sanitize_test_name(name: str): + name = re.sub(r"^test_", "", name) + name = re.sub(r"\]$", "", name) + name = re.sub(r"[\[\]]", "_", name) + return name + + @pytest.fixture() -def physmon(output_path, _physmon_prereqs, tmp_path, request): +def physmon( + output_path: Path, reference_path: Path, _physmon_prereqs, tmp_path, request +): u = acts.UnitConstants srcdir, detector, trackingGeometry, decorators = _physmon_prereqs - name = re.sub(r"^test_", "", request.node.name) - setup = Physmon( detector=detector, trackingGeometry=trackingGeometry, @@ -169,8 +203,10 @@ def physmon(output_path, _physmon_prereqs, tmp_path, request): geoSel=srcdir / "thirdparty/OpenDataDetector/config/odd-seeding-config.json", field=acts.ConstantBField(acts.Vector3(0, 0, 2 * u.T)), output_path=output_path, + reference_path=reference_path, + update_references=request.config.getoption("--physmon-update-references"), tmp_path=tmp_path, - name=name, + name=_sanitize_test_name(request.node.name), ) return setup @@ -251,7 +287,7 @@ def monitor(physmon: Physmon, request, capsys): memory = physmon.output_path / "memory" memory.mkdir(exist_ok=True) - name = re.sub(r"^test_", "", request.node.name) + name = _sanitize_test_name(request.node.name) with (memory / f"mem_{name}.csv").open("w") as fh: mon = Monitor(output=fh, interval=0.1) diff --git a/CI/physmon/tests/test_ckf_tracking.py b/CI/physmon/tests/test_ckf_tracking.py index 4cbcd68f845..c5a798cc837 100644 --- a/CI/physmon/tests/test_ckf_tracking.py +++ b/CI/physmon/tests/test_ckf_tracking.py @@ -142,8 +142,6 @@ def test_ckf_tracking(seeding_algorithm, physmon: "Physmon"): stems = [ "performance_ckf", "tracksummary_ckf", - "performance_ivf", - "performance_amvf", ] associatedParticles = "particles_input" @@ -182,11 +180,15 @@ def test_ckf_tracking(seeding_algorithm, physmon: "Physmon"): s.run() del s + # @TODO: Add plotting into ROOT file for vertexing and ckf tracksummary + for vertexing in ["ivf", "amvf"]: + target = f"performance_{vertexing}.root" physmon.add_output_file( f"{vertexing}/performance_vertexing.root", - rename=f"performance_{vertexing}.root", + rename=target, ) + physmon.histogram_comparison(target, f"Performance {vertexing}") if seeding_algorithm in [ SeedingAlgorithm.Default, @@ -196,7 +198,9 @@ def test_ckf_tracking(seeding_algorithm, physmon: "Physmon"): stems += ["performance_seeding"] for stem in stems: - physmon.add_output_file(f"{stem}.root", rename=f"{stem}_{physmon.name}.root") + target = f"{stem}.root" + physmon.add_output_file(f"{stem}.root", rename=target) + physmon.histogram_comparison(target, f"Performance {physmon.name} {stem}") # ] + ( # ["performance_seeding", "performance_ambi"] diff --git a/conftest.py b/conftest.py index 00764d2ad07..f50062f1286 100644 --- a/conftest.py +++ b/conftest.py @@ -49,3 +49,11 @@ def pytest_addoption(parser): default=Path.cwd() / "physmon", type=Path, ) + parser.addoption( + "--physmon-reference-path", + action="store", + default=Path(__file__).parent / "CI/physmon/reference", + type=Path, + ) + + parser.addoption("--physmon-update-references", action="store_true") From dc0e2e80467680e0f2fc3f2b3d9b191d7b57b9c4 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Tue, 24 Oct 2023 16:21:29 +0200 Subject: [PATCH 4/5] updates --- CI/physmon/tests/test_ckf_tracking.py | 13 --------- ...cking_kalman.py => test_truth_tracking.py} | 29 +++++++++++++++++++ 2 files changed, 29 insertions(+), 13 deletions(-) rename CI/physmon/tests/{test_truth_tracking_kalman.py => test_truth_tracking.py} (51%) diff --git a/CI/physmon/tests/test_ckf_tracking.py b/CI/physmon/tests/test_ckf_tracking.py index c5a798cc837..58bc24dc98f 100644 --- a/CI/physmon/tests/test_ckf_tracking.py +++ b/CI/physmon/tests/test_ckf_tracking.py @@ -35,19 +35,6 @@ u = acts.UnitConstants -# def run_ckf_tracking(truthSmearedSeeded, truthEstimatedSeeded, label): -# with tempfile.TemporaryDirectory() as temp: - - -# for truthSmearedSeeded, truthEstimatedSeeded, label in [ -# (True, False, "truth_smeared"), # if first is true, second is ignored -# (False, True, "truth_estimated"), -# (False, False, "seeded"), -# (False, False, "orthogonal"), -# ]: -# run_ckf_tracking(truthSmearedSeeded, truthEstimatedSeeded, label) - - @pytest.mark.parametrize( "seeding_algorithm", [ diff --git a/CI/physmon/tests/test_truth_tracking_kalman.py b/CI/physmon/tests/test_truth_tracking.py similarity index 51% rename from CI/physmon/tests/test_truth_tracking_kalman.py rename to CI/physmon/tests/test_truth_tracking.py index 553fddb1616..f965ca83b71 100644 --- a/CI/physmon/tests/test_truth_tracking_kalman.py +++ b/CI/physmon/tests/test_truth_tracking.py @@ -3,6 +3,9 @@ import acts.examples from truth_tracking_kalman import runTruthTrackingKalman +from truth_tracking_gsf import runTruthTrackingGsf + +from helpers import failure_threshold def test_truth_tracking_kalman(physmon: "Physmon"): @@ -32,3 +35,29 @@ def test_truth_tracking_kalman(physmon: "Physmon"): physmon.histogram_comparison( "performance_truth_tracking.root", title="Truth tracking KF" ) + + +def test_truth_tracking_gsf(physmon: "Physmon"): + s = acts.examples.Sequencer( + events=500, + numThreads=-1, + logLevel=acts.logging.INFO, + fpeMasks=acts.examples.Sequencer.FpeMask.fromFile( + Path(__file__).parent.parent / "fpe_masks.yml" + ), + ) + + runTruthTrackingGsf( + physmon.trackingGeometry, + physmon.digiConfig, + physmon.field, + outputDir=physmon.tmp_path, + s=s, + ) + + with failure_threshold(acts.logging.FATAL): + s.run() + del s + + physmon.add_output_file("performance_gsf.root", rename="performance_gsf.root") + physmon.histogram_comparison("performance_gsf.root", title="Truth tracking GSF") From 7cf13e6272b892b707c81c82cafe97b8482e6737 Mon Sep 17 00:00:00 2001 From: Paul Gessinger Date: Mon, 18 Dec 2023 15:10:14 +0100 Subject: [PATCH 5/5] add vertexing test file --- .../Python/tests/test_examples_vertexing.py | 133 ++++++++++++++++++ 1 file changed, 133 insertions(+) create mode 100644 Examples/Python/tests/test_examples_vertexing.py diff --git a/Examples/Python/tests/test_examples_vertexing.py b/Examples/Python/tests/test_examples_vertexing.py new file mode 100644 index 00000000000..0b9f442552a --- /dev/null +++ b/Examples/Python/tests/test_examples_vertexing.py @@ -0,0 +1,133 @@ +if False: + from pathlib import Path + + import pytest + + import acts + import acts.examples + from acts.examples import Sequencer, GenericDetector, RootParticleWriter + + from helpers import ( + dd4hepEnabled, + pythia8Enabled, + AssertCollectionExistsAlg, + assert_csv_output, + assert_entries, + assert_has_entries, + ) + + from acts.examples.odd import getOpenDataDetector + from common import getOpenDataDetectorDirectory + + u = acts.UnitConstants + + @pytest.mark.skipif(not dd4hepEnabled, reason="DD4hep not set up") + @pytest.mark.skipif(not pythia8Enabled, reason="Pythia8 not set up") + @pytest.mark.slow + @pytest.mark.odd + @pytest.mark.filterwarnings("ignore::UserWarning") + def test_vertex_fitting(tmp_path): + detector, trackingGeometry, decorators = getOpenDataDetector( + getOpenDataDetectorDirectory() + ) + + field = acts.ConstantBField(acts.Vector3(0, 0, 2 * u.T)) + + from vertex_fitting import runVertexFitting, VertexFinder + + s = Sequencer(events=100) + + runVertexFitting( + field, + vertexFinder=VertexFinder.Truth, + outputDir=tmp_path, + s=s, + ) + + alg = AssertCollectionExistsAlg(["fittedVertices"], name="check_alg") + s.addAlgorithm(alg) + + s.run() + assert alg.events_seen == s.config.events + + @pytest.mark.parametrize( + "finder,inputTracks,entries", + [ + ("Truth", False, 100), + # ("Truth", True, 0), # this combination seems to be not working + ("Iterative", False, 100), + ("Iterative", True, 100), + ("AMVF", False, 100), + ("AMVF", True, 100), + ], + ) + @pytest.mark.filterwarnings("ignore::UserWarning") + @pytest.mark.flaky(reruns=2) + def test_vertex_fitting_reading( + tmp_path, ptcl_gun, rng, finder, inputTracks, entries, assert_root_hash + ): + ptcl_file = tmp_path / "particles.root" + + detector, trackingGeometry, decorators = GenericDetector.create() + field = acts.ConstantBField(acts.Vector3(0, 0, 2 * u.T)) + + from vertex_fitting import runVertexFitting, VertexFinder + + inputTrackSummary = None + if inputTracks: + from truth_tracking_kalman import runTruthTrackingKalman + + s2 = Sequencer(numThreads=1, events=100) + runTruthTrackingKalman( + trackingGeometry, + field, + digiConfigFile=Path( + Path(__file__).parent.parent.parent.parent + / "Examples/Algorithms/Digitization/share/default-smearing-config-generic.json" + ), + outputDir=tmp_path, + s=s2, + ) + s2.run() + del s2 + inputTrackSummary = tmp_path / "tracksummary_fitter.root" + assert inputTrackSummary.exists() + assert ptcl_file.exists() + else: + s0 = Sequencer(events=100, numThreads=1) + evGen = ptcl_gun(s0) + s0.addWriter( + RootParticleWriter( + level=acts.logging.INFO, + inputParticles=evGen.config.outputParticles, + filePath=str(ptcl_file), + ) + ) + s0.run() + del s0 + + assert ptcl_file.exists() + + finder = VertexFinder[finder] + + s3 = Sequencer(numThreads=1) + + runVertexFitting( + field, + inputParticlePath=ptcl_file, + inputTrackSummary=inputTrackSummary, + outputDir=tmp_path, + vertexFinder=finder, + s=s3, + ) + + alg = AssertCollectionExistsAlg(["fittedVertices"], name="check_alg") + s3.addAlgorithm(alg) + + s3.run() + + vertexing_file = tmp_path / "performance_vertexing.root" + assert vertexing_file.exists() + + assert_entries(vertexing_file, "vertexing", entries) + assert_root_hash(vertexing_file.name, vertexing_file)