From 0097fec87d4074ab8824841f4978674935c3c9e7 Mon Sep 17 00:00:00 2001 From: FBumann <117816358+FBumann@users.noreply.github.com> Date: Mon, 6 Jul 2026 12:55:59 +0200 Subject: [PATCH 1/3] fix(stats): apply threshold along breakdown dimension in plot.effects `stats.plot.effects(by='component'|'contributor')` lays entities out along a coordinate of a single variable (e.g. `costs`) rather than as separate data variables. The threshold filter only dropped whole data variables, so non-invested components with a ~0 contribution were still shown (#719). Generalize the filter to also drop small entries along the breakdown dimension, split it into named helpers, and rename it `_drop_small` since it no longer filters variables only. Adds a parametrized regression test. Fixes #719 Co-Authored-By: Claude Opus 4.8 (1M context) --- flixopt/statistics_accessor.py | 63 +++++++++++++++----- tests/plotting/test_solution_and_plotting.py | 49 +++++++++++++++ 2 files changed, 96 insertions(+), 16 deletions(-) diff --git a/flixopt/statistics_accessor.py b/flixopt/statistics_accessor.py index 99ffa0606..a6af124c4 100644 --- a/flixopt/statistics_accessor.py +++ b/flixopt/statistics_accessor.py @@ -280,23 +280,53 @@ def _sort_dataset(ds: xr.Dataset) -> xr.Dataset: return ds[sorted_vars] -def _filter_small_variables(ds: xr.Dataset, threshold: float | None) -> xr.Dataset: - """Remove variables where max absolute value is below threshold. +def _drop_small_data_vars(ds: xr.Dataset, threshold: float) -> xr.Dataset: + """Drop data variables whose max absolute value is below threshold.""" + max_vals = abs(ds).max() + keep = [v for v in ds.data_vars if float(max_vals.variables[v].values) >= threshold] + return ds[keep] if keep else ds + + +def _drop_small_along_dim(ds: xr.Dataset, dim: str, threshold: float) -> xr.Dataset: + """Drop entries along ``dim`` whose max absolute value (over all other axes) is below threshold. + + Needed when a breakdown such as ``by='component'`` lays entities out along a coordinate + of a single variable rather than as separate variables, e.g. non-invested components + with a near-zero contribution (see #719). + """ + if dim not in ds.dims or not ds.data_vars: + return ds + arr = abs(ds.to_dataarray()) + max_along = arr.max(dim=[x for x in arr.dims if x != dim]) + keep_idx = ds[dim].values[(max_along >= threshold).values] + return ds.sel({dim: keep_idx}) if len(keep_idx) else ds - Useful for filtering out solver noise or non-invested components. + +def _drop_small( + ds: xr.Dataset, threshold: float | None, dim: str | list[str] | None = None +) -> xr.Dataset: + """Remove entries whose max absolute value is below threshold. + + Useful for filtering out solver noise or non-invested components. Always drops whole + data variables that are entirely below threshold; when ``dim`` is given, also drops + individual entries along those coordinate dimension(s). Args: ds: Dataset to filter. threshold: Minimum max absolute value to keep. If None, no filtering. + dim: Optional coordinate dimension(s) to filter along (e.g. 'component', + 'contributor'). A single name or a list of names. Returns: Filtered dataset. """ if threshold is None or not ds.data_vars: return ds - max_vals = abs(ds).max() # Single computation for all variables - keep = [v for v in ds.data_vars if float(max_vals.variables[v].values) >= threshold] - return ds[keep] if keep else ds + ds = _drop_small_data_vars(ds, threshold) + dims = [dim] if isinstance(dim, str) else (dim or []) + for d in dims: + ds = _drop_small_along_dim(ds, d, threshold) + return ds def _filter_by_carrier(ds: xr.Dataset, carrier: str | list[str] | None) -> xr.Dataset: @@ -1557,7 +1587,7 @@ def balance( ds = ds.round(round_decimals) # Filter out variables below threshold - ds = _filter_small_variables(ds, threshold) + ds = _drop_small(ds, threshold) # Build color kwargs: bus balance → component colors, component balance → carrier colors color_by: Literal['component', 'carrier'] = 'component' if is_bus else 'carrier' @@ -1713,7 +1743,7 @@ def carrier_balance( ds = ds.round(round_decimals) # Filter out variables below threshold - ds = _filter_small_variables(ds, threshold) + ds = _drop_small(ds, threshold) # Build color kwargs with component colors (flows colored by their parent component) color_kwargs = self._build_color_kwargs(colors, list(ds.data_vars), color_by='component') @@ -1793,7 +1823,7 @@ def heatmap( # Resolve, select, and stack into single DataArray resolved = self._resolve_variable_names(variables, solution) ds = _apply_selection(solution[resolved], select) - ds = _filter_small_variables(ds, threshold) + ds = _drop_small(ds, threshold) ds = _sort_dataset(ds) # Sort for consistent plotting order da = xr.concat([ds[v] for v in ds.data_vars], dim=pd.Index(list(ds.data_vars), name='variable')) @@ -1888,7 +1918,7 @@ def flows( ds = _apply_selection(ds, select) # Filter out variables below threshold - ds = _filter_small_variables(ds, threshold) + ds = _drop_small(ds, threshold) # Early return for data_only mode (skip figure creation for performance) if data_only: @@ -1957,7 +1987,7 @@ def sizes( ds = ds[valid_labels] # Filter out variables below threshold - ds = _filter_small_variables(ds, threshold) + ds = _drop_small(ds, threshold) # Early return for data_only mode (skip figure creation for performance) if data_only: @@ -2052,7 +2082,7 @@ def duration_curve( result_ds = ds.fxstats.to_duration_curve(normalize=normalize) # Filter out variables below threshold - result_ds = _filter_small_variables(result_ds, threshold) + result_ds = _drop_small(result_ds, threshold) # Early return for data_only mode (skip figure creation for performance) if data_only: @@ -2180,8 +2210,9 @@ def effects( else: raise ValueError(f"'by' must be one of 'component', 'contributor', 'time', or None, got {by!r}") - # Filter out variables below threshold - ds = _filter_small_variables(ds, threshold) + # Filter out entries below threshold, including along the breakdown dimension + breakdown_dim = by if by in ('component', 'contributor') else None + ds = _drop_small(ds, threshold, dim=breakdown_dim) # Early return for data_only mode (skip figure creation for performance) if data_only: @@ -2263,7 +2294,7 @@ def charge_states( ds = _apply_selection(ds, select) # Filter out variables below threshold - ds = _filter_small_variables(ds, threshold) + ds = _drop_small(ds, threshold) # Early return for data_only mode (skip figure creation for performance) if data_only: @@ -2377,7 +2408,7 @@ def storage( flow_ds = flow_ds.round(round_decimals) # Filter out flow variables below threshold - flow_ds = _filter_small_variables(flow_ds, threshold) + flow_ds = _drop_small(flow_ds, threshold) # Early return for data_only mode (skip figure creation for performance) if data_only: diff --git a/tests/plotting/test_solution_and_plotting.py b/tests/plotting/test_solution_and_plotting.py index c9c64e65c..619b9fd3a 100644 --- a/tests/plotting/test_solution_and_plotting.py +++ b/tests/plotting/test_solution_and_plotting.py @@ -217,6 +217,55 @@ def test_statistics_flow_hours(self, simple_flow_system, highs_solver): assert isinstance(flow_hours, xr.Dataset) assert len(flow_hours.data_vars) > 0 + @pytest.mark.parametrize('by', ['component', 'contributor']) + def test_effects_threshold_drops_uninvested_component(self, highs_solver, by): + """threshold must drop non-invested components in effects breakdown (issue #719). + + When broken down by component/contributor, entities live along a coordinate of a + single variable rather than as separate variables, so a per-variable threshold + alone leaves the ~0 non-invested entry visible. + """ + timesteps = pd.date_range('2024-01-15 08:00', periods=2, freq='h') + fs = fx.FlowSystem(timesteps) + fs.add_elements( + fx.Bus('Heat', carrier='heat'), + fx.Effect('costs', '€', 'Total Costs', is_standard=True, is_objective=True), + fx.Source( + 'S1', + outputs=[ + fx.Flow( + 'S1', + bus='Heat', + size=fx.InvestParameters(minimum_size=10, maximum_size=500, effects_of_investment_per_size=10, mandatory=False), + effects_per_flow_hour=10, + ) + ], + ), + fx.Source( + 'S2', + outputs=[ + fx.Flow( + 'S2', + bus='Heat', + size=fx.InvestParameters(minimum_size=10, maximum_size=500, effects_of_investment_per_size=100, mandatory=False), + effects_per_flow_hour=100, + ) + ], + ), + fx.Sink('ABC', inputs=[fx.Flow('abc', bus='Heat', size=1, fixed_relative_profile=222)]), + ) + fs.optimize(highs_solver) + + # S2 is the expensive source and stays uninvested -> zero cost contribution. + filtered = fs.stats.plot.effects('periodic', effect='costs', by=by, threshold=1.0, show=False, data_only=True) + kept = list(filtered.data.coords[by].values) + assert all('S2' not in str(label) for label in kept), f'Uninvested S2 should be dropped, got {kept}' + assert any('S1' in str(label) for label in kept), f'Invested S1 should remain, got {kept}' + + # threshold=None keeps everything, including the zero-cost S2. + unfiltered = fs.stats.plot.effects('periodic', effect='costs', by=by, threshold=None, show=False, data_only=True) + assert any('S2' in str(label) for label in unfiltered.data.coords[by].values) + # ============================================================================ # PLOTTING WITH OPTIMIZED DATA TESTS From e177421d31fc00facdbf47054428358b597342dd Mon Sep 17 00:00:00 2001 From: FBumann <117816358+FBumann@users.noreply.github.com> Date: Mon, 6 Jul 2026 12:59:05 +0200 Subject: [PATCH 2/3] style: apply ruff format Co-Authored-By: Claude Opus 4.8 (1M context) --- flixopt/statistics_accessor.py | 4 +--- tests/plotting/test_solution_and_plotting.py | 12 +++++++++--- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/flixopt/statistics_accessor.py b/flixopt/statistics_accessor.py index a6af124c4..a68f78416 100644 --- a/flixopt/statistics_accessor.py +++ b/flixopt/statistics_accessor.py @@ -302,9 +302,7 @@ def _drop_small_along_dim(ds: xr.Dataset, dim: str, threshold: float) -> xr.Data return ds.sel({dim: keep_idx}) if len(keep_idx) else ds -def _drop_small( - ds: xr.Dataset, threshold: float | None, dim: str | list[str] | None = None -) -> xr.Dataset: +def _drop_small(ds: xr.Dataset, threshold: float | None, dim: str | list[str] | None = None) -> xr.Dataset: """Remove entries whose max absolute value is below threshold. Useful for filtering out solver noise or non-invested components. Always drops whole diff --git a/tests/plotting/test_solution_and_plotting.py b/tests/plotting/test_solution_and_plotting.py index 619b9fd3a..a0d2124c1 100644 --- a/tests/plotting/test_solution_and_plotting.py +++ b/tests/plotting/test_solution_and_plotting.py @@ -236,7 +236,9 @@ def test_effects_threshold_drops_uninvested_component(self, highs_solver, by): fx.Flow( 'S1', bus='Heat', - size=fx.InvestParameters(minimum_size=10, maximum_size=500, effects_of_investment_per_size=10, mandatory=False), + size=fx.InvestParameters( + minimum_size=10, maximum_size=500, effects_of_investment_per_size=10, mandatory=False + ), effects_per_flow_hour=10, ) ], @@ -247,7 +249,9 @@ def test_effects_threshold_drops_uninvested_component(self, highs_solver, by): fx.Flow( 'S2', bus='Heat', - size=fx.InvestParameters(minimum_size=10, maximum_size=500, effects_of_investment_per_size=100, mandatory=False), + size=fx.InvestParameters( + minimum_size=10, maximum_size=500, effects_of_investment_per_size=100, mandatory=False + ), effects_per_flow_hour=100, ) ], @@ -263,7 +267,9 @@ def test_effects_threshold_drops_uninvested_component(self, highs_solver, by): assert any('S1' in str(label) for label in kept), f'Invested S1 should remain, got {kept}' # threshold=None keeps everything, including the zero-cost S2. - unfiltered = fs.stats.plot.effects('periodic', effect='costs', by=by, threshold=None, show=False, data_only=True) + unfiltered = fs.stats.plot.effects( + 'periodic', effect='costs', by=by, threshold=None, show=False, data_only=True + ) assert any('S2' in str(label) for label in unfiltered.data.coords[by].values) From 5ae16cea5af74d35106dbc4a683bad41d3a8715b Mon Sep 17 00:00:00 2001 From: FBumann <117816358+FBumann@users.noreply.github.com> Date: Mon, 6 Jul 2026 13:31:13 +0200 Subject: [PATCH 3/3] fix(stats): drop empty breakdown instead of falling back to full dataset When every entry along the breakdown dimension is below threshold, _drop_small_along_dim returned the unfiltered dataset, silently ignoring the user's threshold (an all-below-threshold effects breakdown showed everything). Return the empty selection instead. The full effects render handles the empty dataset without crashing (verified across all aspects / by / effect combos). Scope: only the new dim-path helper is changed. The pre-existing _drop_small_data_vars fallback shared by 8 other plot methods is left as-is. Co-Authored-By: Claude Opus 4.8 (1M context) --- flixopt/statistics_accessor.py | 2 +- tests/plotting/test_solution_and_plotting.py | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/flixopt/statistics_accessor.py b/flixopt/statistics_accessor.py index a68f78416..3a1438ba8 100644 --- a/flixopt/statistics_accessor.py +++ b/flixopt/statistics_accessor.py @@ -299,7 +299,7 @@ def _drop_small_along_dim(ds: xr.Dataset, dim: str, threshold: float) -> xr.Data arr = abs(ds.to_dataarray()) max_along = arr.max(dim=[x for x in arr.dims if x != dim]) keep_idx = ds[dim].values[(max_along >= threshold).values] - return ds.sel({dim: keep_idx}) if len(keep_idx) else ds + return ds.sel({dim: keep_idx}) def _drop_small(ds: xr.Dataset, threshold: float | None, dim: str | list[str] | None = None) -> xr.Dataset: diff --git a/tests/plotting/test_solution_and_plotting.py b/tests/plotting/test_solution_and_plotting.py index a0d2124c1..d5d5bbbad 100644 --- a/tests/plotting/test_solution_and_plotting.py +++ b/tests/plotting/test_solution_and_plotting.py @@ -272,6 +272,14 @@ def test_effects_threshold_drops_uninvested_component(self, highs_solver, by): ) assert any('S2' in str(label) for label in unfiltered.data.coords[by].values) + # All entries below threshold -> empty breakdown, not a fallback to showing everything, + # and the full render must not crash on the empty dataset. + empty = fs.stats.plot.effects('periodic', effect='costs', by=by, threshold=1e12, show=False) + assert len(empty.data.coords[by].values) == 0, ( + f'All-below-threshold should drop everything, got {list(empty.data.coords[by].values)}' + ) + assert len(empty.figure.data) == 0 + # ============================================================================ # PLOTTING WITH OPTIMIZED DATA TESTS