From 69a318878cdec4f2d62fbf741b5c0c5947435af7 Mon Sep 17 00:00:00 2001 From: Jonas Danke Date: Wed, 13 May 2026 16:15:42 +0200 Subject: [PATCH 01/37] Changes by JD before edits of MS --- edisgo/edisgo.py | 21 + edisgo/run/__init__.py | 31 ++ edisgo/run/config.py | 410 ++++++++++++++++++ edisgo/run/context.py | 115 +++++ edisgo/run/presets/basic.yaml | 22 + .../run/presets/r4mu_base_and_scenario.yaml | 49 +++ edisgo/run/presets/uc1_loads_worst_case.yaml | 32 ++ edisgo/run/presets/uc2_flex_opf.yaml | 43 ++ edisgo/run/presets/uc3_oedb_ts.yaml | 36 ++ edisgo/run/registry.py | 113 +++++ edisgo/run/runner.py | 261 +++++++++++ edisgo/run/tasks/__init__.py | 25 ++ edisgo/run/tasks/io.py | 179 ++++++++ edisgo/run/validator.py | 201 +++++++++ setup.py | 1 + tests/run/__init__.py | 1 + tests/run/test_config.py | 154 +++++++ tests/run/test_registry.py | 35 ++ tests/run/test_runner.py | 118 +++++ tests/run/test_validator.py | 97 +++++ 20 files changed, 1944 insertions(+) create mode 100644 edisgo/run/__init__.py create mode 100644 edisgo/run/config.py create mode 100644 edisgo/run/context.py create mode 100644 edisgo/run/presets/basic.yaml create mode 100644 edisgo/run/presets/r4mu_base_and_scenario.yaml create mode 100644 edisgo/run/presets/uc1_loads_worst_case.yaml create mode 100644 edisgo/run/presets/uc2_flex_opf.yaml create mode 100644 edisgo/run/presets/uc3_oedb_ts.yaml create mode 100644 edisgo/run/registry.py create mode 100644 edisgo/run/runner.py create mode 100644 edisgo/run/tasks/__init__.py create mode 100644 edisgo/run/tasks/io.py create mode 100644 edisgo/run/validator.py create mode 100644 tests/run/__init__.py create mode 100644 tests/run/test_config.py create mode 100644 tests/run/test_registry.py create mode 100644 tests/run/test_runner.py create mode 100644 tests/run/test_validator.py diff --git a/edisgo/edisgo.py b/edisgo/edisgo.py index 07324d010..63b7753b9 100755 --- a/edisgo/edisgo.py +++ b/edisgo/edisgo.py @@ -232,6 +232,27 @@ def config(self): def config(self, kwargs): self._config = Config(**kwargs) + def run_pipeline(self, config): + """ + Run a YAML/JSON task pipeline on this EDisGo instance. + + See :mod:`edisgo.run` for the config schema and task list. + + Parameters + ---------- + config : str, :class:`pathlib.Path`, or dict + Pipeline config as path to a YAML/JSON file or as a dict. + + Returns + ------- + :class:`~.EDisGo` + The EDisGo instance after the pipeline has run. + + """ + from edisgo.run import _run_pipeline_on + + return _run_pipeline_on(self, config) + def import_ding0_grid(self, path, legacy_ding0_grids=True): """ Import ding0 topology data from csv files in the format as diff --git a/edisgo/run/__init__.py b/edisgo/run/__init__.py new file mode 100644 index 000000000..cd202ddef --- /dev/null +++ b/edisgo/run/__init__.py @@ -0,0 +1,31 @@ +""" +YAML/JSON-driven pipeline runner for eDisGo. + +Two entry points share the same core: + + from edisgo.run import run_edisgo + edisgo = run_edisgo("presets/uc2_flex_opf.yaml") + + # or, on an existing EDisGo instance: + edisgo = EDisGo(ding0_grid="30879") + edisgo.run_pipeline("my_run.yaml") + +Pipelines are lists of named tasks from :mod:`edisgo.run.tasks`. Each step +is either a string (``worst_case_ts``) or a single-key mapping with +parameters (``import_electromobility: {charging_strategy: dumb}``). Tasks +can be grouped into ordered ``stages`` that can save artifacts and reload +them with ``load_from``, enabling two-phase workflows (base reinforce + +per-scenario reinforce). +""" + +from edisgo.run.context import RunContext +from edisgo.run.registry import known_tasks, register_task +from edisgo.run.runner import _run_pipeline_on, run_edisgo + +__all__ = [ + "RunContext", + "_run_pipeline_on", + "known_tasks", + "register_task", + "run_edisgo", +] diff --git a/edisgo/run/config.py b/edisgo/run/config.py new file mode 100644 index 000000000..5c4ee8573 --- /dev/null +++ b/edisgo/run/config.py @@ -0,0 +1,410 @@ +""" +Config loader and schema normalizer for the eDisGo pipeline runner. + +The loader turns a YAML file, JSON file, or Python dict into the +canonical internal schema consumed by :mod:`edisgo.run.runner`. It +handles four concerns in a fixed order: + +1. **Read** — parse YAML/JSON (auto-detected by extension; unknown + extensions are tried as JSON first, then YAML). +2. **extends** — resolve a ``extends:`` key recursively into the + parent config and deep-merge; the child overrides parent keys. The + ``extends:`` value may be a path (relative to the including file) + or a bare preset name (resolved against + :mod:`edisgo.run.presets`). +3. **external_config** — merge machine-specific overrides from an + ``external_config:`` path (typically ``~/.edisgo/secrets.json`` + with DB credentials). Keys in the external file override keys in + the main config. +4. **eGo-legacy adaptation** — if the config looks like an eGo + ``scenario_setting_*.json`` (has top-level ``eDisGo.tasks``), map + it onto the new schema so old eGo configs run unchanged. +5. **Stage normalization** — collapse a flat ``pipeline:`` into a + single-stage ``stages: [{name: main, pipeline: [...]}]`` so the + runner only ever deals with the stage form. + +Only :func:`load_config` is public. Everything else is implementation +detail. +""" +from __future__ import annotations + +import copy +import json +import logging +import os + +from pathlib import Path +from typing import Any + +import yaml + +logger = logging.getLogger("edisgo.run.config") + + +def load_config(cfg_or_path) -> dict[str, Any]: + """ + Load, merge, adapt, and normalize a pipeline config. + + Accepts a path to a YAML/JSON file or a dict. The returned dict + always has the normalized shape expected by the runner: + + * top-level ``stages`` (list of ``{name, pipeline, ...}``) + * ``scenario`` (may be ``None``) + * optional ``grid``, ``database``, ``results`` sections + * no ``pipeline``, ``extends``, or ``external_config`` keys + (they have been consumed) + + Parameters + ---------- + cfg_or_path : str, pathlib.Path, or dict + Either a path to a YAML/JSON config file, or a dict already + holding the config. A dict is deep-copied so the caller's + dict is not mutated. + + Returns + ------- + dict + The fully resolved, normalized config. + + Raises + ------ + FileNotFoundError + If the given path (or an ``extends`` reference) does not + exist. + ValueError + If the config has both ``pipeline`` and ``stages``, missing + ``pipeline``/``stages``, duplicate stage names, or a stage + without ``name``/``pipeline``. + + """ + if isinstance(cfg_or_path, (dict,)): + cfg = copy.deepcopy(cfg_or_path) + base_dir = Path.cwd() + else: + path = Path(cfg_or_path).expanduser().resolve() + if not path.is_file(): + raise FileNotFoundError(f"Config file not found: {path}") + cfg = _read_file(path) + base_dir = path.parent + + cfg = _resolve_extends(cfg, base_dir) + cfg = _apply_external_config(cfg) + cfg = _adapt_ego_legacy(cfg) + cfg = _normalize_stages(cfg) + return cfg + + +def _read_file(path: Path) -> dict[str, Any]: + """ + Parse a YAML or JSON file into a dict. + + Parameters + ---------- + path : pathlib.Path + File path. Extension (``.json``, ``.yaml``, ``.yml``) selects + the parser. Unknown extensions fall back to JSON first, then + YAML. + + Returns + ------- + dict + Parsed config contents. + + """ + text = path.read_text() + suffix = path.suffix.lower() + if suffix == ".json": + return json.loads(text) + if suffix in (".yaml", ".yml"): + return yaml.safe_load(text) + try: + return json.loads(text) + except json.JSONDecodeError: + return yaml.safe_load(text) + + +def _resolve_extends(cfg: dict, base_dir: Path) -> dict: + """ + Resolve an ``extends:`` reference and deep-merge parent into child. + + The parent is loaded recursively, so a chain of ``extends:`` works. + References are looked up as (1) a bundled preset name under + :mod:`edisgo.run.presets`, (2) a path relative to ``base_dir``. + The child's keys override the parent's on conflicts. + + Parameters + ---------- + cfg : dict + Child config (may contain ``extends:``). + base_dir : pathlib.Path + Directory against which relative ``extends`` paths are + resolved (usually the directory of the child config). + + Returns + ------- + dict + Merged config with ``extends`` consumed. + + Raises + ------ + FileNotFoundError + If the referenced parent config file does not exist. + + """ + ext = cfg.pop("extends", None) + if ext is None: + return cfg + ext_path = Path(ext).expanduser() + if not ext_path.is_absolute(): + preset_path = _preset_path(str(ext_path)) + if preset_path is not None: + ext_path = preset_path + else: + ext_path = (base_dir / ext_path).resolve() + if not ext_path.is_file(): + raise FileNotFoundError(f"extends: file not found: {ext_path}") + parent = _read_file(ext_path) + parent = _resolve_extends(parent, ext_path.parent) + return _deep_merge(parent, cfg) + + +def _preset_path(name: str) -> Path | None: + """ + Look up a preset YAML/JSON by bare name. + + Searches the ``edisgo/run/presets/`` directory for a file matching + ``name``, ``name.yaml``, ``name.yml``, or ``name.json`` (in that + order). + + Parameters + ---------- + name : str + Preset identifier, e.g. ``"uc2_flex_opf"`` or + ``"presets/uc2_flex_opf.yaml"``. + + Returns + ------- + pathlib.Path or None + The resolved preset path, or ``None`` if no match is found. + + """ + presets_dir = Path(__file__).parent / "presets" + candidates = [ + presets_dir / name, + presets_dir / f"{name}.yaml", + presets_dir / f"{name}.yml", + presets_dir / f"{name}.json", + ] + for c in candidates: + if c.is_file(): + return c + return None + + +def _apply_external_config(cfg: dict) -> dict: + """ + Merge an ``external_config:`` file on top of the current config. + + Used to keep machine-specific secrets (DB credentials, result + directories) out of versioned scenario configs. If the referenced + file does not exist, a warning is logged but the config is used + as-is. + + Parameters + ---------- + cfg : dict + Config possibly containing an ``external_config:`` key. + + Returns + ------- + dict + Merged config with ``external_config`` consumed. + + """ + ext = cfg.pop("external_config", None) + if ext is None: + return cfg + path = Path(os.path.expanduser(ext)) + if not path.is_file(): + logger.warning(f"external_config file not found, skipping: {path}") + return cfg + override = _read_file(path) + return _deep_merge(cfg, override) + + +def _deep_merge(base: dict, override: dict) -> dict: + """ + Recursively merge two dicts, with ``override`` winning on conflicts. + + Nested dicts are merged key-by-key. Non-dict values (including + lists) are replaced wholesale — lists are NOT concatenated, to + keep the merge semantics predictable (otherwise a preset could + silently extend the child's pipeline). + + Parameters + ---------- + base : dict + Parent / lower-priority dict. + override : dict + Child / higher-priority dict. + + Returns + ------- + dict + A new dict holding the merge result. Inputs are not mutated. + + """ + out = copy.deepcopy(base) if base else {} + for key, val in (override or {}).items(): + if ( + key in out + and isinstance(out[key], dict) + and isinstance(val, dict) + ): + out[key] = _deep_merge(out[key], val) + else: + out[key] = copy.deepcopy(val) + return out + + +def _normalize_stages(cfg: dict) -> dict: + """ + Collapse a flat ``pipeline:`` into the canonical ``stages`` shape. + + After this step the runner only has to iterate ``cfg["stages"]``; + flat configs become a single stage named ``main``. + + Parameters + ---------- + cfg : dict + Config with either ``pipeline`` or ``stages`` at the top + level. + + Returns + ------- + dict + Config with ``stages`` guaranteed to be present and + ``pipeline`` removed. + + Raises + ------ + ValueError + If both ``pipeline`` and ``stages`` are present, if neither + is present, if any stage is missing ``name``/``pipeline``, or + if stage names are not unique. + + """ + if "stages" in cfg and "pipeline" in cfg: + raise ValueError( + "Config has both top-level 'pipeline' and 'stages'. " + "Use only one." + ) + if "stages" not in cfg: + pipeline = cfg.pop("pipeline", None) + if pipeline is None: + raise ValueError( + "Config must define either 'pipeline' or 'stages'." + ) + cfg["stages"] = [{"name": "main", "pipeline": pipeline}] + + seen = set() + for stage in cfg["stages"]: + if "name" not in stage: + raise ValueError("Every stage needs a 'name' key.") + if stage["name"] in seen: + raise ValueError( + f"Duplicate stage name: {stage['name']}" + ) + seen.add(stage["name"]) + if "pipeline" not in stage: + raise ValueError( + f"Stage '{stage['name']}' is missing 'pipeline'." + ) + return cfg + + +_EGO_TASK_MAP = { + "1_setup_grid": "setup_grid", + "5_grid_reinforcement": "reinforce", + "4_optimisation": "optimize", + "worst_case_ts": "worst_case_ts", + "base_reinforce": "base_reinforce", + "oedb_ts": "oedb_ts", + "import_heat_pumps_from_db": "import_heat_pumps", + "import_home_batteries_from_db": "import_home_batteries", + "import_dsm_from_db": "import_dsm", + "import_electromobility_from_db": "import_electromobility", + "load_charging_from_files": "load_charging_from_files", + "load_from_base": "load_from_base", +} +"""Mapping from eGo task names to edisgo.run task names. eGo-specific +tasks with no eDisGo equivalent (e.g. ``2_specs_overlying_grid``, +``3_temporal_complexity_reduction``) are intentionally missing — they +require eTraGo and are logged as "skipped" when adapted.""" + + +def _adapt_ego_legacy(cfg: dict) -> dict: + """ + Map an eGo-style ``scenario_setting_*.json`` onto the new schema. + + Recognizes an eGo config by the presence of an ``eDisGo.tasks`` + key at the top level together with the absence of + ``pipeline``/``stages``. Translates: + + * ``eDisGo.grid_path`` → ``grid.ding0_path`` + * ``eDisGo.results`` → ``results.directory`` + * ``eTraGo.scn_name`` → ``scenario`` + * ``eDisGo.tasks`` → ``pipeline`` (via :data:`_EGO_TASK_MAP`) + * top-level ``database``/``ssh`` kept under ``database`` + + eGo-only tasks (overlying grid / temporal reduction) are + dropped with a warning. Cosmetic keys (``eGo``, ``eTraGo``, + ``_comment``, ``_workflow``) are stripped. + + Parameters + ---------- + cfg : dict + Possibly-legacy config. + + Returns + ------- + dict + Adapted config. If the input is not an eGo-legacy config, it + is returned unchanged. + + """ + if "eDisGo" not in cfg or "pipeline" in cfg or "stages" in cfg: + return cfg + + edisgo_cfg = cfg["eDisGo"] + tasks = edisgo_cfg.get("tasks") + if tasks is None: + return cfg + + logger.info( + "Detected legacy eGo config schema — adapting to edisgo.run." + ) + mapped = [] + for t in tasks: + if t not in _EGO_TASK_MAP: + logger.warning( + f"eGo task '{t}' has no eDisGo equivalent — skipping " + "(likely eTraGo-specific)." + ) + continue + mapped.append(_EGO_TASK_MAP[t]) + + adapted: dict[str, Any] = { + "scenario": cfg.get("eTraGo", {}).get("scn_name", "eGon2035"), + "grid": {"ding0_path": edisgo_cfg.get("grid_path")}, + "results": {"directory": edisgo_cfg.get("results")}, + "pipeline": mapped, + } + if "database" in cfg: + adapted["database"] = cfg["database"] + if "ssh" in cfg: + adapted["database"]["ssh"] = cfg["ssh"] + for side_key in ("eGo", "eTraGo", "ssh", "_comment", "_workflow"): + cfg.pop(side_key, None) + cfg.pop("eDisGo", None) + return _deep_merge(adapted, cfg) diff --git a/edisgo/run/context.py b/edisgo/run/context.py new file mode 100644 index 000000000..c2fbce234 --- /dev/null +++ b/edisgo/run/context.py @@ -0,0 +1,115 @@ +""" +Runtime context passed to every task during pipeline execution. + +The context is a small mutable object that threads shared state between +tasks without polluting the :class:`~edisgo.EDisGo` instance itself. +Typical uses: + +* ``scenario`` — the active eGon scenario name (``eGon2035``, + ``eGon100RE``, …) so tasks don't have to re-read it from the config. +* ``engine`` — a SQLAlchemy engine, lazily created on first DB access + via :meth:`RunContext.ensure_engine`. Tasks that don't touch the + database never pay connection cost. +* ``results_dir`` — base directory for stage artifacts and ``save``. +* ``flags`` — free-form boolean/state flags tasks set to coordinate + with each other (``has_heat_pumps``, ``timeseries_set``, …). +* ``stage_artifacts`` — map ``stage_name -> path`` of zip/dir artifacts + emitted by ``save``, consumed by later stages via ``load_from``. + +Tasks should treat ``flags`` as advisory — they MAY short-circuit based +on a flag but MUST NOT assume a flag is present. +""" +from __future__ import annotations + +import logging + +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + + +@dataclass +class RunContext: + """ + Mutable per-run state shared across all tasks of a pipeline. + + Attributes + ---------- + scenario : str or None + Active scenario name from the top-level ``scenario:`` key. + engine : sqlalchemy.engine.Engine or None + Database engine for oedb-backed imports. Created lazily; + see :meth:`ensure_engine`. + results_dir : pathlib.Path or None + Base directory for stage outputs. Resolved from + ``results.directory`` in the config. + logger : logging.Logger + Logger instance used by tasks and the runner. Defaults to + the ``edisgo.run`` logger. + flags : dict + Free-form state flags that tasks use to communicate. Common + keys: ``grid_loaded``, ``timeseries_set``, + ``reactive_power_set``, ``has_heat_pumps``, ``has_dsm``, + ``has_home_batteries``, ``has_electromobility``, + ``base_reinforced``, ``last_saved``. + stage_artifacts : dict + Map ``stage_name -> Path`` of save-artifacts. Populated by the + ``save`` task when running inside a named stage, consumed by + subsequent stages that set ``load_from:``. + current_stage : str or None + Name of the stage currently executing. Set by the runner. + raw_config : dict + The fully resolved pipeline config (after ``extends``, + ``external_config``, and eGo-legacy adaptation). Tasks can + read supplementary keys like ``database.*`` from here. + + """ + + scenario: str | None = None + engine: Any = None + results_dir: Path | None = None + logger: logging.Logger = field( + default_factory=lambda: logging.getLogger("edisgo.run") + ) + flags: dict[str, Any] = field(default_factory=dict) + stage_artifacts: dict[str, Path] = field(default_factory=dict) + current_stage: str | None = None + raw_config: dict[str, Any] = field(default_factory=dict) + + def ensure_engine(self): + """ + Return a database engine, creating it on first call. + + Reads the ``database`` section of :attr:`raw_config` and calls + :func:`edisgo.io.db.engine`. Caches the engine on the context + so subsequent calls reuse the same connection. + + Returns + ------- + sqlalchemy.engine.Engine + The active database engine. + + Raises + ------ + RuntimeError + If the config has no ``database`` section — indicates the + pipeline wants to reach the database without configuring + it. + + """ + if self.engine is not None: + return self.engine + db_cfg = self.raw_config.get("database") + if not db_cfg: + raise RuntimeError( + "Task needs a database engine but no 'database' section " + "is configured." + ) + from edisgo.io.db import engine as egon_engine + + ssh_cfg = db_cfg.get("ssh") or {} + self.engine = egon_engine( + path=db_cfg.get("credentials_path"), + ssh=bool(ssh_cfg.get("enabled", False)), + ) + return self.engine diff --git a/edisgo/run/presets/basic.yaml b/edisgo/run/presets/basic.yaml new file mode 100644 index 000000000..136161855 --- /dev/null +++ b/edisgo/run/presets/basic.yaml @@ -0,0 +1,22 @@ +_comment: | + Basic preset: worst-case pre-reinforce → reinforce. + Minimal end-to-end example with no database dependency. + Reproduces the core of example_01 without flex imports. + +_workflow: + - setup_grid: load ding0 topology + - worst_case_ts: set worst-case time series (feed-in + load) + - reactive_power: fix reactive power control + - check_integrity: validate grid consistency + - reinforce: run grid reinforcement + - save: persist topology + timeseries + results + +scenario: eGon2035 + +pipeline: + - setup_grid + - worst_case_ts + - reactive_power + - check_integrity + - reinforce + - save diff --git a/edisgo/run/presets/r4mu_base_and_scenario.yaml b/edisgo/run/presets/r4mu_base_and_scenario.yaml new file mode 100644 index 000000000..6412f36ca --- /dev/null +++ b/edisgo/run/presets/r4mu_base_and_scenario.yaml @@ -0,0 +1,49 @@ +_comment: | + R4MU — two-stage base + scenario reinforcement: + Stage 1 produces a base-reinforced grid (generators + heat pumps) + and saves it as an artifact. Stage 2 loads that artifact, integrates + scenario-specific charging stations from a GeoPackage/CSV directory, + applies worst-case time series, and runs a scenario-specific + reinforce. Cost delta = extra reinforcement caused by the charging + scenario. + +_workflow: + - stage base: + - setup_grid: load ding0 topology + import generators + - import_heat_pumps: from egon_data + - worst_case_ts + - reactive_power + - reinforce + - save (artifact consumed by next stage) + - stage scenario: + - load_from: base + - load_charging_from_files: integrate scenario charging + - worst_case_ts + - reactive_power + - reinforce (delta only) + - save + +scenario: eGon2035 + +stages: + - name: base + pipeline: + - setup_grid: {import_generators: true} + - import_heat_pumps + - worst_case_ts + - reactive_power + - reinforce + - save + - name: scenario + load_from: base + params: + charging_dir: "./charging_scenario_1" + mv_threshold_kw: 100 + pipeline: + - load_charging_from_files: + charging_dir: "{{params.charging_dir}}" + mv_threshold_kw: "{{params.mv_threshold_kw}}" + - worst_case_ts + - reactive_power + - reinforce + - save diff --git a/edisgo/run/presets/uc1_loads_worst_case.yaml b/edisgo/run/presets/uc1_loads_worst_case.yaml new file mode 100644 index 000000000..2204d3ce9 --- /dev/null +++ b/edisgo/run/presets/uc1_loads_worst_case.yaml @@ -0,0 +1,32 @@ +_comment: | + UC1 — worst-case flexibility loads: + load grid, base-reinforce (generators only), then import flex assets + (heat pumps, home batteries, DSM, electromobility) and apply worst-case + time series before a final reinforce. Cost delta = extra reinforcement + caused by the new assets under worst-case conditions. + +_workflow: + - setup_grid: load ding0 topology, import generators + - base_reinforce: worst-case TS + reinforce + reset equipment_changes + - import_heat_pumps: from egon_data + - import_home_batteries: from egon_data + - import_dsm: from egon_data + - import_electromobility: from egon_data (dumb charging) + - worst_case_ts: synthetic worst case incl. new assets + - reactive_power: fix reactive power control + - reinforce: final reinforcement — delta only + - save: persist topology + results + +scenario: eGon2035 + +pipeline: + - setup_grid: {import_generators: true} + - base_reinforce + - import_heat_pumps + - import_home_batteries + - import_dsm + - import_electromobility: {charging_strategy: dumb} + - worst_case_ts + - reactive_power + - reinforce + - save diff --git a/edisgo/run/presets/uc2_flex_opf.yaml b/edisgo/run/presets/uc2_flex_opf.yaml new file mode 100644 index 000000000..c09cae87e --- /dev/null +++ b/edisgo/run/presets/uc2_flex_opf.yaml @@ -0,0 +1,43 @@ +_comment: | + UC2 — OPF with full flexibility: + Like UC1 but loads real egon_data time series (oedb) and runs a + powermodels OPF over flexibilities (heat pumps, EV, DSM, storage) + before the final reinforce. Cost delta = extra reinforcement needed + under optimal flex dispatch. + +_workflow: + - setup_grid: load ding0 topology, import generators + - base_reinforce: worst-case TS + reinforce + reset equipment_changes + - import_heat_pumps: from egon_data + - import_home_batteries: from egon_data + - import_dsm: from egon_data + - import_electromobility: from egon_data (dumb charging, flex bands) + - oedb_ts: real wind/solar + load time series (168 h, 2035) + - apply_heat_pump_strategy: uncontrolled (overwritten by OPF) + - reactive_power + - check_integrity + - optimize: pm_optimize with flex assets (SOC, opf v2) + - reinforce: final reinforcement + - save + +scenario: eGon2035 + +pipeline: + - setup_grid: {import_generators: true} + - base_reinforce + - import_heat_pumps + - import_home_batteries + - import_dsm + - import_electromobility: {charging_strategy: dumb} + - oedb_ts: + timeindex: {start: "2035-01-01", periods: 168, freq: h} + dispatchable: {other: 0.7} + - apply_heat_pump_strategy: {strategy: uncontrolled} + - reactive_power + - check_integrity + - optimize: + flexible: [heat_pumps, storage] + method: soc + opf_version: 2 + - reinforce + - save diff --git a/edisgo/run/presets/uc3_oedb_ts.yaml b/edisgo/run/presets/uc3_oedb_ts.yaml new file mode 100644 index 000000000..59c184cdd --- /dev/null +++ b/edisgo/run/presets/uc3_oedb_ts.yaml @@ -0,0 +1,36 @@ +_comment: | + UC3 — real-world time series without OPF: + Like UC1 but uses real egon_data time series (oedb) instead of + synthetic worst cases. No optimization, no eTraGo. Difference to + UC1 is the data source for the final TS; difference to UC2 is no + OPF. + +_workflow: + - setup_grid: load ding0 topology, import generators + - base_reinforce: worst-case TS + reinforce + reset equipment_changes + - import_heat_pumps: from egon_data + - import_home_batteries: from egon_data + - import_dsm: from egon_data + - import_electromobility: from egon_data (dumb charging) + - oedb_ts: real egon_data time series + - apply_heat_pump_strategy: uncontrolled + - reactive_power + - reinforce: final reinforcement + - save + +scenario: eGon2035 + +pipeline: + - setup_grid: {import_generators: true} + - base_reinforce + - import_heat_pumps + - import_home_batteries + - import_dsm + - import_electromobility: {charging_strategy: dumb} + - oedb_ts: + timeindex: {start: "2035-01-01", periods: 168, freq: h} + dispatchable: {other: 0.7} + - apply_heat_pump_strategy: {strategy: uncontrolled} + - reactive_power + - reinforce + - save diff --git a/edisgo/run/registry.py b/edisgo/run/registry.py new file mode 100644 index 000000000..8aed4f3f5 --- /dev/null +++ b/edisgo/run/registry.py @@ -0,0 +1,113 @@ +""" +Task registry for the eDisGo pipeline runner. + +This module holds the global, process-wide mapping of task names to task +functions. Tasks are registered via the :func:`register_task` decorator +and looked up by name at pipeline execution time by the runner. Keeping +the registry separate from both the runner and the task implementations +lets external projects add their own tasks without patching eDisGo — +just import ``register_task`` and decorate a function. + +Registered tasks all share the signature ``(edisgo, ctx, **params)`` +where ``edisgo`` is the current :class:`~edisgo.EDisGo` instance (or +``None`` before it has been created by the first task), ``ctx`` is a +:class:`~edisgo.run.context.RunContext`, and ``**params`` are the +parameters passed from the YAML/JSON step definition. A task may return +an updated ``edisgo`` object (e.g. ``setup_grid`` creates it, ``load_*`` +replaces it); otherwise the runner keeps using the same instance. +""" +from __future__ import annotations + +from typing import Callable + +_TASKS: dict[str, Callable] = {} + + +def register_task(name: str) -> Callable[[Callable], Callable]: + """ + Decorator to register a task function under the given name. + + The decorated function becomes addressable from YAML/JSON pipelines + as either a plain string ``name`` or a single-key mapping + ``name: {param: value, ...}``. The name must be unique globally — + re-registering raises :class:`ValueError` to prevent silent + overrides across plugins. + + Parameters + ---------- + name : str + Unique task name used in pipeline definitions. + + Returns + ------- + Callable + A decorator that registers ``fn`` and returns it unchanged. + + Raises + ------ + ValueError + If ``name`` is already registered. + + Examples + -------- + >>> @register_task("set_timeindex_weekly") + ... def task_weekly(edisgo, ctx, *, start): + ... import pandas as pd + ... edisgo.set_timeindex(pd.date_range(start, periods=168, freq="h")) + + """ + def deco(fn: Callable) -> Callable: + if name in _TASKS: + raise ValueError( + f"Task '{name}' is already registered " + f"(existing={_TASKS[name].__qualname__}, " + f"new={fn.__qualname__})." + ) + _TASKS[name] = fn + return fn + + return deco + + +def get_task(name: str) -> Callable: + """ + Look up a registered task function by name. + + Parameters + ---------- + name : str + Task name as used in pipeline definitions. + + Returns + ------- + Callable + The task function registered under ``name``. + + Raises + ------ + KeyError + If ``name`` is not registered. The error message lists all + known task names to aid typo debugging. + + """ + if name not in _TASKS: + raise KeyError( + f"Unknown task: '{name}'. Known tasks: {sorted(_TASKS)}" + ) + return _TASKS[name] + + +def known_tasks() -> list[str]: + """ + Return a sorted list of all registered task names. + + Useful for error messages, CLI completion, and tests that assert + core tasks exist. + + Returns + ------- + list of str + All registered task names in alphabetical order. + + """ + return sorted(_TASKS) diff --git a/edisgo/run/runner.py b/edisgo/run/runner.py new file mode 100644 index 000000000..63f30aa07 --- /dev/null +++ b/edisgo/run/runner.py @@ -0,0 +1,261 @@ +""" +Pipeline execution engine for the eDisGo runner. + +This module ties the other three pieces — :mod:`edisgo.run.config` +(loader), :mod:`edisgo.run.validator` (static checks), and +:mod:`edisgo.run.registry` (task lookup) — together into a linear +stage-by-stage executor. + +The execution model: + +1. Load and validate the config. +2. Build a :class:`~edisgo.run.context.RunContext`. +3. For each stage, if the stage declares ``load_from: X``, reload + the EDisGo object from stage ``X``'s save-artifact (topology + + results only; time series are dropped to let the new stage set + fresh ones). +4. For each step in the stage's pipeline, look up the task function + in the registry and call it with the current EDisGo object and + the context. A task may return a new EDisGo object (``setup_grid``, + ``load_from_base``) which then replaces the current one. +5. Repeat for all stages, finally return the EDisGo object. + +Two entry points are exposed: + +* :func:`run_edisgo` — starts from no EDisGo object; the first task + must create one (usually ``setup_grid``). +* :func:`_run_pipeline_on` — starts from an existing EDisGo instance; + used by :meth:`edisgo.EDisGo.run_pipeline`. +""" +from __future__ import annotations + +import logging + +from pathlib import Path +from typing import Any + +from edisgo.run import tasks as _tasks # noqa: F401 — triggers registration +from edisgo.run.config import load_config +from edisgo.run.context import RunContext +from edisgo.run.registry import get_task +from edisgo.run.validator import _split_step, validate + +logger = logging.getLogger("edisgo.run.runner") + + +def run_edisgo(config) -> Any: + """ + Run an eDisGo pipeline from a YAML/JSON config or dict. + + This is the standalone entry point. The pipeline's first task is + typically ``setup_grid`` or ``load_from_base`` to bootstrap the + :class:`~edisgo.EDisGo` instance. If you already have one, + prefer :meth:`edisgo.EDisGo.run_pipeline` instead. + + Parameters + ---------- + config : str, pathlib.Path, or dict + Path to a YAML/JSON pipeline config, or an in-memory dict of + the same shape. + + Returns + ------- + :class:`~edisgo.EDisGo` + The EDisGo instance after the last stage has run. For + multi-stage configs this is the object produced by the final + stage. + + """ + return _run_pipeline_on(None, config) + + +def _run_pipeline_on(edisgo, config): + """ + Internal runner shared by :func:`run_edisgo` and the EDisGo method. + + Parameters + ---------- + edisgo : edisgo.EDisGo or None + Existing EDisGo instance to operate on, or ``None`` to have + the first task create one. + config : str, pathlib.Path, or dict + Config to execute. Passed through to + :func:`edisgo.run.config.load_config`. + + Returns + ------- + edisgo.EDisGo + The final EDisGo instance. + + Raises + ------ + RuntimeError + If a stage declares ``load_from: X`` but ``X`` produced no + artifact (typically because validate() was skipped). + + """ + cfg = load_config(config) + validate(cfg) + ctx = _build_context(cfg) + + for stage in cfg["stages"]: + ctx.current_stage = stage["name"] + ctx.logger.info(f"=== stage '{stage['name']}' ===") + + load_from = stage.get("load_from") + if load_from is not None: + artifact = ctx.stage_artifacts.get(load_from) + if artifact is None: + raise RuntimeError( + f"Stage '{stage['name']}' wants to load from " + f"'{load_from}' but no artifact is registered." + ) + edisgo = _load_artifact(str(artifact)) + + params = stage.get("params", {}) or {} + for step in stage["pipeline"]: + name, step_params = _split_step(step) + step_params = _resolve_templating(step_params, params) + ctx.logger.info(f" -> task '{name}'") + task_fn = get_task(name) + result = task_fn(edisgo, ctx, **step_params) + if result is not None: + edisgo = result + + return edisgo + + +def _build_context(cfg: dict) -> RunContext: + """ + Build a :class:`~edisgo.run.context.RunContext` from a config. + + Wires ``scenario`` and ``results.directory`` into the context and + stores the full config under :attr:`RunContext.raw_config` so + tasks can read supplementary sections. + + Parameters + ---------- + cfg : dict + Normalized config. + + Returns + ------- + RunContext + Initialized context with no engine, no artifacts, empty flags. + + """ + results_cfg = cfg.get("results") or {} + results_dir = results_cfg.get("directory") + return RunContext( + scenario=cfg.get("scenario"), + results_dir=Path(results_dir) if results_dir else None, + raw_config=cfg, + ) + + +def _load_artifact(path: str): + """ + Reload an EDisGo instance from a save-artifact for a ``load_from``. + + Loads topology + results only; time series and flex data are + dropped so the consuming stage can set them fresh. Equipment + changes are reset so the next stage's reinforce accounts only + for its own scenario. + + Parameters + ---------- + path : str + Path to a directory or ``.zip`` produced by the ``save`` + task. + + Returns + ------- + edisgo.EDisGo + The restored EDisGo instance. + + """ + import pandas as pd + + from edisgo.edisgo import import_edisgo_from_files + + from_zip = path.endswith(".zip") + edisgo = import_edisgo_from_files( + edisgo_path=path, + import_topology=True, + import_timeseries=False, + import_results=True, + import_electromobility=False, + import_heat_pump=False, + import_dsm=False, + import_overlying_grid=False, + from_zip_archive=from_zip, + ) + edisgo.legacy_grids = False + edisgo.results.equipment_changes = pd.DataFrame() + return edisgo + + +def _resolve_templating(step_params: dict, stage_params: dict) -> dict: + """ + Substitute ``{{params.x}}`` placeholders in step parameters. + + Stage-level ``params:`` allows a preset to expose a few knobs that + individual step parameters can reference. Only simple + ``{{params.KEY}}`` expansions inside string values are supported + (no filters, no conditionals, no nested expressions) — deliberately + kept trivial to avoid a Jinja dependency. + + Parameters + ---------- + step_params : dict + Keyword arguments for a single step. + stage_params : dict + Stage-level ``params:`` dict. + + Returns + ------- + dict + ``step_params`` with template strings resolved. + + """ + if not stage_params or not step_params: + return step_params + out = {} + for k, v in step_params.items(): + if isinstance(v, str) and "{{" in v: + out[k] = _render_template(v, stage_params) + else: + out[k] = v + return out + + +def _render_template(s: str, stage_params: dict) -> str: + """ + Expand ``{{params.KEY}}`` references in a single string. + + Parameters + ---------- + s : str + Source string. + stage_params : dict + Mapping of stage-level parameters. + + Returns + ------- + str + Rendered string. Unknown keys are left in place (the original + placeholder remains) so downstream errors point at the + typo-ed key rather than silently turning into an empty + string. + + """ + import re + + def repl(match): + expr = match.group(1).strip() + if expr.startswith("params."): + key = expr.split(".", 1)[1] + return str(stage_params.get(key, match.group(0))) + return match.group(0) + + return re.sub(r"\{\{\s*([^}]+)\s*\}\}", repl, s) diff --git a/edisgo/run/tasks/__init__.py b/edisgo/run/tasks/__init__.py new file mode 100644 index 000000000..0d59ea02a --- /dev/null +++ b/edisgo/run/tasks/__init__.py @@ -0,0 +1,25 @@ +""" +Task implementations for the eDisGo pipeline runner. + +Importing this package as a side effect registers every task defined +in its submodules with :func:`edisgo.run.registry.register_task`, so +that the runner sees them at execution time. The submodules are: + +* :mod:`.grid` — ``setup_grid``, ``load_from_base`` +* :mod:`.timeseries` — ``worst_case_ts``, ``oedb_ts``, ``manual_ts``, + ``set_timeindex``, ``reactive_power`` +* :mod:`.flex` — flex imports + (``import_heat_pumps``, ``import_home_batteries``, ``import_dsm``, + ``import_electromobility``, ``import_generators``) and operating + strategies (``apply_charging_strategy``, + ``apply_heat_pump_strategy``) +* :mod:`.analysis` — ``check_integrity``, ``analyze``, ``reinforce``, + ``base_reinforce``, ``optimize`` +* :mod:`.io` — ``save``, ``load_charging_from_files`` + +Task signature convention: ``(edisgo, ctx, **params)``. A task may +mutate ``edisgo`` in place and/or return a new EDisGo instance (the +returned value, if non-None, replaces the current one in the runner's +loop). +""" +from edisgo.run.tasks import analysis, flex, grid, io, timeseries # noqa: F401 diff --git a/edisgo/run/tasks/io.py b/edisgo/run/tasks/io.py new file mode 100644 index 000000000..3f604914e --- /dev/null +++ b/edisgo/run/tasks/io.py @@ -0,0 +1,179 @@ +""" +Input/output tasks — persisting results and ingesting external files. + +* :func:`task_save` (``save``) — persist topology, time series, and + results to disk (directory or zip). Also publishes the artifact + path into ``ctx.stage_artifacts`` so a later stage can + ``load_from:``. +* :func:`task_load_charging_from_files` + (``load_charging_from_files``) — R4MU-specific placeholder for + integrating scenario charging stations from a directory of CSV / + GeoPackage files; implementation is deferred until needed. +""" +from __future__ import annotations + +import os + +from edisgo.run.registry import register_task + + +@register_task("save") +def task_save(edisgo, ctx, *, directory=None, save_topology=True, + save_timeseries=True, save_results=True, + save_electromobility=None, save_opf_results=False, + save_heatpump=None, save_overlying_grid=False, + save_dsm=None, archive=False, archive_type="zip", + reduce_memory=False, parameters=None): + """ + Save the current EDisGo state to disk. + + If ``directory`` is not given, the artifact is written under + ``ctx.results_dir / `` so every stage gets its own + subdirectory. When ``archive=True`` the result is a single zip; + the artifact path (including ``.zip``) is recorded in + ``ctx.stage_artifacts[]`` so a downstream stage can + declare ``load_from: ``. + + Flags drive smart defaults for the optional ``save_*`` switches: + if flex data is absent (per ``ctx.flags``), saving it is skipped. + + Parameters + ---------- + edisgo : edisgo.EDisGo + EDisGo instance to persist. + ctx : RunContext + Run context. Uses ``ctx.results_dir``, ``ctx.current_stage``, + and reads ``has_heat_pumps`` / ``has_dsm`` / + ``has_electromobility`` flags. + directory : str, optional + Absolute target directory. If omitted, derived from + ``ctx.results_dir / ctx.current_stage``. + save_topology : bool, optional + Write the topology CSVs. Default ``True``. + save_timeseries : bool, optional + Write time-series CSVs. Default ``True``. + save_results : bool, optional + Write the results CSVs (equipment changes, expansion costs, + etc.). Default ``True``. + save_electromobility : bool or None, optional + If ``None``, auto-enabled iff + ``ctx.flags['has_electromobility']`` is truthy. + save_opf_results : bool, optional + Write OPF results if present. + save_heatpump : bool or None, optional + If ``None``, auto-enabled iff ``ctx.flags['has_heat_pumps']`` + is truthy. + save_overlying_grid : bool, optional + Write overlying-grid (eTraGo) specs if present. + save_dsm : bool or None, optional + If ``None``, auto-enabled iff ``ctx.flags['has_dsm']`` is + truthy. + archive : bool, optional + Pack the directory into a single ``.zip`` archive. + archive_type : str, optional + Archive format (currently only ``"zip"``). + reduce_memory : bool, optional + Downcast float time-series to ``float32`` to save disk. + parameters : dict, optional + Fine-grained selection of which results fields to write, + e.g. ``{"grid_expansion_results": ["equipment_changes"]}``. + + Returns + ------- + edisgo.EDisGo + The unchanged EDisGo instance. + + Raises + ------ + ValueError + If no ``directory`` is given and ``ctx.results_dir`` is also + unset. + + """ + if directory is None: + if ctx.results_dir is None: + raise ValueError( + "Task 'save' needs a 'directory' parameter or " + "config.results.directory." + ) + stage = ctx.current_stage or "main" + directory = os.path.join(str(ctx.results_dir), stage) + + if save_heatpump is None: + save_heatpump = ctx.flags.get("has_heat_pumps", False) + if save_dsm is None: + save_dsm = ctx.flags.get("has_dsm", False) + if save_electromobility is None: + save_electromobility = ctx.flags.get("has_electromobility", False) + + kwargs = dict( + directory=directory, + save_topology=save_topology, + save_timeseries=save_timeseries, + save_results=save_results, + save_electromobility=save_electromobility, + save_opf_results=save_opf_results, + save_heatpump=save_heatpump, + save_overlying_grid=save_overlying_grid, + save_dsm=save_dsm, + ) + if archive: + kwargs["archive"] = True + kwargs["archive_type"] = archive_type + if reduce_memory: + kwargs["reduce_memory"] = True + if parameters is not None: + kwargs["parameters"] = parameters + + edisgo.save(**kwargs) + + saved_path = directory + (".zip" if archive else "") + if ctx.current_stage: + ctx.stage_artifacts[ctx.current_stage] = saved_path + ctx.flags["last_saved"] = saved_path + return edisgo + + +@register_task("load_charging_from_files") +def task_load_charging_from_files(edisgo, ctx, *, charging_dir, + use_case_to_sector=None, + mv_threshold_kw=100.0): + """ + Integrate scenario charging stations from files (R4MU workflow). + + PLACEHOLDER — the full implementation lives in eGo's + ``_run_edisgo_task_load_charging_from_files`` and needs to be + ported when R4MU is prioritised. The eGo version reads a + GeoPackage / CSV of charging locations, filters by the MV grid + district geometry, and integrates them into the topology via + :func:`find_nearest_bus` / ``integrate_component_based_on_geolocation`` + with a use-case-to-sector mapping and an MV/LV connection + threshold. + + Parameters + ---------- + edisgo : edisgo.EDisGo + EDisGo instance to modify in place. + ctx : RunContext + Run context. + charging_dir : str + Directory containing the charging-station source files. + use_case_to_sector : dict, optional + Maps raw use-case labels (``"home_detached"`` etc.) to + eDisGo sector names (``"home"``, ``"work"``, …). + mv_threshold_kw : float, optional + Capacity threshold above which stations connect to an MV + bus; below connect to LV. + + Raises + ------ + NotImplementedError + Always — port the eGo implementation before using. + + """ + raise NotImplementedError( + "Task 'load_charging_from_files' is a placeholder port from " + "eGo R4MU. Port the logic from eGo's " + "_run_edisgo_task_load_charging_from_files when R4MU is " + "needed." + ) diff --git a/edisgo/run/validator.py b/edisgo/run/validator.py new file mode 100644 index 000000000..3989b8398 --- /dev/null +++ b/edisgo/run/validator.py @@ -0,0 +1,201 @@ +""" +Static validator for pipeline configs. + +The validator enforces structural and ordering rules that the runner +would otherwise hit at execution time — often after 20 minutes of work. +Running these checks up-front turns "cryptic AttributeError after half +the pipeline" into a clear ``ValueError`` at startup. + +Checked rules: + +* every step maps to a known, registered task name; +* ``reactive_power`` comes after every time-series task in a stage, + never before — ``set_time_series_reactive_power_control`` overwrites + reactive power on the currently set active-power time series; +* ``analyze`` and ``reinforce`` require a time-series task earlier in + the stage (or a ``load_from:`` that brings a prepared grid); +* ``optimize`` requires both a time-series task and at least one flex + import earlier in the stage — OPF without flexibility is meaningless; +* flex imports (``import_heat_pumps``, …) require a loaded grid, i.e. + an earlier ``setup_grid`` / ``load_from_base`` / a stage-level + ``load_from:``; +* ``base_reinforce`` likewise requires a loaded grid; +* a stage that declares ``load_from: X`` can only run if stage ``X`` + ran earlier AND contains a ``save`` step. +""" +from __future__ import annotations + +from typing import Any + +from edisgo.run.registry import known_tasks + +_TS_TASKS = {"worst_case_ts", "oedb_ts", "manual_ts", "set_timeindex"} +_GRID_CREATING_TASKS = {"setup_grid", "load_from_base"} +_FLEX_IMPORTS = { + "import_heat_pumps", + "import_home_batteries", + "import_dsm", + "import_electromobility", +} + + +def validate(cfg: dict) -> None: + """ + Validate a normalized pipeline config against the ordering rules. + + This function does not return a value. On success it simply + returns; on any rule violation it raises :class:`ValueError` with + a message identifying the offending stage and task. + + Parameters + ---------- + cfg : dict + Normalized config as returned by + :func:`edisgo.run.config.load_config`. Must have a ``stages`` + list at the top level. + + Raises + ------ + ValueError + If the config has no stages, an unknown task name, a + structural problem (reactive before TS, reinforce without TS, + optimize without flex, flex import without grid, …), or a + stage references a ``load_from`` source that doesn't exist or + has no ``save`` step. + + """ + stages = cfg.get("stages") or [] + if not stages: + raise ValueError("Config has no stages to run.") + + available_artifacts: set[str] = set() + + for stage in stages: + name = stage["name"] + pipeline = stage.get("pipeline") or [] + load_from = stage.get("load_from") + + if load_from is not None and load_from not in available_artifacts: + raise ValueError( + f"Stage '{name}' requires 'load_from: {load_from}' but " + f"that stage has not run or did not save. Available: " + f"{sorted(available_artifacts)}" + ) + + grid_available = load_from is not None + ts_set = False + reactive_set = False + flex_imported = False + has_save = False + + for step in pipeline: + task_name, _params = _split_step(step) + if task_name not in known_tasks(): + raise ValueError( + f"Unknown task '{task_name}' in stage '{name}'. " + f"Known: {known_tasks()}" + ) + + if task_name in _GRID_CREATING_TASKS: + grid_available = True + if task_name in _TS_TASKS: + if reactive_set: + raise ValueError( + f"Stage '{name}': time-series task " + f"'{task_name}' comes after 'reactive_power' " + f"— reactive_power must be the last " + f"time-series-altering step." + ) + ts_set = True + if task_name == "reactive_power": + reactive_set = True + if task_name in _FLEX_IMPORTS: + flex_imported = True + if not grid_available: + raise ValueError( + f"Stage '{name}': task '{task_name}' requires " + f"a loaded grid (setup_grid or " + f"load_from_base) before it." + ) + if task_name in {"analyze", "reinforce"} and not ( + ts_set or load_from + ): + raise ValueError( + f"Stage '{name}': task '{task_name}' requires time " + f"series to be set (e.g. worst_case_ts or " + f"oedb_ts) before it." + ) + if task_name == "optimize": + if not ts_set and not load_from: + raise ValueError( + f"Stage '{name}': 'optimize' requires time " + f"series." + ) + if not flex_imported and not load_from: + raise ValueError( + f"Stage '{name}': 'optimize' requires at least " + f"one flex asset to be imported." + ) + if task_name == "base_reinforce" and not grid_available: + raise ValueError( + f"Stage '{name}': 'base_reinforce' requires a " + f"loaded grid before it." + ) + if task_name == "save": + has_save = True + + if has_save: + available_artifacts.add(name) + + +def _split_step(step: Any) -> tuple[str, dict]: + """ + Normalize a pipeline step into ``(task_name, params)``. + + Steps are allowed in two forms in YAML/JSON: + + * bare string — ``worst_case_ts`` → ``("worst_case_ts", {})`` + * single-key mapping — + ``import_electromobility: {charging_strategy: dumb}`` + → ``("import_electromobility", {"charging_strategy": "dumb"})`` + + ``None`` as the parameter value is treated as an empty dict so + that YAML's ``task:`` (with nothing after the colon) works. + + Parameters + ---------- + step : str or dict + Raw step as it appears in the pipeline list. + + Returns + ------- + tuple of (str, dict) + The task name and its keyword arguments. + + Raises + ------ + ValueError + If ``step`` is not a string or a single-key mapping, or if + the parameter value is not a mapping. + + """ + if isinstance(step, str): + return step, {} + if isinstance(step, dict): + if len(step) != 1: + raise ValueError( + f"Task step must be a string or single-key mapping, " + f"got: {step}" + ) + (name, params), = step.items() + if params is None: + params = {} + if not isinstance(params, dict): + raise ValueError( + f"Parameters for task '{name}' must be a mapping, " + f"got: {type(params).__name__}" + ) + return name, params + raise ValueError( + f"Task step must be string or mapping, got: {step!r}" + ) diff --git a/setup.py b/setup.py index a07f355c3..706b047b8 100644 --- a/setup.py +++ b/setup.py @@ -100,6 +100,7 @@ def read(fname): "edisgo": [ os.path.join("config", "*.cfg"), os.path.join("equipment", "*.csv"), + os.path.join("run", "presets", "*.yaml"), ] }, ) diff --git a/tests/run/__init__.py b/tests/run/__init__.py new file mode 100644 index 000000000..baf15e08b --- /dev/null +++ b/tests/run/__init__.py @@ -0,0 +1 @@ +"""Tests for the :mod:`edisgo.run` pipeline runner.""" diff --git a/tests/run/test_config.py b/tests/run/test_config.py new file mode 100644 index 000000000..10b4ec474 --- /dev/null +++ b/tests/run/test_config.py @@ -0,0 +1,154 @@ +""" +Unit tests for :mod:`edisgo.run.config` — loader, merger, adapter. + +Covers YAML/JSON parity, ``extends`` resolution (preset-by-name and +relative paths), deep-merge semantics, stage normalization, and the +eGo-legacy adapter. +""" +import json + +import pytest +import yaml + +from edisgo.run.config import _deep_merge, load_config + + +def _write(tmp_path, name, data): + """ + Helper: write ``data`` to ``tmp_path/name`` as YAML or JSON. + + Parameters + ---------- + tmp_path : pathlib.Path + Pytest-provided temporary directory. + name : str + File name with extension (``.yaml``/``.yml``/``.json``). + data : dict + Payload. + + Returns + ------- + pathlib.Path + Path to the written file. + + """ + path = tmp_path / name + if name.endswith(".json"): + path.write_text(json.dumps(data)) + else: + path.write_text(yaml.safe_dump(data)) + return path + + +def test_load_flat_pipeline_normalized_to_stages(tmp_path): + """A flat ``pipeline:`` must normalize to a single 'main' stage.""" + p = _write(tmp_path, "cfg.yaml", { + "scenario": "eGon2035", + "pipeline": ["setup_grid", "worst_case_ts", "reinforce"], + }) + cfg = load_config(str(p)) + assert "pipeline" not in cfg + assert cfg["stages"] == [ + {"name": "main", + "pipeline": ["setup_grid", "worst_case_ts", "reinforce"]} + ] + + +def test_yaml_and_json_equivalent(tmp_path): + """YAML and JSON payloads with identical content must load equal.""" + data = { + "scenario": "eGon2035", + "pipeline": ["setup_grid", "worst_case_ts", "reinforce"], + } + yaml_path = _write(tmp_path, "cfg.yaml", data) + json_path = _write(tmp_path, "cfg.json", data) + assert load_config(str(yaml_path)) == load_config(str(json_path)) + + +def test_extends_merges_parent(tmp_path): + """Child config must deep-merge with its ``extends:`` parent.""" + parent = _write(tmp_path, "parent.yaml", { + "scenario": "eGon2035", + "grid": {"legacy_ding0_grids": False}, + "pipeline": ["setup_grid", "reinforce"], + }) + child = _write(tmp_path, "child.yaml", { + "extends": str(parent), + "grid": {"ding0_path": "/tmp/xyz"}, + }) + cfg = load_config(str(child)) + assert cfg["scenario"] == "eGon2035" + assert cfg["grid"] == { + "legacy_ding0_grids": False, "ding0_path": "/tmp/xyz" + } + assert cfg["stages"][0]["pipeline"] == ["setup_grid", "reinforce"] + + +def test_extends_preset_by_name(tmp_path): + """``extends: basic`` must resolve to the bundled basic preset.""" + child = _write(tmp_path, "child.yaml", { + "extends": "basic", + "grid": {"ding0_path": "/tmp/xyz"}, + }) + cfg = load_config(str(child)) + assert "stages" in cfg + assert cfg["grid"]["ding0_path"] == "/tmp/xyz" + + +def test_deep_merge_nested(): + """Nested dicts must be merged key-by-key, child wins on conflict.""" + base = {"a": {"b": 1, "c": 2}, "d": 4} + over = {"a": {"b": 99, "e": 5}} + merged = _deep_merge(base, over) + assert merged == {"a": {"b": 99, "c": 2, "e": 5}, "d": 4} + + +def test_both_pipeline_and_stages_rejected(tmp_path): + """Top-level ``pipeline`` and ``stages`` are mutually exclusive.""" + p = _write(tmp_path, "cfg.yaml", { + "pipeline": ["setup_grid"], + "stages": [{"name": "x", "pipeline": ["setup_grid"]}], + }) + with pytest.raises(ValueError, match="both"): + load_config(str(p)) + + +def test_duplicate_stage_names_rejected(tmp_path): + """Stage names must be unique; duplicates raise ValueError.""" + p = _write(tmp_path, "cfg.yaml", { + "stages": [ + {"name": "x", "pipeline": ["setup_grid"]}, + {"name": "x", "pipeline": ["reinforce"]}, + ], + }) + with pytest.raises(ValueError, match="Duplicate stage"): + load_config(str(p)) + + +def test_ego_legacy_adapter(tmp_path): + """An eGo ``scenario_setting_*.json`` must adapt to the new schema.""" + ego_cfg = { + "eGo": {"eDisGo": True}, + "eTraGo": {"scn_name": "eGon2035"}, + "eDisGo": { + "grid_path": "/some/path", + "results": "/tmp/results", + "tasks": [ + "1_setup_grid", + "base_reinforce", + "import_heat_pumps_from_db", + "worst_case_ts", + "5_grid_reinforcement", + ], + }, + "database": {"host": "localhost"}, + } + p = _write(tmp_path, "legacy.json", ego_cfg) + cfg = load_config(str(p)) + assert cfg["scenario"] == "eGon2035" + assert cfg["grid"]["ding0_path"] == "/some/path" + assert cfg["stages"][0]["pipeline"] == [ + "setup_grid", "base_reinforce", "import_heat_pumps", + "worst_case_ts", "reinforce", + ] + assert cfg["database"]["host"] == "localhost" diff --git a/tests/run/test_registry.py b/tests/run/test_registry.py new file mode 100644 index 000000000..a56070bf6 --- /dev/null +++ b/tests/run/test_registry.py @@ -0,0 +1,35 @@ +""" +Unit tests for :mod:`edisgo.run.registry`. + +Verifies that core tasks are discoverable, that ``get_task`` raises a +useful error on typos, and that duplicate registrations are rejected. +""" +import pytest + +from edisgo.run.registry import get_task, known_tasks, register_task + + +def test_known_tasks_contains_core(): + """All core task names must be registered on import.""" + tasks = known_tasks() + for core in ["setup_grid", "worst_case_ts", "reactive_power", + "reinforce", "analyze", "save"]: + assert core in tasks + + +def test_get_task_unknown_raises(): + """Unknown task names must surface as a descriptive KeyError.""" + with pytest.raises(KeyError, match="Unknown task"): + get_task("does_not_exist") + + +def test_register_task_duplicate_raises(): + """Registering the same task name twice is a bug — must raise.""" + @register_task("_test_task_for_dup_check") + def _a(edisgo, ctx): + """Marker task #1 — test fixture only.""" + + with pytest.raises(ValueError, match="already registered"): + @register_task("_test_task_for_dup_check") + def _b(edisgo, ctx): + """Marker task #2 — test fixture only, must not register.""" diff --git a/tests/run/test_runner.py b/tests/run/test_runner.py new file mode 100644 index 000000000..0ecb6d08e --- /dev/null +++ b/tests/run/test_runner.py @@ -0,0 +1,118 @@ +""" +End-to-end tests for the eDisGo pipeline runner. + +Uses the small test grid under ``tests/data/ding0_test_network_2`` +(exposed by :mod:`tests.conftest` as +``pytest.ding0_test_network_2_path``) to run full pipelines without +touching the database. Covers: + +* the standalone ``run_edisgo`` entry point with a flat pipeline, +* the instance method ``EDisGo.run_pipeline``, +* the stage mechanism with ``save`` + ``load_from``. +""" +import os + +import pytest + +from edisgo.run import run_edisgo + + +@pytest.fixture +def basic_cfg(tmp_path): + """ + Minimal end-to-end config fixture. + + Produces a config that loads the small ding0 test grid, sets + worst-case time series, fixes reactive power, checks integrity, + runs reinforcement, and saves — no database needed. + + Parameters + ---------- + tmp_path : pathlib.Path + Pytest-provided temp directory for the run's artifacts. + + Returns + ------- + dict + The config dict. + + """ + return { + "scenario": "eGon2035", + "grid": { + "ding0_path": pytest.ding0_test_network_2_path, + "legacy_ding0_grids": True, + }, + "results": {"directory": str(tmp_path)}, + "pipeline": [ + "setup_grid", + "worst_case_ts", + "reactive_power", + "check_integrity", + "reinforce", + "save", + ], + } + + +def test_runner_basic_end_to_end(basic_cfg): + """A flat-pipeline run must execute and persist the expected artifact.""" + edisgo = run_edisgo(basic_cfg) + assert edisgo is not None + assert edisgo.topology is not None + assert os.path.isdir(os.path.join(basic_cfg["results"]["directory"], + "main")) + + +def test_runner_method_on_edisgo(basic_cfg): + """``EDisGo.run_pipeline`` must operate on the existing instance.""" + from edisgo import EDisGo + + basic_cfg["pipeline"] = basic_cfg["pipeline"][1:] # skip setup_grid + edisgo = EDisGo( + ding0_grid=basic_cfg["grid"]["ding0_path"], + legacy_ding0_grids=True, + ) + edisgo = edisgo.run_pipeline(basic_cfg) + assert edisgo.topology is not None + + +def test_runner_two_stages_with_load_from(tmp_path): + """ + A two-stage run must save the first stage and reload it via + ``load_from`` in the second stage, producing both artifacts. + """ + cfg = { + "scenario": "eGon2035", + "grid": { + "ding0_path": pytest.ding0_test_network_2_path, + "legacy_ding0_grids": True, + }, + "results": {"directory": str(tmp_path)}, + "stages": [ + { + "name": "base", + "pipeline": [ + "setup_grid", + "worst_case_ts", + "reactive_power", + "reinforce", + {"save": {"archive": True}}, + ], + }, + { + "name": "scenario", + "load_from": "base", + "pipeline": [ + "worst_case_ts", + "reactive_power", + "reinforce", + "save", + ], + }, + ], + } + edisgo = run_edisgo(cfg) + assert edisgo.topology is not None + assert os.path.exists(os.path.join(str(tmp_path), "base.zip")) + assert os.path.isdir(os.path.join(str(tmp_path), "scenario")) diff --git a/tests/run/test_validator.py b/tests/run/test_validator.py new file mode 100644 index 000000000..4b86f40cf --- /dev/null +++ b/tests/run/test_validator.py @@ -0,0 +1,97 @@ +""" +Unit tests for :mod:`edisgo.run.validator`. + +Each test pins one ordering rule: reactive-before-TS, reinforce +without TS, optimize without flex, flex import without grid, and the +stage-level ``load_from`` constraints. +""" +import pytest + +from edisgo.run.validator import validate + + +def _wrap(pipeline): + """ + Wrap a flat pipeline into a single-stage config dict. + + Parameters + ---------- + pipeline : list + Ordered list of task names / single-key mappings. + + Returns + ------- + dict + Minimal config in the shape expected by :func:`validate`. + + """ + return {"stages": [{"name": "main", "pipeline": pipeline}]} + + +def test_valid_pipeline(): + """A well-formed pipeline must pass validation without raising.""" + validate(_wrap(["setup_grid", "worst_case_ts", "reactive_power", + "reinforce", "save"])) + + +def test_unknown_task_rejected(): + """Typo'd task names must be rejected.""" + with pytest.raises(ValueError, match="Unknown task"): + validate(_wrap(["setup_grid", "nonexistent_task"])) + + +def test_reactive_before_ts_rejected(): + """reactive_power before a TS task violates the ordering rule.""" + with pytest.raises(ValueError, match="reactive_power"): + validate(_wrap(["setup_grid", "reactive_power", "worst_case_ts"])) + + +def test_reinforce_without_ts_rejected(): + """reinforce without any prior time-series step must fail.""" + with pytest.raises(ValueError, match="time series"): + validate(_wrap(["setup_grid", "reinforce"])) + + +def test_optimize_without_flex_rejected(): + """optimize requires at least one flex asset to be imported.""" + with pytest.raises(ValueError, match="flex asset"): + validate(_wrap(["setup_grid", "worst_case_ts", "optimize"])) + + +def test_flex_import_before_grid_rejected(): + """Flex imports require a loaded grid — pre-loading is not enough.""" + with pytest.raises(ValueError, match="loaded grid"): + validate(_wrap(["import_heat_pumps", "worst_case_ts", "reinforce"])) + + +def test_stage_load_from_missing_rejected(): + """``load_from: X`` where X has not run must fail.""" + cfg = {"stages": [ + {"name": "a", "pipeline": ["setup_grid", "worst_case_ts", + "reinforce"]}, + {"name": "b", "load_from": "nonexistent", + "pipeline": ["reinforce"]}, + ]} + with pytest.raises(ValueError, match="load_from"): + validate(cfg) + + +def test_stage_load_from_requires_save_in_source(): + """A stage consumed by ``load_from`` must itself end with ``save``.""" + cfg = {"stages": [ + {"name": "a", "pipeline": ["setup_grid", "worst_case_ts", + "reinforce"]}, # no save + {"name": "b", "load_from": "a", "pipeline": ["reinforce"]}, + ]} + with pytest.raises(ValueError, match="load_from"): + validate(cfg) + + +def test_stage_load_from_with_save_ok(): + """Stage chain with a save in the source must validate successfully.""" + cfg = {"stages": [ + {"name": "a", "pipeline": ["setup_grid", "worst_case_ts", + "reinforce", "save"]}, + {"name": "b", "load_from": "a", "pipeline": ["reinforce", "save"]}, + ]} + validate(cfg) From 78d07bceab9ede3b0557f327c5daf87cfd44357f Mon Sep 17 00:00:00 2001 From: Jonas Danke Date: Wed, 13 May 2026 16:18:51 +0200 Subject: [PATCH 02/37] Add example yaml for full example --- edisgo/run/presets/uc4_example_MS.yaml | 56 ++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100644 edisgo/run/presets/uc4_example_MS.yaml diff --git a/edisgo/run/presets/uc4_example_MS.yaml b/edisgo/run/presets/uc4_example_MS.yaml new file mode 100644 index 000000000..a72573496 --- /dev/null +++ b/edisgo/run/presets/uc4_example_MS.yaml @@ -0,0 +1,56 @@ +_comment: | + UC3 — OPF with full flexibility: + Like UC1 but loads real egon_data time series (oedb) and runs a + powermodels OPF over flexibilities (heat pumps, EV, DSM, storage) + before the final reinforce. Cost delta = extra reinforcement needed + under optimal flex dispatch. + +_workflow: + - setup_grid: load ding0 topology, import generators + - base_reinforce: worst-case TS + reinforce + reset equipment_changes + - import_generators: from edon-data + - import_heat_pumps: from egon_data + - import_home_batteries: from egon_data + - import_dsm: from egon_data + - import_electromobility: from egon_data (dumb charging, flex bands) + - oedb_ts: real wind/solar + load time series (24 h, 2035) + - apply_heat_pump_strategy: uncontrolled (overwritten by OPF) + - reactive_power + - check_integrity + - optimize: pm_optimize with flex assets (SOC, opf v2) + - reinforce: final reinforcement + - save + +scenario: eGon2035 +grid: + ding0_path: "/home/gurobi/.ding0/2024-07-25T17:38:34_new_planning_new_edisgo/ding0_grids/32377" + legacy_ding0_grids: false + +database: + ssh: + enabled: false + +timeindex: {start: "2035-01-01", periods: 24, freq: h} + +results: + directory: results/uc4_example + +pipeline: + - setup_grid + - base_reinforce + - import_generators + - import_home_batteries + - import_heat_pumps + - import_dsm + - import_electromobility: {charging_strategy: dumb, flexibility_bands_ucs : ["home", "work", "public", "hpc"]} + - apply_heat_pump_strategy: {strategy: uncontrolled} + - oedb_ts: + dispatchable: {other: 0.7} + - reactive_power + - check_integrity + - optimize: + flexible: [heat_pumps, storage, charging_points, dsm] + method: soc + opf_version: 2 + - reinforce + - save From de69b53812ce865cb765cba942e795fdd46d30cc Mon Sep 17 00:00:00 2001 From: Jonas Danke Date: Wed, 13 May 2026 16:20:22 +0200 Subject: [PATCH 03/37] Add file for analysis-tasks, Add short cut for DSM --- edisgo/run/tasks/analysis.py | 321 +++++++++++++++++++++++++++++++++++ 1 file changed, 321 insertions(+) create mode 100644 edisgo/run/tasks/analysis.py diff --git a/edisgo/run/tasks/analysis.py b/edisgo/run/tasks/analysis.py new file mode 100644 index 000000000..f028bb31c --- /dev/null +++ b/edisgo/run/tasks/analysis.py @@ -0,0 +1,321 @@ +""" +Power-flow, reinforcement, and optimization tasks. + +The three analysis layers: + +* :func:`task_analyze` (``analyze``) — non-linear AC load flow over + the active time series; does not modify the topology. +* :func:`task_reinforce` (``reinforce``) — iterative reinforcement + that adds/upgrades equipment until all technical constraints are + met. Populates ``results.equipment_changes``. +* :func:`task_optimize` (``optimize``) — powermodels OPF over + flexibilities (heat pumps, EV, DSM, storage) to minimize + reinforcement need. + +In addition: + +* :func:`task_check_integrity` (``check_integrity``) — a cheap + sanity check before the expensive steps. +* :func:`task_base_reinforce` (``base_reinforce``) — two-phase helper: + worst-case TS → reinforce → reset ``equipment_changes``. Used to + produce a "base" grid whose subsequent reinforce costs reflect + only a scenario overlay. +""" +from __future__ import annotations + +import pandas as pd + +from edisgo.run.registry import register_task + + +@register_task("check_integrity") +def task_check_integrity(edisgo, ctx): + """ + Run EDisGo's integrity checks on the topology and time series. + + Catches bus mismatches, missing time series for components, and + similar structural problems. Raises if something is off — do not + swallow it silently. + + Parameters + ---------- + edisgo : edisgo.EDisGo + EDisGo instance to check. + ctx : RunContext + Run context (unused). + + Returns + ------- + edisgo.EDisGo + The unchanged EDisGo instance. + + """ + edisgo.check_integrity() + return edisgo + + +@register_task("analyze") +def task_analyze(edisgo, ctx, *, mode=None, timesteps=None, + raise_not_converged=False, troubleshooting_mode=None): + """ + Run AC power flow over the active time series. + + Parameters + ---------- + edisgo : edisgo.EDisGo + EDisGo instance to analyze. + ctx : RunContext + Run context. Stores the number of non-converged time steps + under ``ctx.flags['not_converged_steps']`` and warns if any. + mode : str, optional + ``None`` (default) runs the full grid; ``"mv"`` runs only the + medium-voltage level; ``"lv"`` runs only LV. + timesteps : pandas.DatetimeIndex, optional + Restrict the analysis to these time steps. + raise_not_converged : bool, optional + If ``True``, raise on non-convergence. Default ``False`` so + the pipeline can continue and ``reinforce`` can attempt to + resolve the issue. + troubleshooting_mode : str, optional + Extra diagnostic mode passed through to + :meth:`EDisGo.analyze`. + + Returns + ------- + edisgo.EDisGo + The analyzed EDisGo instance. + + """ + result = edisgo.analyze( + mode=mode, + timesteps=timesteps, + raise_not_converged=raise_not_converged, + troubleshooting_mode=troubleshooting_mode, + ) + if isinstance(result, tuple) and len(result) == 2: + converged, not_converged = result + ctx.flags["not_converged_steps"] = len(not_converged) + if len(not_converged) > 0: + ctx.logger.warning( + f"Power flow did not converge for {len(not_converged)} " + f"time steps." + ) + return edisgo + + +@register_task("reinforce") +def task_reinforce(edisgo, ctx, *, timesteps_pfa=None, reduced_analysis=False, + copy_grid=False, max_while_iterations=20, + split_voltage_band=True, mode=None, + without_generator_import=False, n_minus_one=False, + catch_convergence_problems=False): + """ + Run iterative grid reinforcement. + + Adds/upgrades lines and transformers until voltage and loading + constraints are met for all time steps. Results accumulate in + :attr:`EDisGo.results.equipment_changes` and + :attr:`~EDisGo.results.grid_expansion_costs`. + + Parameters + ---------- + edisgo : edisgo.EDisGo + EDisGo instance to reinforce. + ctx : RunContext + Run context (unused beyond logging). + timesteps_pfa : pandas.DatetimeIndex, optional + Restrict the reinforcement's analysis to these time steps. + reduced_analysis : bool, optional + If ``True``, use a cheaper convergence check during + reinforcement. + copy_grid : bool, optional + If ``True``, operate on a copy and return it as a new + instance (default ``False``). + max_while_iterations : int, optional + Cap on the outer iteration loop. + split_voltage_band : bool, optional + Split the allowed voltage deviation between MV and LV + (typical MV/LV coupling rule). + mode : str, optional + ``None``, ``"mv"``, ``"lv"``, or ``"mvlv"``. Restricts + reinforcement to a voltage level. + without_generator_import : bool, optional + Skip the implicit generator import step. + n_minus_one : bool, optional + Enable (N-1) contingency reinforcement. Expensive. + catch_convergence_problems : bool, optional + Wrap in the catch-convergence helper for troublesome grids. + + Returns + ------- + edisgo.EDisGo + The reinforced EDisGo instance. + + """ + edisgo.reinforce( + timesteps_pfa=timesteps_pfa, + reduced_analysis=reduced_analysis, + copy_grid=copy_grid, + max_while_iterations=max_while_iterations, + split_voltage_band=split_voltage_band, + mode=mode, + without_generator_import=without_generator_import, + n_minus_one=n_minus_one, + catch_convergence_problems=catch_convergence_problems, + ) + return edisgo + + +@register_task("base_reinforce") +def task_base_reinforce(edisgo, ctx, *, cases=None, + reset_equipment_changes=True, save_artifact=True): + """ + Produce a base-reinforced grid and reset the cost accumulator. + + This is the composite step ported from eGo's two-phase reinforce + workflow: + + 1. Set synthetic worst-case time series (``feed-in_case`` + + ``load_case``). + 2. Run :meth:`EDisGo.reinforce` to bring the grid to a neutral + baseline. + 3. Optionally save the resulting grid so downstream stages can + ``load_from: ...``. + 4. Clear :attr:`Results.equipment_changes` so the next reinforce + captures only scenario-specific deltas. + 5. Restore the prior time index so the next TS-setting task + starts from a clean state. + + Parameters + ---------- + edisgo : edisgo.EDisGo + EDisGo instance to base-reinforce. + ctx : RunContext + Run context. ``ctx.results_dir`` is the artifact destination. + Sets ``ctx.flags['base_reinforced'] = True`` and + ``ctx.stage_artifacts['__base_reinforce__']`` on save. + cases : list of str, optional + Which worst cases to set (subset of + ``{"load_case", "feed-in_case"}``). Default is both. + reset_equipment_changes : bool, optional + Clear the equipment-changes DataFrame after reinforcement. + save_artifact : bool, optional + Write a ``grid_data_base_reinforcement.zip`` next to the + other results. + + Returns + ------- + edisgo.EDisGo + The base-reinforced EDisGo instance. + + """ + import os + + prev_timeindex = edisgo.timeseries.timeindex + + edisgo.set_time_series_worst_case_analysis(cases=cases) + edisgo.reinforce() + + if save_artifact and ctx.results_dir is not None: + artifact_dir = os.path.join( + str(ctx.results_dir), "grid_data_base_reinforcement" + ) + edisgo.save( + directory=artifact_dir, + save_topology=True, + save_timeseries=False, + save_results=True, + archive=True, + archive_type="zip", + parameters={"grid_expansion_results": ["equipment_changes"]}, + ) + ctx.stage_artifacts["__base_reinforce__"] = artifact_dir + ".zip" + + if reset_equipment_changes: + edisgo.results.equipment_changes = pd.DataFrame() + + if len(prev_timeindex) > 0: + edisgo.set_timeindex(prev_timeindex) + + ctx.flags["base_reinforced"] = True + return edisgo + + +@register_task("optimize") +def task_optimize(edisgo, ctx, *, flexible=None, flexible_cps=None, + flexible_hps=None, flexible_loads=None, + flexible_storage_units=None, opf_version=2, method="soc", + warm_start=False, s_base=1): + """ + Run a powermodels optimal-power-flow (OPF) over flexibilities. + + If ``flexible`` is given (high-level shortcut), it expands to the + lower-level ``flexible_*`` lists automatically: + + * ``"heat_pumps"`` → all loads of type ``heat_pump`` + * ``"charging_points"`` → all loads of type ``charging_point`` + * ``"storage"`` → all storage-unit indices + * ``"loads"`` → all DSM-ready load indices + + Explicit ``flexible_*`` kwargs override the shortcut. + + Parameters + ---------- + edisgo : edisgo.EDisGo + EDisGo instance to optimize. + ctx : RunContext + Run context (unused). + flexible : list of str, optional + High-level selector, subset of ``{"heat_pumps", + "charging_points", "storage"}``. If ``None``, nothing is + auto-populated. + flexible_cps : list of str, optional + Explicit list of flexible charging-point names. + flexible_hps : list of str, optional + Explicit list of flexible heat-pump load names. + flexible_loads : list of str, optional + Explicit list of flexible DSM load names. + flexible_storage_units : list of str, optional + Explicit list of flexible storage-unit names. + opf_version : int, optional + Powermodels OPF formulation version (1 or 2, default 2). + method : str, optional + OPF relaxation method, e.g. ``"soc"`` (second-order cone). + warm_start : bool, optional + Reuse a previous solution as the starting point. + s_base : float, optional + Per-unit base power for normalization. + + Returns + ------- + edisgo.EDisGo + The optimized EDisGo instance. + + """ + flexible = flexible or [] + + if flexible_hps is None and "heat_pumps" in flexible: + flexible_hps = edisgo.topology.loads_df.loc[ + edisgo.topology.loads_df.type == "heat_pump" + ].index.tolist() + if flexible_cps is None and "charging_points" in flexible: + flexible_cps = edisgo.topology.loads_df.loc[ + edisgo.topology.loads_df.type == "charging_point" + ].index.tolist() + if flexible_storage_units is None and "storage" in flexible: + flexible_storage_units = edisgo.topology.storage_units_df.index.tolist() + if flexible_loads is not None and "dsm" in flexbile: + flexible_loads = edisgo.dsm.p_min.columns.values + + + edisgo.pm_optimize( + flexible_cps=flexible_cps or [], + flexible_hps=flexible_hps or [], + flexible_loads=flexible_loads or [], + flexible_storage_units=flexible_storage_units or [], + opf_version=opf_version, + method=method, + warm_start=warm_start, + s_base=s_base, + ) + return edisgo From fb5d32b533d7ec9a7d0206c84f13f8e8b10a30c9 Mon Sep 17 00:00:00 2001 From: Jonas Danke Date: Wed, 13 May 2026 16:21:10 +0200 Subject: [PATCH 04/37] Add file for flex-tasks, Add flexibility band generation, --- edisgo/run/tasks/flex.py | 282 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 282 insertions(+) create mode 100644 edisgo/run/tasks/flex.py diff --git a/edisgo/run/tasks/flex.py b/edisgo/run/tasks/flex.py new file mode 100644 index 000000000..fd914a2a0 --- /dev/null +++ b/edisgo/run/tasks/flex.py @@ -0,0 +1,282 @@ +""" +Flex-asset import and operation-strategy tasks. + +These tasks either pull flex assets (heat pumps, home batteries, DSM, +electromobility, generators) from egon_data / OEP into the topology, +or apply an operating strategy on assets already present. They must +run AFTER the grid is loaded (``setup_grid`` or ``load_from_base``) +and typically BEFORE the time-series step, so the time series can +cover the new assets. +""" +from __future__ import annotations + +from edisgo.run.registry import register_task + + +@register_task("import_heat_pumps") +def task_import_heat_pumps(edisgo, ctx, *, import_types=None, timeindex=None): + """ + Import heat pumps from egon_data into the topology. + + Parameters + ---------- + edisgo : edisgo.EDisGo + EDisGo instance to modify in place. + ctx : RunContext + Run context. Uses ``ctx.scenario`` and + ``ctx.ensure_engine()``. Sets + ``ctx.flags['has_heat_pumps']`` to the observed count. + import_types : list of str, optional + Subset of ``["individual_heat_pumps", "central_heat_pumps"]``; + default imports both. + timeindex : pandas.DatetimeIndex, optional + Restrict COP / heat-demand time series to this index. + + Returns + ------- + edisgo.EDisGo + The modified EDisGo instance. + + """ + edisgo.import_heat_pumps( + scenario=ctx.scenario, + engine=ctx.ensure_engine(), + timeindex=timeindex, + import_types=import_types, + ) + ctx.flags["has_heat_pumps"] = len( + edisgo.topology.loads_df.loc[ + edisgo.topology.loads_df.type == "heat_pump" + ] + ) > 0 + return edisgo + + +@register_task("import_home_batteries") +def task_import_home_batteries(edisgo, ctx): + """ + Import home batteries from egon_data into the topology. + + Parameters + ---------- + edisgo : edisgo.EDisGo + EDisGo instance to modify in place. + ctx : RunContext + Run context. Uses ``ctx.scenario`` and + ``ctx.ensure_engine()``. Sets + ``ctx.flags['has_home_batteries']``. + + Returns + ------- + edisgo.EDisGo + The modified EDisGo instance. + + """ + edisgo.import_home_batteries( + scenario=ctx.scenario, engine=ctx.ensure_engine() + ) + ctx.flags["has_home_batteries"] = ( + not edisgo.topology.storage_units_df.empty + ) + return edisgo + + +@register_task("import_dsm") +def task_import_dsm(edisgo, ctx, *, timeindex=None): + """ + Import demand-side-management potential from egon_data. + + Parameters + ---------- + edisgo : edisgo.EDisGo + EDisGo instance to modify in place. + ctx : RunContext + Run context. Uses ``ctx.scenario`` and + ``ctx.ensure_engine()``. Sets ``ctx.flags['has_dsm']``. + timeindex : pandas.DatetimeIndex, optional + Restrict DSM availability time series to this index. + + Returns + ------- + edisgo.EDisGo + The modified EDisGo instance. + + """ + edisgo.import_dsm( + scenario=ctx.scenario, + engine=ctx.ensure_engine(), + timeindex=timeindex, + ) + ctx.flags["has_dsm"] = ( + edisgo.dsm.p_max is not None and not edisgo.dsm.p_max.empty + ) + return edisgo + + +@register_task("import_electromobility") +def task_import_electromobility(edisgo, ctx, *, data_source="oedb", + charging_strategy="dumb", + flexibility_bands_ucs = None, + import_electromobility_data_kwds=None, + allocate_charging_demand_kwds=None): + """ + Import electromobility data (charging processes + parks). + + Optionally applies a charging strategy directly after import to + turn the raw charging processes into active-power time series on + the charging points. + + Parameters + ---------- + edisgo : edisgo.EDisGo + EDisGo instance to modify in place. + ctx : RunContext + Run context. Uses ``ctx.scenario`` and + ``ctx.ensure_engine()`` (for ``data_source='oedb'``). Sets + ``ctx.flags['has_electromobility'] = True``. + data_source : str, optional + ``"oedb"`` (egon_data) or ``"directory"`` (requires + ``import_electromobility_data_kwds={"charging_processes_dir": + ..., "potential_charging_points_dir": ...}``). + charging_strategy : str or None, optional + Charging strategy applied right after import. ``"dumb"`` + (uncontrolled, default), ``"reduced"``, ``"residual"``, or + ``None`` to skip. + flexibility_bands_ucs : str or list of str, optional + Charging-point use case(s) to compute flexibility bands for + via :meth:`Electromobility.get_flexibility_bands` after import + and charging-strategy application. Valid entries: + ``"home"``, ``"work"``, ``"public"``, ``"hpc"``. Pass a single + string for one use case or a list for multiple. ``None`` + (default) skips flexibility-band computation. + import_electromobility_data_kwds : dict, optional + Extra kwargs passed through to the underlying importer. + allocate_charging_demand_kwds : dict, optional + Extra kwargs for charging-demand allocation. + + Returns + ------- + edisgo.EDisGo + The modified EDisGo instance. + + """ + edisgo.import_electromobility( + data_source=data_source, + scenario=ctx.scenario, + engine=ctx.ensure_engine(), + import_electromobility_data_kwds=import_electromobility_data_kwds, + allocate_charging_demand_kwds=allocate_charging_demand_kwds, + ) + if charging_strategy: + edisgo.apply_charging_strategy(strategy=charging_strategy) + if flexibility_bands_ucs is not None: + edisgo.electromobility.get_flexibility_bands( + edisgo, + use_case=flexibility_bands_ucs, + ) + ctx.flags["has_electromobility"] = True + return edisgo + + +@register_task("apply_charging_strategy") +def task_apply_charging_strategy(edisgo, ctx, *, strategy="dumb", + charging_park_ids=None): + """ + Apply a charging strategy to the already-imported EV fleet. + + Standalone variant of the step that ``import_electromobility`` + does inline. Useful when you want to import once and then try + multiple strategies in different runs. + + Parameters + ---------- + edisgo : edisgo.EDisGo + EDisGo instance to modify in place. + ctx : RunContext + Run context. + strategy : str, optional + Strategy name (``"dumb"`` / ``"reduced"`` / ``"residual"``). + charging_park_ids : list of int, optional + Restrict the strategy to these charging-park IDs. + + Returns + ------- + edisgo.EDisGo + The modified EDisGo instance. + + """ + edisgo.apply_charging_strategy( + strategy=strategy, charging_park_ids=charging_park_ids + ) + return edisgo + + +@register_task("apply_heat_pump_strategy") +def task_apply_heat_pump_strategy(edisgo, ctx, *, strategy="uncontrolled", + heat_pump_names=None): + """ + Apply a heat-pump operating strategy. + + Skipped with an info-log if no heat pumps are present + (``ctx.flags['has_heat_pumps']`` is falsy), so pipelines can + safely include this step without a conditional guard. + + Parameters + ---------- + edisgo : edisgo.EDisGo + EDisGo instance to modify in place. + ctx : RunContext + Run context. + strategy : str, optional + Operating strategy (``"uncontrolled"``, ``"flexible"``, …). + heat_pump_names : list of str, optional + Restrict to specific heat-pump load names; default is all. + + Returns + ------- + edisgo.EDisGo + The modified EDisGo instance. + + """ + if not ctx.flags.get("has_heat_pumps"): + ctx.logger.info( + "Skipping 'apply_heat_pump_strategy': no heat pumps " + "present." + ) + return edisgo + edisgo.apply_heat_pump_operating_strategy( + strategy=strategy, heat_pump_names=heat_pump_names + ) + return edisgo + + +@register_task("import_generators") +def task_import_generators(edisgo, ctx, *, generator_scenario=None): + """ + Import future generators for the active scenario. + + Thin wrapper around :meth:`EDisGo.import_generators`. Mostly + useful when you want to split grid loading and generator import + into two separate pipeline steps. + + Parameters + ---------- + edisgo : edisgo.EDisGo + EDisGo instance to modify in place. + ctx : RunContext + Run context. ``ctx.scenario`` is used if + ``generator_scenario`` is not given. + generator_scenario : str, optional + Scenario name, e.g. ``"nep2035"`` or ``"ego100"``. Defaults + to ``ctx.scenario``. + + Returns + ------- + edisgo.EDisGo + The modified EDisGo instance. + + """ + edisgo.import_generators( + generator_scenario=generator_scenario or ctx.scenario + ) + return edisgo From bd5eaa8d5f214e206549f0837ac591706c77dbdf Mon Sep 17 00:00:00 2001 From: Jonas Danke Date: Wed, 13 May 2026 16:22:18 +0200 Subject: [PATCH 05/37] Add file for grid-tasks, Add timeindex in setup task --- edisgo/run/tasks/grid.py | 172 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 172 insertions(+) create mode 100644 edisgo/run/tasks/grid.py diff --git a/edisgo/run/tasks/grid.py b/edisgo/run/tasks/grid.py new file mode 100644 index 000000000..b472f3244 --- /dev/null +++ b/edisgo/run/tasks/grid.py @@ -0,0 +1,172 @@ +""" +Grid loading tasks — bring an EDisGo instance into existence. + +Two ways to start a pipeline: + +* :func:`task_setup_grid` (``setup_grid``) — read a ding0 topology + from disk. This is the typical first step of every pipeline. +* :func:`task_load_from_base` (``load_from_base``) — reload a + previously saved EDisGo instance. Used to split a computation into + a slow "base" phase and one or more fast "scenario" phases that + reuse the base-reinforced grid. +""" +from __future__ import annotations + +from edisgo.run.registry import register_task + + +@register_task("setup_grid") +def task_setup_grid(edisgo, ctx, *, timeindex = None, ding0_path=None, legacy_ding0_grids=None, + import_generators=False, generator_scenario=None): + """ + Load a ding0 grid into an EDisGo instance. + + If the runner was started without an EDisGo object (via + :func:`edisgo.run.run_edisgo`) this task creates one from the + ding0 CSV directory. If an EDisGo object is already present (via + :meth:`edisgo.EDisGo.run_pipeline`), it imports the topology into + that existing instance. + + Parameters + ---------- + edisgo : edisgo.EDisGo or None + Current EDisGo instance, or ``None`` to create a fresh one. + ctx : RunContext + Run context. ``ctx.raw_config['grid']`` is consulted when + parameters are not passed explicitly. + ding0_path : str, optional + Path to the ding0 grid directory. Falls back to + ``ctx.raw_config['grid']['ding0_path']``. + legacy_ding0_grids : bool, optional + Whether to treat the ding0 directory as the legacy format. + Falls back to ``ctx.raw_config['grid']['legacy_ding0_grids']`` + and ultimately to ``False``. + import_generators : bool, optional + If ``True``, call :meth:`EDisGo.import_generators` after + loading the grid. + generator_scenario : str, optional + Generator scenario name passed to + :meth:`EDisGo.import_generators` (only if + ``import_generators=True``). + + Returns + ------- + edisgo.EDisGo + The EDisGo instance with the ding0 topology loaded. + + Raises + ------ + ValueError + If no ``ding0_path`` is given either as a task parameter or + under ``config.grid.ding0_path``. + + """ + from edisgo import EDisGo + + grid_cfg = ctx.raw_config.get("grid", {}) + ding0_path = ding0_path or grid_cfg.get("ding0_path") + if ding0_path is None: + raise ValueError( + "Task 'setup_grid' requires 'ding0_path' either as task " + "parameter or under config.grid.ding0_path." + ) + if legacy_ding0_grids is None: + legacy_ding0_grids = grid_cfg.get("legacy_ding0_grids", False) + + if edisgo is None: + edisgo = EDisGo( + ding0_grid=str(ding0_path), + legacy_ding0_grids=legacy_ding0_grids, + ) + else: + edisgo.import_ding0_grid( + path=str(ding0_path), legacy_ding0_grids=legacy_ding0_grids + ) + + if import_generators: + edisgo.import_generators(generator_scenario=generator_scenario) + + if timeindex is not None: + ti_df = pd.date_range( + start=timeindex["start"], + periods=timeindex["periods"], + freq=timeindex.get("freq", "h"), + ) + edisgo.set_timeindex(ti_df) + + ctx.flags["grid_loaded"] = True + return edisgo + + +@register_task("load_from_base") +def task_load_from_base(edisgo, ctx, *, path, reset_equipment_changes=True, + import_timeseries=False, import_results=False, + import_electromobility=False, import_heat_pump=False, + import_dsm=False, import_overlying_grid=False): + """ + Reload an EDisGo instance from a previously saved directory/zip. + + This is the two-phase R4MU workflow's entry point: stage 1 + produces a base-reinforced grid and saves it, stage 2 (or N) + starts from ``load_from_base`` to pick up that grid and apply + scenario-specific modifications. The cost of the scenario then + shows up cleanly in ``equipment_changes`` because we reset it on + load. + + Parameters + ---------- + edisgo : edisgo.EDisGo or None + Unused — the task always replaces whatever was there. + ctx : RunContext + Run context (logger only). + path : str + Directory or ``.zip`` produced by :func:`task_save`. + reset_equipment_changes : bool, optional + If ``True`` (default), clear + :attr:`Results.equipment_changes` so only the scenario's + reinforce is tracked. + import_timeseries : bool, optional + Whether to import the saved time series. Default: ``False`` + so the next stage sets its own. + import_results : bool, optional + Whether to import saved results. Default: ``False``. + import_electromobility : bool, optional + Whether to import saved electromobility data. + import_heat_pump : bool, optional + Whether to import saved heat-pump data. + import_dsm : bool, optional + Whether to import saved DSM data. + import_overlying_grid : bool, optional + Whether to import saved overlying-grid data (eTraGo + specifications). + + Returns + ------- + edisgo.EDisGo + The restored EDisGo instance. + + """ + import os + + import pandas as pd + + from edisgo.edisgo import import_edisgo_from_files + + path = str(path) + from_zip = path.endswith(".zip") or not os.path.isdir(path) + edisgo = import_edisgo_from_files( + edisgo_path=path, + import_topology=True, + import_timeseries=import_timeseries, + import_results=import_results, + import_electromobility=import_electromobility, + import_heat_pump=import_heat_pump, + import_dsm=import_dsm, + import_overlying_grid=import_overlying_grid, + from_zip_archive=from_zip, + ) + edisgo.legacy_grids = False + if reset_equipment_changes: + edisgo.results.equipment_changes = pd.DataFrame() + ctx.flags["grid_loaded"] = True + return edisgo From aa7800482446dca7df914250b5f6b9c3fd5d2ca2 Mon Sep 17 00:00:00 2001 From: Jonas Danke Date: Wed, 13 May 2026 16:23:27 +0200 Subject: [PATCH 06/37] Add file for timeseries-tasks --- edisgo/run/tasks/timeseries.py | 298 +++++++++++++++++++++++++++++++++ 1 file changed, 298 insertions(+) create mode 100644 edisgo/run/tasks/timeseries.py diff --git a/edisgo/run/tasks/timeseries.py b/edisgo/run/tasks/timeseries.py new file mode 100644 index 000000000..f738aa056 --- /dev/null +++ b/edisgo/run/tasks/timeseries.py @@ -0,0 +1,298 @@ +""" +Time-series tasks — set active/reactive power profiles on EDisGo. + +Time series drive every downstream step: ``analyze``, ``reinforce`` +and ``optimize`` all operate on the time index and power time series +attached to the EDisGo object. The order inside a stage matters: + +1. Set the time index and active-power profiles with one of + :func:`task_worst_case_ts`, :func:`task_oedb_ts`, + :func:`task_manual_ts`, possibly :func:`task_set_timeindex`. +2. Finally call :func:`task_reactive_power` to fix reactive power + control — this MUST come last because it overwrites whatever + reactive power was set by the earlier steps. +""" +from __future__ import annotations + +import pandas as pd + +from edisgo.run.registry import register_task + + +@register_task("worst_case_ts") +def task_worst_case_ts(edisgo, ctx, *, cases=None, + generators_names=None, loads_names=None, + storage_units_names=None): + """ + Set synthetic worst-case active-power time series. + + Produces two snapshots (load case and feed-in case) that + represent the network's extremes. Useful for a coarse first + reinforce that does not require real load/generation data. + + Parameters + ---------- + edisgo : edisgo.EDisGo + EDisGo instance to modify in place. + ctx : RunContext + Run context. Sets ``ctx.flags['timeseries_set'] = True``. + cases : list of str, optional + Subset of ``{"load_case", "feed-in_case"}``. Default is both. + generators_names : list of str, optional + Restrict to these generator names; default is all. + loads_names : list of str, optional + Restrict to these load names; default is all. + storage_units_names : list of str, optional + Restrict to these storage units; default is all. + + Returns + ------- + edisgo.EDisGo + The modified EDisGo instance. + + """ + edisgo.set_time_series_worst_case_analysis( + cases=cases, + generators_names=generators_names, + loads_names=loads_names, + storage_units_names=storage_units_names, + ) + ctx.flags["timeseries_set"] = True + return edisgo + + +@register_task("set_timeindex") +def task_set_timeindex(edisgo, ctx, *, start, periods=None, end=None, + freq="h"): + """ + Set the time index on the EDisGo object. + + Useful as a stand-alone step when you want a specific hourly + range without immediately attaching time-series data (the + ``oedb_ts`` task already accepts a ``timeindex`` argument and + does this internally). + + Parameters + ---------- + edisgo : edisgo.EDisGo + EDisGo instance to modify in place. + ctx : RunContext + Run context. + start : str or pandas.Timestamp + First timestamp of the range. + periods : int, optional + Number of periods; mutually exclusive with ``end``. + end : str or pandas.Timestamp, optional + Last timestamp; mutually exclusive with ``periods``. + freq : str, optional + pandas frequency string, default hourly (``"h"``). + + Returns + ------- + edisgo.EDisGo + The modified EDisGo instance. + + Raises + ------ + ValueError + If neither ``periods`` nor ``end`` is provided. + + """ + if end is not None: + timeindex = pd.date_range(start=start, end=end, freq=freq) + else: + if periods is None: + raise ValueError( + "set_timeindex needs either 'periods' or 'end'." + ) + timeindex = pd.date_range(start=start, periods=periods, freq=freq) + edisgo.set_timeindex(timeindex) + return edisgo + + +@register_task("oedb_ts") +def task_oedb_ts(edisgo, ctx, *, timeindex=None, dispatchable=None, + fluctuating="oedb", conventional_loads="oedb", + charging_points_ts=None): + """ + Set active-power time series from egon_data (OEP) plus overrides. + + This is the "real data" path: wind and solar profiles come from + ``egon_era5_renewable_feedin``, conventional loads come from the + egon demand tables. Dispatchable generators (conventional, + etc.) are set via a per-technology-type profile since egon_data + does not dispatch them. Storage units default to zero if not + already set. + + Parameters + ---------- + edisgo : edisgo.EDisGo + EDisGo instance to modify in place. + ctx : RunContext + Run context. Uses ``ctx.scenario`` and + ``ctx.ensure_engine()`` when any source is ``"oedb"``. Sets + ``ctx.flags['timeseries_set'] = True``. + timeindex : dict, optional + ``{"start": ..., "periods": N, "freq": "h"}``. If present, a + matching :class:`~pandas.DatetimeIndex` is set before + importing data. + dispatchable : dict, optional + Per-technology scaling factors, e.g. ``{"other": 0.7}`` → + constant profile of 0.7 p.u. for all non-fluctuating + generators of type "other". + fluctuating : str or pandas.DataFrame, optional + How to populate wind/solar. ``"oedb"`` pulls egon_data, + ``"default"`` uses bundled standard profiles, or a DataFrame + with columns "solar" / "wind" is passed through. + conventional_loads : str, optional + Source for conventional loads (not heat pumps / charging + points). ``"oedb"`` or ``"demandlib"``. + charging_points_ts : pandas.DataFrame, optional + Explicit active-power profile for charging points; default + ``None`` leaves them untouched so + :func:`task_apply_charging_strategy` can set them. + + Returns + ------- + edisgo.EDisGo + The modified EDisGo instance. + + """ + if timeindex is not None: + ti_df = pd.date_range( + start=timeindex["start"], + periods=timeindex["periods"], + freq=timeindex.get("freq", "h"), + ) + edisgo.set_timeindex(ti_df) + + dispatchable_df = None + if dispatchable is not None: + ti = edisgo.timeseries.timeindex + dispatchable_df = pd.DataFrame(dispatchable, index=ti) + + conv_loads_names = None + if conventional_loads == "oedb": + conv_loads_names = edisgo.topology.loads_df.loc[ + ~edisgo.topology.loads_df.type.isin( + ["heat_pump", "charging_point"] + ) + ].index.tolist() + + edisgo.set_time_series_active_power_predefined( + fluctuating_generators_ts=fluctuating, + conventional_loads_ts=conventional_loads, + conventional_loads_names=conv_loads_names, + dispatchable_generators_ts=dispatchable_df, + charging_points_ts=charging_points_ts, + scenario=ctx.scenario, + engine=ctx.ensure_engine() if fluctuating == "oedb" + or conventional_loads == "oedb" else None, + ) + + su_names = edisgo.topology.storage_units_df.index + if len(su_names) > 0 and edisgo.timeseries.storage_units_active_power.empty: + edisgo.timeseries.storage_units_active_power = pd.DataFrame( + 0.0, index=edisgo.timeseries.timeindex, columns=su_names, + ) + ctx.flags["timeseries_set"] = True + return edisgo + + +@register_task("manual_ts") +def task_manual_ts(edisgo, ctx, *, + generators_active_power=None, + generators_reactive_power=None, + loads_active_power=None, + loads_reactive_power=None, + storage_units_active_power=None, + storage_units_reactive_power=None): + """ + Set active/reactive power time series from explicit DataFrames. + + Used when the caller already has the raw profiles (e.g. from a + coupled run) and wants to inject them directly. Any argument left + at ``None`` is not touched. + + Parameters + ---------- + edisgo : edisgo.EDisGo + EDisGo instance to modify in place. + ctx : RunContext + Run context. Sets ``ctx.flags['timeseries_set'] = True``. + generators_active_power : dict or pandas.DataFrame, optional + Generator active-power profile(s). Converted via + :class:`pandas.DataFrame`. + generators_reactive_power : dict or pandas.DataFrame, optional + Generator reactive-power profile(s). + loads_active_power : dict or pandas.DataFrame, optional + Load active-power profile(s). + loads_reactive_power : dict or pandas.DataFrame, optional + Load reactive-power profile(s). + storage_units_active_power : dict or pandas.DataFrame, optional + Storage-unit active-power profile(s). + storage_units_reactive_power : dict or pandas.DataFrame, optional + Storage-unit reactive-power profile(s). + + Returns + ------- + edisgo.EDisGo + The modified EDisGo instance. + + """ + def _as_df(obj): + return pd.DataFrame(obj) if obj is not None else None + + edisgo.set_time_series_manual( + generators_active_power=_as_df(generators_active_power), + generators_reactive_power=_as_df(generators_reactive_power), + loads_active_power=_as_df(loads_active_power), + loads_reactive_power=_as_df(loads_reactive_power), + storage_units_active_power=_as_df(storage_units_active_power), + storage_units_reactive_power=_as_df(storage_units_reactive_power), + ) + ctx.flags["timeseries_set"] = True + return edisgo + + +@register_task("reactive_power") +def task_reactive_power(edisgo, ctx, *, control="fixed_cosphi", + generators_parametrisation="default", + loads_parametrisation="default", + storage_units_parametrisation="default"): + """ + Apply reactive-power control on top of the active-power time series. + + This MUST be the last time-series-altering step before + ``analyze`` / ``reinforce`` / ``optimize``. The validator + enforces this ordering rule statically. + + Parameters + ---------- + edisgo : edisgo.EDisGo + EDisGo instance to modify in place. + ctx : RunContext + Run context. Sets ``ctx.flags['reactive_power_set'] = True``. + control : str, optional + Reactive-power control strategy; typically ``"fixed_cosphi"``. + generators_parametrisation : str or dict, optional + Per-generator parametrisation, ``"default"`` uses the config. + loads_parametrisation : str or dict, optional + Per-load parametrisation. + storage_units_parametrisation : str or dict, optional + Per-storage-unit parametrisation. + + Returns + ------- + edisgo.EDisGo + The modified EDisGo instance. + + """ + edisgo.set_time_series_reactive_power_control( + control=control, + generators_parametrisation=generators_parametrisation, + loads_parametrisation=loads_parametrisation, + storage_units_parametrisation=storage_units_parametrisation, + ) + ctx.flags["reactive_power_set"] = True + return edisgo From 3a288806d17c6fed76dd4aa8f6f115119ad1775d Mon Sep 17 00:00:00 2001 From: Moritz Schloesser Date: Tue, 19 May 2026 16:40:05 +0200 Subject: [PATCH 07/37] Edit uc4 example --- edisgo/run/presets/uc4_example_MS.yaml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/edisgo/run/presets/uc4_example_MS.yaml b/edisgo/run/presets/uc4_example_MS.yaml index a72573496..13e08b258 100644 --- a/edisgo/run/presets/uc4_example_MS.yaml +++ b/edisgo/run/presets/uc4_example_MS.yaml @@ -24,16 +24,21 @@ _workflow: scenario: eGon2035 grid: ding0_path: "/home/gurobi/.ding0/2024-07-25T17:38:34_new_planning_new_edisgo/ding0_grids/32377" + # grid path should be set in run file legacy_ding0_grids: false + # legacy parameter too database: ssh: enabled: false timeindex: {start: "2035-01-01", periods: 24, freq: h} +# only set in preset, not set in run-functions (eDisGo and eGo) results: directory: results/uc4_example +# actually all parameters + pipeline: - setup_grid @@ -42,6 +47,7 @@ pipeline: - import_home_batteries - import_heat_pumps - import_dsm + # where is decided, which electromobility use cases are used? - import_electromobility: {charging_strategy: dumb, flexibility_bands_ucs : ["home", "work", "public", "hpc"]} - apply_heat_pump_strategy: {strategy: uncontrolled} - oedb_ts: @@ -49,6 +55,7 @@ pipeline: - reactive_power - check_integrity - optimize: + # where is decided which flexibilities are used in the OPF? flexible: [heat_pumps, storage, charging_points, dsm] method: soc opf_version: 2 From b7e741531adc7bb57c625aec1dcca78b5c81b1de Mon Sep 17 00:00:00 2001 From: Moritz Schloesser Date: Wed, 20 May 2026 18:17:15 +0200 Subject: [PATCH 08/37] Wire overlying-grid data through pipeline runner Adds optional overlying_grid_data kwarg to run_edisgo() that is stashed on RunContext for downstream tasks instead of being passed as a keyword to every task (which broke task signatures). task_import_overlying_grid_data now: - accepts the standard (edisgo, ctx, *, ...) signature - reads overlying_grid_data from ctx, falls back to overlying_grid.path in the runner config - loads dispatchable + renewables_potential CSVs and applies them via set_time_series_active_power_predefined - shifts the year and reindexes overlying-grid attributes to the active edisgo timeindex so CSV-based input lines up with OEDB time series task_set_timeindex now reduces existing time-series data to the new index (via reduce_timeseries_data_to_given_timeindex) instead of silently leaving stale data behind. Renames the uc4_example_MS preset to uc4_example. --- .../{uc4_example_MS.yaml => uc4_example.yaml} | 9 +- edisgo/run/runner.py | 8 +- edisgo/run/tasks/io.py | 158 ++++++++++++++++-- edisgo/run/tasks/timeseries.py | 85 ++++++---- 4 files changed, 214 insertions(+), 46 deletions(-) rename edisgo/run/presets/{uc4_example_MS.yaml => uc4_example.yaml} (88%) diff --git a/edisgo/run/presets/uc4_example_MS.yaml b/edisgo/run/presets/uc4_example.yaml similarity index 88% rename from edisgo/run/presets/uc4_example_MS.yaml rename to edisgo/run/presets/uc4_example.yaml index 13e08b258..2d2650279 100644 --- a/edisgo/run/presets/uc4_example_MS.yaml +++ b/edisgo/run/presets/uc4_example.yaml @@ -23,7 +23,7 @@ _workflow: scenario: eGon2035 grid: - ding0_path: "/home/gurobi/.ding0/2024-07-25T17:38:34_new_planning_new_edisgo/ding0_grids/32377" + ding0_path: "/path/to/ding0_grid" # grid path should be set in run file legacy_ding0_grids: false # legacy parameter too @@ -48,10 +48,13 @@ pipeline: - import_heat_pumps - import_dsm # where is decided, which electromobility use cases are used? - - import_electromobility: {charging_strategy: dumb, flexibility_bands_ucs : ["home", "work", "public", "hpc"]} - - apply_heat_pump_strategy: {strategy: uncontrolled} + - import_electromobility: {charging_strategy: null, flexibility_bands_ucs : ["home", "work", "public", "hpc"]} - oedb_ts: dispatchable: {other: 0.7} + timeindex: {start: "2035-01-01", periods: 24, freq: h} + - apply_charging_strategy: {strategy: dumb} + - apply_heat_pump_strategy: {strategy: uncontrolled} + - import_overlying_grid_data - reactive_power - check_integrity - optimize: diff --git a/edisgo/run/runner.py b/edisgo/run/runner.py index 63f30aa07..2d08fd3bc 100644 --- a/edisgo/run/runner.py +++ b/edisgo/run/runner.py @@ -27,6 +27,7 @@ * :func:`_run_pipeline_on` — starts from an existing EDisGo instance; used by :meth:`edisgo.EDisGo.run_pipeline`. """ + from __future__ import annotations import logging @@ -43,7 +44,7 @@ logger = logging.getLogger("edisgo.run.runner") -def run_edisgo(config) -> Any: +def run_edisgo(config, overlying_grid_data=None) -> Any: """ Run an eDisGo pipeline from a YAML/JSON config or dict. @@ -66,10 +67,10 @@ def run_edisgo(config) -> Any: stage. """ - return _run_pipeline_on(None, config) + return _run_pipeline_on(None, config, overlying_grid_data=overlying_grid_data) -def _run_pipeline_on(edisgo, config): +def _run_pipeline_on(edisgo, config, overlying_grid_data=None): """ Internal runner shared by :func:`run_edisgo` and the EDisGo method. @@ -97,6 +98,7 @@ def _run_pipeline_on(edisgo, config): cfg = load_config(config) validate(cfg) ctx = _build_context(cfg) + ctx.overlying_grid_data = overlying_grid_data for stage in cfg["stages"]: ctx.current_stage = stage["name"] diff --git a/edisgo/run/tasks/io.py b/edisgo/run/tasks/io.py index 3f604914e..4e2e84d15 100644 --- a/edisgo/run/tasks/io.py +++ b/edisgo/run/tasks/io.py @@ -10,6 +10,7 @@ integrating scenario charging stations from a directory of CSV / GeoPackage files; implementation is deferred until needed. """ + from __future__ import annotations import os @@ -18,12 +19,24 @@ @register_task("save") -def task_save(edisgo, ctx, *, directory=None, save_topology=True, - save_timeseries=True, save_results=True, - save_electromobility=None, save_opf_results=False, - save_heatpump=None, save_overlying_grid=False, - save_dsm=None, archive=False, archive_type="zip", - reduce_memory=False, parameters=None): +def task_save( + edisgo, + ctx, + *, + directory=None, + save_topology=True, + save_timeseries=True, + save_results=True, + save_electromobility=None, + save_opf_results=False, + save_heatpump=None, + save_overlying_grid=False, + save_dsm=None, + archive=False, + archive_type="zip", + reduce_memory=False, + parameters=None, +): """ Save the current EDisGo state to disk. @@ -93,8 +106,7 @@ def task_save(edisgo, ctx, *, directory=None, save_topology=True, if directory is None: if ctx.results_dir is None: raise ValueError( - "Task 'save' needs a 'directory' parameter or " - "config.results.directory." + "Task 'save' needs a 'directory' parameter or config.results.directory." ) stage = ctx.current_stage or "main" directory = os.path.join(str(ctx.results_dir), stage) @@ -135,9 +147,9 @@ def task_save(edisgo, ctx, *, directory=None, save_topology=True, @register_task("load_charging_from_files") -def task_load_charging_from_files(edisgo, ctx, *, charging_dir, - use_case_to_sector=None, - mv_threshold_kw=100.0): +def task_load_charging_from_files( + edisgo, ctx, *, charging_dir, use_case_to_sector=None, mv_threshold_kw=100.0 +): """ Integrate scenario charging stations from files (R4MU workflow). @@ -177,3 +189,127 @@ def task_load_charging_from_files(edisgo, ctx, *, charging_dir, "_run_edisgo_task_load_charging_from_files when R4MU is " "needed." ) + + +@register_task("import_overlying_grid_data") +def task_import_overlying_grid_data(edisgo, ctx, *, overlying_grid_path=None): + """ + Import overlying grid data into the EDisGo instance. + + When ``overlying_grid_data`` is a dict of DataFrames (as returned by + ``get_etrago_results_per_bus``), the overlying-grid attributes and + dispatchable/fluctuating generator time series are set from it. + + When ``overlying_grid_path`` is a directory path, the overlying-grid + attributes are loaded from CSV files in that directory, and + ``dispatchable_generators_active_power.csv`` / + ``renewables_potential.csv`` are applied as generator time series + if present. + + Falls back to ``ctx.raw_config['eDisGo']['overlying_grid_source']`` + as the directory path when neither argument is given. + + Parameters + ---------- + edisgo : edisgo.EDisGo + EDisGo instance to modify in place. + ctx : RunContext + Run context. + overlying_grid_path : str, optional + Directory containing overlying-grid CSV files. + overlying_grid_data : dict, optional + Dict of DataFrames as returned by ``get_etrago_results_per_bus``. + + Returns + ------- + edisgo.EDisGo + The modified EDisGo instance. + + """ + import pandas as pd + + overlying_grid_data = getattr(ctx, "overlying_grid_data", None) + + if overlying_grid_data is not None: + # eTraGo results dict — set standard overlying-grid attributes + for attr in edisgo.overlying_grid._attributes: + if attr in overlying_grid_data: + setattr(edisgo.overlying_grid, attr, overlying_grid_data[attr]) + # set generator time series + edisgo.set_time_series_active_power_predefined( + dispatchable_generators_ts=overlying_grid_data.get( + "dispatchable_generators_active_power" + ), + fluctuating_generators_ts=overlying_grid_data.get("renewables_potential"), + ) + return edisgo + + # resolve path: explicit arg → runner config overlying_grid.path → skip + if overlying_grid_path is None: + overlying_grid_path = (ctx.raw_config.get("overlying_grid") or {}).get("path") + + if overlying_grid_path is None: + ctx.logger.warning( + "task 'import_overlying_grid_data': no overlying_grid_data or " + "overlying_grid_path provided — skipping." + ) + return edisgo + + # load overlying-grid attributes from CSV directory + edisgo.overlying_grid.from_csv(overlying_grid_path) + + # reindex overlying-grid attributes to match edisgo timeindex + # CSVs may use a different year — shift year then reindex + edisgo_ti = edisgo.timeseries.timeindex + if not edisgo_ti.empty: + for attr in edisgo.overlying_grid._attributes: + ts = getattr(edisgo.overlying_grid, attr) + if ts.empty: + continue + csv_year = ts.index[0].year + edisgo_year = edisgo_ti[0].year + if csv_year != edisgo_year: + ts.index = ts.index + pd.DateOffset(years=edisgo_year - csv_year) + if isinstance(ts, pd.Series): + setattr(edisgo.overlying_grid, attr, ts.reindex(edisgo_ti)) + else: + setattr(edisgo.overlying_grid, attr, ts.reindex(edisgo_ti)) + + # load dispatchable generator and renewables time series from the same dir + disp_path = os.path.join( + overlying_grid_path, "dispatchable_generators_active_power.csv" + ) + if os.path.isfile(disp_path): + disp_ts = pd.read_csv(disp_path, index_col=0, parse_dates=True) + if not edisgo_ti.empty: + csv_year = disp_ts.index[0].year + edisgo_year = edisgo_ti[0].year + if csv_year != edisgo_year: + disp_ts.index = disp_ts.index + pd.DateOffset( + years=edisgo_year - csv_year + ) + disp_ts = disp_ts.reindex(edisgo_ti) + else: + disp_ts = None + + pot_path = os.path.join(overlying_grid_path, "renewables_potential.csv") + if os.path.isfile(pot_path): + pot_ts = pd.read_csv(pot_path, index_col=0, parse_dates=True) + if not edisgo_ti.empty: + csv_year = pot_ts.index[0].year + edisgo_year = edisgo_ti[0].year + if csv_year != edisgo_year: + pot_ts.index = pot_ts.index + pd.DateOffset( + years=edisgo_year - csv_year + ) + pot_ts = pot_ts.reindex(edisgo_ti) + else: + pot_ts = None + + if disp_ts is not None or pot_ts is not None: + edisgo.set_time_series_active_power_predefined( + dispatchable_generators_ts=disp_ts, + fluctuating_generators_ts=pot_ts, + ) + + return edisgo diff --git a/edisgo/run/tasks/timeseries.py b/edisgo/run/tasks/timeseries.py index f738aa056..cf918c235 100644 --- a/edisgo/run/tasks/timeseries.py +++ b/edisgo/run/tasks/timeseries.py @@ -12,6 +12,7 @@ control — this MUST come last because it overwrites whatever reactive power was set by the earlier steps. """ + from __future__ import annotations import pandas as pd @@ -20,9 +21,15 @@ @register_task("worst_case_ts") -def task_worst_case_ts(edisgo, ctx, *, cases=None, - generators_names=None, loads_names=None, - storage_units_names=None): +def task_worst_case_ts( + edisgo, + ctx, + *, + cases=None, + generators_names=None, + loads_names=None, + storage_units_names=None, +): """ Set synthetic worst-case active-power time series. @@ -62,8 +69,7 @@ def task_worst_case_ts(edisgo, ctx, *, cases=None, @register_task("set_timeindex") -def task_set_timeindex(edisgo, ctx, *, start, periods=None, end=None, - freq="h"): +def task_set_timeindex(edisgo, ctx, *, start, periods=None, end=None, freq="h"): """ Set the time index on the EDisGo object. @@ -98,22 +104,32 @@ def task_set_timeindex(edisgo, ctx, *, start, periods=None, end=None, If neither ``periods`` nor ``end`` is provided. """ + from edisgo.tools.tools import reduce_timeseries_data_to_given_timeindex + if end is not None: timeindex = pd.date_range(start=start, end=end, freq=freq) else: if periods is None: - raise ValueError( - "set_timeindex needs either 'periods' or 'end'." - ) + raise ValueError("set_timeindex needs either 'periods' or 'end'.") timeindex = pd.date_range(start=start, periods=periods, freq=freq) - edisgo.set_timeindex(timeindex) + if edisgo.timeseries.timeindex.empty: + edisgo.set_timeindex(timeindex) + else: + reduce_timeseries_data_to_given_timeindex(edisgo, timeindex) return edisgo @register_task("oedb_ts") -def task_oedb_ts(edisgo, ctx, *, timeindex=None, dispatchable=None, - fluctuating="oedb", conventional_loads="oedb", - charging_points_ts=None): +def task_oedb_ts( + edisgo, + ctx, + *, + timeindex=None, + dispatchable=None, + fluctuating="oedb", + conventional_loads="oedb", + charging_points_ts=None, +): """ Set active-power time series from egon_data (OEP) plus overrides. @@ -174,9 +190,7 @@ def task_oedb_ts(edisgo, ctx, *, timeindex=None, dispatchable=None, conv_loads_names = None if conventional_loads == "oedb": conv_loads_names = edisgo.topology.loads_df.loc[ - ~edisgo.topology.loads_df.type.isin( - ["heat_pump", "charging_point"] - ) + ~edisgo.topology.loads_df.type.isin(["heat_pump", "charging_point"]) ].index.tolist() edisgo.set_time_series_active_power_predefined( @@ -186,27 +200,34 @@ def task_oedb_ts(edisgo, ctx, *, timeindex=None, dispatchable=None, dispatchable_generators_ts=dispatchable_df, charging_points_ts=charging_points_ts, scenario=ctx.scenario, - engine=ctx.ensure_engine() if fluctuating == "oedb" - or conventional_loads == "oedb" else None, + engine=ctx.ensure_engine() + if fluctuating == "oedb" or conventional_loads == "oedb" + else None, ) su_names = edisgo.topology.storage_units_df.index if len(su_names) > 0 and edisgo.timeseries.storage_units_active_power.empty: edisgo.timeseries.storage_units_active_power = pd.DataFrame( - 0.0, index=edisgo.timeseries.timeindex, columns=su_names, + 0.0, + index=edisgo.timeseries.timeindex, + columns=su_names, ) ctx.flags["timeseries_set"] = True return edisgo @register_task("manual_ts") -def task_manual_ts(edisgo, ctx, *, - generators_active_power=None, - generators_reactive_power=None, - loads_active_power=None, - loads_reactive_power=None, - storage_units_active_power=None, - storage_units_reactive_power=None): +def task_manual_ts( + edisgo, + ctx, + *, + generators_active_power=None, + generators_reactive_power=None, + loads_active_power=None, + loads_reactive_power=None, + storage_units_active_power=None, + storage_units_reactive_power=None, +): """ Set active/reactive power time series from explicit DataFrames. @@ -240,6 +261,7 @@ def task_manual_ts(edisgo, ctx, *, The modified EDisGo instance. """ + def _as_df(obj): return pd.DataFrame(obj) if obj is not None else None @@ -256,10 +278,15 @@ def _as_df(obj): @register_task("reactive_power") -def task_reactive_power(edisgo, ctx, *, control="fixed_cosphi", - generators_parametrisation="default", - loads_parametrisation="default", - storage_units_parametrisation="default"): +def task_reactive_power( + edisgo, + ctx, + *, + control="fixed_cosphi", + generators_parametrisation="default", + loads_parametrisation="default", + storage_units_parametrisation="default", +): """ Apply reactive-power control on top of the active-power time series. From b55e5a03f841cdec2366d8f2177158ea3174f2b5 Mon Sep 17 00:00:00 2001 From: Moritz Schloesser Date: Thu, 21 May 2026 16:26:21 +0200 Subject: [PATCH 09/37] Fix OPF result write-back, flex resolution and overlying-grid SOC reindex MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit powermodels_io.from_powermodels now slices the destination time index explicitly when writing OPF flex results back to the EDisGo timeseries (gen_nd, heatpumps, electromobility, dsm, storage). Using `loc[:, names]` overwrites every row of the underlying DataFrame even when the OPF only covered a subset of timesteps; restricting to `timeseries.timeindex` keeps untouched rows intact. task_optimize: - fixes typo `flexbile` → `flexible` in the dsm shortcut - corrects the dsm condition (was checking `is not None`, should populate when `None`) - materializes empty flex lists once instead of repeating `or []` at every call site task_import_overlying_grid_data reindexes the three SOC attributes (storage_units_soc, thermal_storage_units_{central,decentral}_soc) to timeindex + 1 extra step, because PowerModels expects the end-of-period SOC value; non-SOC overlying-grid attributes still reindex to the plain timeindex. uc4_example preset reworked: switches to opf_version 3 (HV constraints from overlying grid), drops base_reinforce + check_integrity from the pipeline, adds an explicit `overlying_grid.path` config slot, splits import_electromobility kwargs onto separate lines, and enables archive + save_opf_results on the final save step. --- edisgo/io/powermodels_io.py | 33 +++++++----- edisgo/run/presets/uc4_example.yaml | 47 ++++++++--------- edisgo/run/tasks/analysis.py | 79 +++++++++++++++++++++-------- edisgo/run/tasks/io.py | 14 +++-- 4 files changed, 108 insertions(+), 65 deletions(-) diff --git a/edisgo/io/powermodels_io.py b/edisgo/io/powermodels_io.py index e82cc4734..6bf21affb 100644 --- a/edisgo/io/powermodels_io.py +++ b/edisgo/io/powermodels_io.py @@ -250,10 +250,10 @@ def from_powermodels( Base value of apparent power for per unit system. Default: 1 MVA. """ - if type(pm_results) == str: + if isinstance(pm_results, str): with open(pm_results) as f: pm = json.loads(json.load(f)) - elif type(pm_results) == dict: + elif isinstance(pm_results, dict): pm = pm_results else: raise ValueError( @@ -306,17 +306,20 @@ def from_powermodels( ] results = pd.DataFrame(index=timesteps, columns=names, data=data) if (flex == "gen_nd") & (pm["nw"]["1"]["opf_version"] in [3, 4]): - edisgo_object.timeseries._generators_active_power.loc[:, names] = ( + ti = edisgo_object.timeseries.timeindex + edisgo_object.timeseries._generators_active_power.loc[ti, names] = ( edisgo_object.timeseries.generators_active_power.loc[:, names].values - results[names].values ) elif flex in ["heatpumps", "electromobility"]: - edisgo_object.timeseries._loads_active_power.loc[:, names] = results[ + ti = edisgo_object.timeseries.timeindex + edisgo_object.timeseries._loads_active_power.loc[ti, names] = results[ names ].values elif flex == "dsm": - edisgo_object.timeseries._loads_active_power.loc[:, names] = ( - edisgo_object.timeseries._loads_active_power.loc[:, names].values + ti = edisgo_object.timeseries.timeindex + edisgo_object.timeseries._loads_active_power.loc[ti, names] = ( + edisgo_object.timeseries._loads_active_power.loc[ti, names].values + results[names].values ) elif flex == "storage": @@ -328,8 +331,9 @@ def from_powermodels( data=results[names].values, ) else: + ti = edisgo_object.timeseries.timeindex edisgo_object.timeseries._storage_units_active_power.loc[ - :, names + ti, names ] = results[names].values except AttributeError: setattr( @@ -787,8 +791,8 @@ def _build_branch(edisgo_obj, psa_net, pm, flexible_storage_units, s_base): # only modify r, x and l values if min value is too small branches[par] = val.clip(lower=min_value) logger.warning( - f"Min value of {text} is too small. Lowest {100 * quant}% of {text} values will be set " - f"to {min_value} {unit}" + f"Min value of {text} is too small. Lowest {100 * quant}% of " + f"{text} values will be set to {min_value} {unit}" ) for branch_i in np.arange(len(branches.index)): @@ -933,8 +937,8 @@ def _build_load( pf, sign = _get_pf(edisgo_obj, pm, idx_bus, "charging_point") else: logger.warning( - f"No type specified for load {loads_df.index[load_i]}. Power factor and sign will" - "be set for conventional load." + f"No type specified for load {loads_df.index[load_i]}. " + "Power factor and sign will be set for conventional load." ) pf, sign = _get_pf(edisgo_obj, pm, idx_bus, "conventional_load") p_d = psa_net.loads_t.p_set[loads_df.index[load_i]] @@ -1219,9 +1223,10 @@ def _build_heatpump(psa_net, pm, edisgo_obj, s_base, flexible_hps): comparison = (heat_df2[hp_p_nom.index] > hp_cop * hp_p_nom.squeeze()).any() if comparison.any(): logger.warning( - "Heat demand is higher than rated heatpump power" - f" of heatpumps: {comparison.index[comparison.values].values}. Demand can not be covered if no sufficient" - " heat storage capacities are available." + "Heat demand is higher than rated heatpump power of heatpumps: " + f"{comparison.index[comparison.values].values}. " + "Demand can not be covered if no sufficient heat storage " + "capacities are available." ) for hp_i in np.arange(len(heat_df.index)): idx_bus = _mapping(psa_net, edisgo_obj, heat_df.bus.iloc[hp_i]) diff --git a/edisgo/run/presets/uc4_example.yaml b/edisgo/run/presets/uc4_example.yaml index 2d2650279..d3f9efd90 100644 --- a/edisgo/run/presets/uc4_example.yaml +++ b/edisgo/run/presets/uc4_example.yaml @@ -1,54 +1,51 @@ _comment: | - UC3 — OPF with full flexibility: - Like UC1 but loads real egon_data time series (oedb) and runs a - powermodels OPF over flexibilities (heat pumps, EV, DSM, storage) - before the final reinforce. Cost delta = extra reinforcement needed - under optimal flex dispatch. + UC4 — OPF with full flexibility: + Loads real egon_data time series (oedb) and runs a powermodels OPF + over flexibilities (heat pumps, EV, DSM, storage) with HV requirements + from overlying grid. opf_version 3 activates HV-constraints from + overlying_grid CSV directory. _workflow: - - setup_grid: load ding0 topology, import generators - - base_reinforce: worst-case TS + reinforce + reset equipment_changes - - import_generators: from edon-data - - import_heat_pumps: from egon_data + - setup_grid: load ding0 topology + - import_generators: from egon_data - import_home_batteries: from egon_data + - import_heat_pumps: from egon_data - import_dsm: from egon_data - import_electromobility: from egon_data (dumb charging, flex bands) - oedb_ts: real wind/solar + load time series (24 h, 2035) + - apply_charging_strategy: dumb - apply_heat_pump_strategy: uncontrolled (overwritten by OPF) - - reactive_power - - check_integrity - - optimize: pm_optimize with flex assets (SOC, opf v2) - - reinforce: final reinforcement - - save + - import_overlying_grid_data: HV constraints from CSV dir + - optimize: pm_optimize with flex assets (SOC, opf v3 = HV constraints) scenario: eGon2035 + grid: ding0_path: "/path/to/ding0_grid" - # grid path should be set in run file legacy_ding0_grids: false - # legacy parameter too database: ssh: enabled: false timeindex: {start: "2035-01-01", periods: 24, freq: h} -# only set in preset, not set in run-functions (eDisGo and eGo) + +overlying_grid: + path: "/path/to/overlying_grid_csv_dir" results: directory: results/uc4_example -# actually all parameters pipeline: - setup_grid - - base_reinforce - import_generators - import_home_batteries - import_heat_pumps - import_dsm - # where is decided, which electromobility use cases are used? - - import_electromobility: {charging_strategy: null, flexibility_bands_ucs : ["home", "work", "public", "hpc"]} + - import_electromobility: + charging_strategy: null + flexibility_bands_ucs: ["home", "work", "public", "hpc"] - oedb_ts: dispatchable: {other: 0.7} timeindex: {start: "2035-01-01", periods: 24, freq: h} @@ -56,11 +53,11 @@ pipeline: - apply_heat_pump_strategy: {strategy: uncontrolled} - import_overlying_grid_data - reactive_power - - check_integrity - optimize: - # where is decided which flexibilities are used in the OPF? flexible: [heat_pumps, storage, charging_points, dsm] method: soc - opf_version: 2 + opf_version: 3 - reinforce - - save + - save: + archive: true + save_opf_results: true diff --git a/edisgo/run/tasks/analysis.py b/edisgo/run/tasks/analysis.py index f028bb31c..becff93b9 100644 --- a/edisgo/run/tasks/analysis.py +++ b/edisgo/run/tasks/analysis.py @@ -21,6 +21,7 @@ produce a "base" grid whose subsequent reinforce costs reflect only a scenario overlay. """ + from __future__ import annotations import pandas as pd @@ -55,8 +56,15 @@ def task_check_integrity(edisgo, ctx): @register_task("analyze") -def task_analyze(edisgo, ctx, *, mode=None, timesteps=None, - raise_not_converged=False, troubleshooting_mode=None): +def task_analyze( + edisgo, + ctx, + *, + mode=None, + timesteps=None, + raise_not_converged=False, + troubleshooting_mode=None, +): """ Run AC power flow over the active time series. @@ -97,18 +105,26 @@ def task_analyze(edisgo, ctx, *, mode=None, timesteps=None, ctx.flags["not_converged_steps"] = len(not_converged) if len(not_converged) > 0: ctx.logger.warning( - f"Power flow did not converge for {len(not_converged)} " - f"time steps." + f"Power flow did not converge for {len(not_converged)} time steps." ) return edisgo @register_task("reinforce") -def task_reinforce(edisgo, ctx, *, timesteps_pfa=None, reduced_analysis=False, - copy_grid=False, max_while_iterations=20, - split_voltage_band=True, mode=None, - without_generator_import=False, n_minus_one=False, - catch_convergence_problems=False): +def task_reinforce( + edisgo, + ctx, + *, + timesteps_pfa=None, + reduced_analysis=False, + copy_grid=False, + max_while_iterations=20, + split_voltage_band=True, + mode=None, + without_generator_import=False, + n_minus_one=False, + catch_convergence_problems=False, +): """ Run iterative grid reinforcement. @@ -167,8 +183,9 @@ def task_reinforce(edisgo, ctx, *, timesteps_pfa=None, reduced_analysis=False, @register_task("base_reinforce") -def task_base_reinforce(edisgo, ctx, *, cases=None, - reset_equipment_changes=True, save_artifact=True): +def task_base_reinforce( + edisgo, ctx, *, cases=None, reset_equipment_changes=True, save_artifact=True +): """ Produce a base-reinforced grid and reset the cost accumulator. @@ -242,10 +259,20 @@ def task_base_reinforce(edisgo, ctx, *, cases=None, @register_task("optimize") -def task_optimize(edisgo, ctx, *, flexible=None, flexible_cps=None, - flexible_hps=None, flexible_loads=None, - flexible_storage_units=None, opf_version=2, method="soc", - warm_start=False, s_base=1): +def task_optimize( + edisgo, + ctx, + *, + flexible=None, + flexible_cps=None, + flexible_hps=None, + flexible_loads=None, + flexible_storage_units=None, + opf_version=2, + method="soc", + warm_start=False, + s_base=1, +): """ Run a powermodels optimal-power-flow (OPF) over flexibilities. @@ -304,15 +331,23 @@ def task_optimize(edisgo, ctx, *, flexible=None, flexible_cps=None, ].index.tolist() if flexible_storage_units is None and "storage" in flexible: flexible_storage_units = edisgo.topology.storage_units_df.index.tolist() - if flexible_loads is not None and "dsm" in flexbile: - flexible_loads = edisgo.dsm.p_min.columns.values - + if flexible_loads is None and "dsm" in flexible: + flexible_loads = edisgo.dsm.p_min.columns.values + + if flexible_cps is None: + flexible_cps = [] + if flexible_hps is None: + flexible_hps = [] + if flexible_loads is None: + flexible_loads = [] + if flexible_storage_units is None: + flexible_storage_units = [] edisgo.pm_optimize( - flexible_cps=flexible_cps or [], - flexible_hps=flexible_hps or [], - flexible_loads=flexible_loads or [], - flexible_storage_units=flexible_storage_units or [], + flexible_cps=flexible_cps, + flexible_hps=flexible_hps, + flexible_loads=flexible_loads, + flexible_storage_units=flexible_storage_units, opf_version=opf_version, method=method, warm_start=warm_start, diff --git a/edisgo/run/tasks/io.py b/edisgo/run/tasks/io.py index 4e2e84d15..c2b748cf0 100644 --- a/edisgo/run/tasks/io.py +++ b/edisgo/run/tasks/io.py @@ -262,6 +262,14 @@ def task_import_overlying_grid_data(edisgo, ctx, *, overlying_grid_path=None): # CSVs may use a different year — shift year then reindex edisgo_ti = edisgo.timeseries.timeindex if not edisgo_ti.empty: + # SOC needs one extra step at the end (end-of-period state) + ti_freq = edisgo_ti.freq or (edisgo_ti[1] - edisgo_ti[0]) + edisgo_ti_plus1 = edisgo_ti.union([edisgo_ti[-1] + ti_freq]) + soc_attrs = { + "storage_units_soc", + "thermal_storage_units_decentral_soc", + "thermal_storage_units_central_soc", + } for attr in edisgo.overlying_grid._attributes: ts = getattr(edisgo.overlying_grid, attr) if ts.empty: @@ -270,10 +278,8 @@ def task_import_overlying_grid_data(edisgo, ctx, *, overlying_grid_path=None): edisgo_year = edisgo_ti[0].year if csv_year != edisgo_year: ts.index = ts.index + pd.DateOffset(years=edisgo_year - csv_year) - if isinstance(ts, pd.Series): - setattr(edisgo.overlying_grid, attr, ts.reindex(edisgo_ti)) - else: - setattr(edisgo.overlying_grid, attr, ts.reindex(edisgo_ti)) + target_ti = edisgo_ti_plus1 if attr in soc_attrs else edisgo_ti + setattr(edisgo.overlying_grid, attr, ts.reindex(target_ti)) # load dispatchable generator and renewables time series from the same dir disp_path = os.path.join( From bb3a720a09e78db8693bac72322e816a2b6f8694 Mon Sep 17 00:00:00 2001 From: Jonas Danke Date: Wed, 13 May 2026 16:15:42 +0200 Subject: [PATCH 10/37] Changes by JD before edits of MS --- edisgo/edisgo.py | 21 + edisgo/run/__init__.py | 31 ++ edisgo/run/config.py | 410 ++++++++++++++++++ edisgo/run/context.py | 115 +++++ edisgo/run/presets/basic.yaml | 22 + .../run/presets/r4mu_base_and_scenario.yaml | 49 +++ edisgo/run/presets/uc1_loads_worst_case.yaml | 32 ++ edisgo/run/presets/uc2_flex_opf.yaml | 43 ++ edisgo/run/presets/uc3_oedb_ts.yaml | 36 ++ edisgo/run/registry.py | 113 +++++ edisgo/run/runner.py | 261 +++++++++++ edisgo/run/tasks/__init__.py | 25 ++ edisgo/run/tasks/io.py | 179 ++++++++ edisgo/run/validator.py | 201 +++++++++ setup.py | 1 + tests/run/__init__.py | 1 + tests/run/test_config.py | 154 +++++++ tests/run/test_registry.py | 35 ++ tests/run/test_runner.py | 118 +++++ tests/run/test_validator.py | 97 +++++ 20 files changed, 1944 insertions(+) create mode 100644 edisgo/run/__init__.py create mode 100644 edisgo/run/config.py create mode 100644 edisgo/run/context.py create mode 100644 edisgo/run/presets/basic.yaml create mode 100644 edisgo/run/presets/r4mu_base_and_scenario.yaml create mode 100644 edisgo/run/presets/uc1_loads_worst_case.yaml create mode 100644 edisgo/run/presets/uc2_flex_opf.yaml create mode 100644 edisgo/run/presets/uc3_oedb_ts.yaml create mode 100644 edisgo/run/registry.py create mode 100644 edisgo/run/runner.py create mode 100644 edisgo/run/tasks/__init__.py create mode 100644 edisgo/run/tasks/io.py create mode 100644 edisgo/run/validator.py create mode 100644 tests/run/__init__.py create mode 100644 tests/run/test_config.py create mode 100644 tests/run/test_registry.py create mode 100644 tests/run/test_runner.py create mode 100644 tests/run/test_validator.py diff --git a/edisgo/edisgo.py b/edisgo/edisgo.py index ed6ed376d..80579bf48 100755 --- a/edisgo/edisgo.py +++ b/edisgo/edisgo.py @@ -232,6 +232,27 @@ def config(self): def config(self, kwargs): self._config = Config(**kwargs) + def run_pipeline(self, config): + """ + Run a YAML/JSON task pipeline on this EDisGo instance. + + See :mod:`edisgo.run` for the config schema and task list. + + Parameters + ---------- + config : str, :class:`pathlib.Path`, or dict + Pipeline config as path to a YAML/JSON file or as a dict. + + Returns + ------- + :class:`~.EDisGo` + The EDisGo instance after the pipeline has run. + + """ + from edisgo.run import _run_pipeline_on + + return _run_pipeline_on(self, config) + def import_ding0_grid(self, path, legacy_ding0_grids=True): """ Import ding0 topology data from csv files in the format as diff --git a/edisgo/run/__init__.py b/edisgo/run/__init__.py new file mode 100644 index 000000000..cd202ddef --- /dev/null +++ b/edisgo/run/__init__.py @@ -0,0 +1,31 @@ +""" +YAML/JSON-driven pipeline runner for eDisGo. + +Two entry points share the same core: + + from edisgo.run import run_edisgo + edisgo = run_edisgo("presets/uc2_flex_opf.yaml") + + # or, on an existing EDisGo instance: + edisgo = EDisGo(ding0_grid="30879") + edisgo.run_pipeline("my_run.yaml") + +Pipelines are lists of named tasks from :mod:`edisgo.run.tasks`. Each step +is either a string (``worst_case_ts``) or a single-key mapping with +parameters (``import_electromobility: {charging_strategy: dumb}``). Tasks +can be grouped into ordered ``stages`` that can save artifacts and reload +them with ``load_from``, enabling two-phase workflows (base reinforce + +per-scenario reinforce). +""" + +from edisgo.run.context import RunContext +from edisgo.run.registry import known_tasks, register_task +from edisgo.run.runner import _run_pipeline_on, run_edisgo + +__all__ = [ + "RunContext", + "_run_pipeline_on", + "known_tasks", + "register_task", + "run_edisgo", +] diff --git a/edisgo/run/config.py b/edisgo/run/config.py new file mode 100644 index 000000000..5c4ee8573 --- /dev/null +++ b/edisgo/run/config.py @@ -0,0 +1,410 @@ +""" +Config loader and schema normalizer for the eDisGo pipeline runner. + +The loader turns a YAML file, JSON file, or Python dict into the +canonical internal schema consumed by :mod:`edisgo.run.runner`. It +handles four concerns in a fixed order: + +1. **Read** — parse YAML/JSON (auto-detected by extension; unknown + extensions are tried as JSON first, then YAML). +2. **extends** — resolve a ``extends:`` key recursively into the + parent config and deep-merge; the child overrides parent keys. The + ``extends:`` value may be a path (relative to the including file) + or a bare preset name (resolved against + :mod:`edisgo.run.presets`). +3. **external_config** — merge machine-specific overrides from an + ``external_config:`` path (typically ``~/.edisgo/secrets.json`` + with DB credentials). Keys in the external file override keys in + the main config. +4. **eGo-legacy adaptation** — if the config looks like an eGo + ``scenario_setting_*.json`` (has top-level ``eDisGo.tasks``), map + it onto the new schema so old eGo configs run unchanged. +5. **Stage normalization** — collapse a flat ``pipeline:`` into a + single-stage ``stages: [{name: main, pipeline: [...]}]`` so the + runner only ever deals with the stage form. + +Only :func:`load_config` is public. Everything else is implementation +detail. +""" +from __future__ import annotations + +import copy +import json +import logging +import os + +from pathlib import Path +from typing import Any + +import yaml + +logger = logging.getLogger("edisgo.run.config") + + +def load_config(cfg_or_path) -> dict[str, Any]: + """ + Load, merge, adapt, and normalize a pipeline config. + + Accepts a path to a YAML/JSON file or a dict. The returned dict + always has the normalized shape expected by the runner: + + * top-level ``stages`` (list of ``{name, pipeline, ...}``) + * ``scenario`` (may be ``None``) + * optional ``grid``, ``database``, ``results`` sections + * no ``pipeline``, ``extends``, or ``external_config`` keys + (they have been consumed) + + Parameters + ---------- + cfg_or_path : str, pathlib.Path, or dict + Either a path to a YAML/JSON config file, or a dict already + holding the config. A dict is deep-copied so the caller's + dict is not mutated. + + Returns + ------- + dict + The fully resolved, normalized config. + + Raises + ------ + FileNotFoundError + If the given path (or an ``extends`` reference) does not + exist. + ValueError + If the config has both ``pipeline`` and ``stages``, missing + ``pipeline``/``stages``, duplicate stage names, or a stage + without ``name``/``pipeline``. + + """ + if isinstance(cfg_or_path, (dict,)): + cfg = copy.deepcopy(cfg_or_path) + base_dir = Path.cwd() + else: + path = Path(cfg_or_path).expanduser().resolve() + if not path.is_file(): + raise FileNotFoundError(f"Config file not found: {path}") + cfg = _read_file(path) + base_dir = path.parent + + cfg = _resolve_extends(cfg, base_dir) + cfg = _apply_external_config(cfg) + cfg = _adapt_ego_legacy(cfg) + cfg = _normalize_stages(cfg) + return cfg + + +def _read_file(path: Path) -> dict[str, Any]: + """ + Parse a YAML or JSON file into a dict. + + Parameters + ---------- + path : pathlib.Path + File path. Extension (``.json``, ``.yaml``, ``.yml``) selects + the parser. Unknown extensions fall back to JSON first, then + YAML. + + Returns + ------- + dict + Parsed config contents. + + """ + text = path.read_text() + suffix = path.suffix.lower() + if suffix == ".json": + return json.loads(text) + if suffix in (".yaml", ".yml"): + return yaml.safe_load(text) + try: + return json.loads(text) + except json.JSONDecodeError: + return yaml.safe_load(text) + + +def _resolve_extends(cfg: dict, base_dir: Path) -> dict: + """ + Resolve an ``extends:`` reference and deep-merge parent into child. + + The parent is loaded recursively, so a chain of ``extends:`` works. + References are looked up as (1) a bundled preset name under + :mod:`edisgo.run.presets`, (2) a path relative to ``base_dir``. + The child's keys override the parent's on conflicts. + + Parameters + ---------- + cfg : dict + Child config (may contain ``extends:``). + base_dir : pathlib.Path + Directory against which relative ``extends`` paths are + resolved (usually the directory of the child config). + + Returns + ------- + dict + Merged config with ``extends`` consumed. + + Raises + ------ + FileNotFoundError + If the referenced parent config file does not exist. + + """ + ext = cfg.pop("extends", None) + if ext is None: + return cfg + ext_path = Path(ext).expanduser() + if not ext_path.is_absolute(): + preset_path = _preset_path(str(ext_path)) + if preset_path is not None: + ext_path = preset_path + else: + ext_path = (base_dir / ext_path).resolve() + if not ext_path.is_file(): + raise FileNotFoundError(f"extends: file not found: {ext_path}") + parent = _read_file(ext_path) + parent = _resolve_extends(parent, ext_path.parent) + return _deep_merge(parent, cfg) + + +def _preset_path(name: str) -> Path | None: + """ + Look up a preset YAML/JSON by bare name. + + Searches the ``edisgo/run/presets/`` directory for a file matching + ``name``, ``name.yaml``, ``name.yml``, or ``name.json`` (in that + order). + + Parameters + ---------- + name : str + Preset identifier, e.g. ``"uc2_flex_opf"`` or + ``"presets/uc2_flex_opf.yaml"``. + + Returns + ------- + pathlib.Path or None + The resolved preset path, or ``None`` if no match is found. + + """ + presets_dir = Path(__file__).parent / "presets" + candidates = [ + presets_dir / name, + presets_dir / f"{name}.yaml", + presets_dir / f"{name}.yml", + presets_dir / f"{name}.json", + ] + for c in candidates: + if c.is_file(): + return c + return None + + +def _apply_external_config(cfg: dict) -> dict: + """ + Merge an ``external_config:`` file on top of the current config. + + Used to keep machine-specific secrets (DB credentials, result + directories) out of versioned scenario configs. If the referenced + file does not exist, a warning is logged but the config is used + as-is. + + Parameters + ---------- + cfg : dict + Config possibly containing an ``external_config:`` key. + + Returns + ------- + dict + Merged config with ``external_config`` consumed. + + """ + ext = cfg.pop("external_config", None) + if ext is None: + return cfg + path = Path(os.path.expanduser(ext)) + if not path.is_file(): + logger.warning(f"external_config file not found, skipping: {path}") + return cfg + override = _read_file(path) + return _deep_merge(cfg, override) + + +def _deep_merge(base: dict, override: dict) -> dict: + """ + Recursively merge two dicts, with ``override`` winning on conflicts. + + Nested dicts are merged key-by-key. Non-dict values (including + lists) are replaced wholesale — lists are NOT concatenated, to + keep the merge semantics predictable (otherwise a preset could + silently extend the child's pipeline). + + Parameters + ---------- + base : dict + Parent / lower-priority dict. + override : dict + Child / higher-priority dict. + + Returns + ------- + dict + A new dict holding the merge result. Inputs are not mutated. + + """ + out = copy.deepcopy(base) if base else {} + for key, val in (override or {}).items(): + if ( + key in out + and isinstance(out[key], dict) + and isinstance(val, dict) + ): + out[key] = _deep_merge(out[key], val) + else: + out[key] = copy.deepcopy(val) + return out + + +def _normalize_stages(cfg: dict) -> dict: + """ + Collapse a flat ``pipeline:`` into the canonical ``stages`` shape. + + After this step the runner only has to iterate ``cfg["stages"]``; + flat configs become a single stage named ``main``. + + Parameters + ---------- + cfg : dict + Config with either ``pipeline`` or ``stages`` at the top + level. + + Returns + ------- + dict + Config with ``stages`` guaranteed to be present and + ``pipeline`` removed. + + Raises + ------ + ValueError + If both ``pipeline`` and ``stages`` are present, if neither + is present, if any stage is missing ``name``/``pipeline``, or + if stage names are not unique. + + """ + if "stages" in cfg and "pipeline" in cfg: + raise ValueError( + "Config has both top-level 'pipeline' and 'stages'. " + "Use only one." + ) + if "stages" not in cfg: + pipeline = cfg.pop("pipeline", None) + if pipeline is None: + raise ValueError( + "Config must define either 'pipeline' or 'stages'." + ) + cfg["stages"] = [{"name": "main", "pipeline": pipeline}] + + seen = set() + for stage in cfg["stages"]: + if "name" not in stage: + raise ValueError("Every stage needs a 'name' key.") + if stage["name"] in seen: + raise ValueError( + f"Duplicate stage name: {stage['name']}" + ) + seen.add(stage["name"]) + if "pipeline" not in stage: + raise ValueError( + f"Stage '{stage['name']}' is missing 'pipeline'." + ) + return cfg + + +_EGO_TASK_MAP = { + "1_setup_grid": "setup_grid", + "5_grid_reinforcement": "reinforce", + "4_optimisation": "optimize", + "worst_case_ts": "worst_case_ts", + "base_reinforce": "base_reinforce", + "oedb_ts": "oedb_ts", + "import_heat_pumps_from_db": "import_heat_pumps", + "import_home_batteries_from_db": "import_home_batteries", + "import_dsm_from_db": "import_dsm", + "import_electromobility_from_db": "import_electromobility", + "load_charging_from_files": "load_charging_from_files", + "load_from_base": "load_from_base", +} +"""Mapping from eGo task names to edisgo.run task names. eGo-specific +tasks with no eDisGo equivalent (e.g. ``2_specs_overlying_grid``, +``3_temporal_complexity_reduction``) are intentionally missing — they +require eTraGo and are logged as "skipped" when adapted.""" + + +def _adapt_ego_legacy(cfg: dict) -> dict: + """ + Map an eGo-style ``scenario_setting_*.json`` onto the new schema. + + Recognizes an eGo config by the presence of an ``eDisGo.tasks`` + key at the top level together with the absence of + ``pipeline``/``stages``. Translates: + + * ``eDisGo.grid_path`` → ``grid.ding0_path`` + * ``eDisGo.results`` → ``results.directory`` + * ``eTraGo.scn_name`` → ``scenario`` + * ``eDisGo.tasks`` → ``pipeline`` (via :data:`_EGO_TASK_MAP`) + * top-level ``database``/``ssh`` kept under ``database`` + + eGo-only tasks (overlying grid / temporal reduction) are + dropped with a warning. Cosmetic keys (``eGo``, ``eTraGo``, + ``_comment``, ``_workflow``) are stripped. + + Parameters + ---------- + cfg : dict + Possibly-legacy config. + + Returns + ------- + dict + Adapted config. If the input is not an eGo-legacy config, it + is returned unchanged. + + """ + if "eDisGo" not in cfg or "pipeline" in cfg or "stages" in cfg: + return cfg + + edisgo_cfg = cfg["eDisGo"] + tasks = edisgo_cfg.get("tasks") + if tasks is None: + return cfg + + logger.info( + "Detected legacy eGo config schema — adapting to edisgo.run." + ) + mapped = [] + for t in tasks: + if t not in _EGO_TASK_MAP: + logger.warning( + f"eGo task '{t}' has no eDisGo equivalent — skipping " + "(likely eTraGo-specific)." + ) + continue + mapped.append(_EGO_TASK_MAP[t]) + + adapted: dict[str, Any] = { + "scenario": cfg.get("eTraGo", {}).get("scn_name", "eGon2035"), + "grid": {"ding0_path": edisgo_cfg.get("grid_path")}, + "results": {"directory": edisgo_cfg.get("results")}, + "pipeline": mapped, + } + if "database" in cfg: + adapted["database"] = cfg["database"] + if "ssh" in cfg: + adapted["database"]["ssh"] = cfg["ssh"] + for side_key in ("eGo", "eTraGo", "ssh", "_comment", "_workflow"): + cfg.pop(side_key, None) + cfg.pop("eDisGo", None) + return _deep_merge(adapted, cfg) diff --git a/edisgo/run/context.py b/edisgo/run/context.py new file mode 100644 index 000000000..c2fbce234 --- /dev/null +++ b/edisgo/run/context.py @@ -0,0 +1,115 @@ +""" +Runtime context passed to every task during pipeline execution. + +The context is a small mutable object that threads shared state between +tasks without polluting the :class:`~edisgo.EDisGo` instance itself. +Typical uses: + +* ``scenario`` — the active eGon scenario name (``eGon2035``, + ``eGon100RE``, …) so tasks don't have to re-read it from the config. +* ``engine`` — a SQLAlchemy engine, lazily created on first DB access + via :meth:`RunContext.ensure_engine`. Tasks that don't touch the + database never pay connection cost. +* ``results_dir`` — base directory for stage artifacts and ``save``. +* ``flags`` — free-form boolean/state flags tasks set to coordinate + with each other (``has_heat_pumps``, ``timeseries_set``, …). +* ``stage_artifacts`` — map ``stage_name -> path`` of zip/dir artifacts + emitted by ``save``, consumed by later stages via ``load_from``. + +Tasks should treat ``flags`` as advisory — they MAY short-circuit based +on a flag but MUST NOT assume a flag is present. +""" +from __future__ import annotations + +import logging + +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + + +@dataclass +class RunContext: + """ + Mutable per-run state shared across all tasks of a pipeline. + + Attributes + ---------- + scenario : str or None + Active scenario name from the top-level ``scenario:`` key. + engine : sqlalchemy.engine.Engine or None + Database engine for oedb-backed imports. Created lazily; + see :meth:`ensure_engine`. + results_dir : pathlib.Path or None + Base directory for stage outputs. Resolved from + ``results.directory`` in the config. + logger : logging.Logger + Logger instance used by tasks and the runner. Defaults to + the ``edisgo.run`` logger. + flags : dict + Free-form state flags that tasks use to communicate. Common + keys: ``grid_loaded``, ``timeseries_set``, + ``reactive_power_set``, ``has_heat_pumps``, ``has_dsm``, + ``has_home_batteries``, ``has_electromobility``, + ``base_reinforced``, ``last_saved``. + stage_artifacts : dict + Map ``stage_name -> Path`` of save-artifacts. Populated by the + ``save`` task when running inside a named stage, consumed by + subsequent stages that set ``load_from:``. + current_stage : str or None + Name of the stage currently executing. Set by the runner. + raw_config : dict + The fully resolved pipeline config (after ``extends``, + ``external_config``, and eGo-legacy adaptation). Tasks can + read supplementary keys like ``database.*`` from here. + + """ + + scenario: str | None = None + engine: Any = None + results_dir: Path | None = None + logger: logging.Logger = field( + default_factory=lambda: logging.getLogger("edisgo.run") + ) + flags: dict[str, Any] = field(default_factory=dict) + stage_artifacts: dict[str, Path] = field(default_factory=dict) + current_stage: str | None = None + raw_config: dict[str, Any] = field(default_factory=dict) + + def ensure_engine(self): + """ + Return a database engine, creating it on first call. + + Reads the ``database`` section of :attr:`raw_config` and calls + :func:`edisgo.io.db.engine`. Caches the engine on the context + so subsequent calls reuse the same connection. + + Returns + ------- + sqlalchemy.engine.Engine + The active database engine. + + Raises + ------ + RuntimeError + If the config has no ``database`` section — indicates the + pipeline wants to reach the database without configuring + it. + + """ + if self.engine is not None: + return self.engine + db_cfg = self.raw_config.get("database") + if not db_cfg: + raise RuntimeError( + "Task needs a database engine but no 'database' section " + "is configured." + ) + from edisgo.io.db import engine as egon_engine + + ssh_cfg = db_cfg.get("ssh") or {} + self.engine = egon_engine( + path=db_cfg.get("credentials_path"), + ssh=bool(ssh_cfg.get("enabled", False)), + ) + return self.engine diff --git a/edisgo/run/presets/basic.yaml b/edisgo/run/presets/basic.yaml new file mode 100644 index 000000000..136161855 --- /dev/null +++ b/edisgo/run/presets/basic.yaml @@ -0,0 +1,22 @@ +_comment: | + Basic preset: worst-case pre-reinforce → reinforce. + Minimal end-to-end example with no database dependency. + Reproduces the core of example_01 without flex imports. + +_workflow: + - setup_grid: load ding0 topology + - worst_case_ts: set worst-case time series (feed-in + load) + - reactive_power: fix reactive power control + - check_integrity: validate grid consistency + - reinforce: run grid reinforcement + - save: persist topology + timeseries + results + +scenario: eGon2035 + +pipeline: + - setup_grid + - worst_case_ts + - reactive_power + - check_integrity + - reinforce + - save diff --git a/edisgo/run/presets/r4mu_base_and_scenario.yaml b/edisgo/run/presets/r4mu_base_and_scenario.yaml new file mode 100644 index 000000000..6412f36ca --- /dev/null +++ b/edisgo/run/presets/r4mu_base_and_scenario.yaml @@ -0,0 +1,49 @@ +_comment: | + R4MU — two-stage base + scenario reinforcement: + Stage 1 produces a base-reinforced grid (generators + heat pumps) + and saves it as an artifact. Stage 2 loads that artifact, integrates + scenario-specific charging stations from a GeoPackage/CSV directory, + applies worst-case time series, and runs a scenario-specific + reinforce. Cost delta = extra reinforcement caused by the charging + scenario. + +_workflow: + - stage base: + - setup_grid: load ding0 topology + import generators + - import_heat_pumps: from egon_data + - worst_case_ts + - reactive_power + - reinforce + - save (artifact consumed by next stage) + - stage scenario: + - load_from: base + - load_charging_from_files: integrate scenario charging + - worst_case_ts + - reactive_power + - reinforce (delta only) + - save + +scenario: eGon2035 + +stages: + - name: base + pipeline: + - setup_grid: {import_generators: true} + - import_heat_pumps + - worst_case_ts + - reactive_power + - reinforce + - save + - name: scenario + load_from: base + params: + charging_dir: "./charging_scenario_1" + mv_threshold_kw: 100 + pipeline: + - load_charging_from_files: + charging_dir: "{{params.charging_dir}}" + mv_threshold_kw: "{{params.mv_threshold_kw}}" + - worst_case_ts + - reactive_power + - reinforce + - save diff --git a/edisgo/run/presets/uc1_loads_worst_case.yaml b/edisgo/run/presets/uc1_loads_worst_case.yaml new file mode 100644 index 000000000..2204d3ce9 --- /dev/null +++ b/edisgo/run/presets/uc1_loads_worst_case.yaml @@ -0,0 +1,32 @@ +_comment: | + UC1 — worst-case flexibility loads: + load grid, base-reinforce (generators only), then import flex assets + (heat pumps, home batteries, DSM, electromobility) and apply worst-case + time series before a final reinforce. Cost delta = extra reinforcement + caused by the new assets under worst-case conditions. + +_workflow: + - setup_grid: load ding0 topology, import generators + - base_reinforce: worst-case TS + reinforce + reset equipment_changes + - import_heat_pumps: from egon_data + - import_home_batteries: from egon_data + - import_dsm: from egon_data + - import_electromobility: from egon_data (dumb charging) + - worst_case_ts: synthetic worst case incl. new assets + - reactive_power: fix reactive power control + - reinforce: final reinforcement — delta only + - save: persist topology + results + +scenario: eGon2035 + +pipeline: + - setup_grid: {import_generators: true} + - base_reinforce + - import_heat_pumps + - import_home_batteries + - import_dsm + - import_electromobility: {charging_strategy: dumb} + - worst_case_ts + - reactive_power + - reinforce + - save diff --git a/edisgo/run/presets/uc2_flex_opf.yaml b/edisgo/run/presets/uc2_flex_opf.yaml new file mode 100644 index 000000000..c09cae87e --- /dev/null +++ b/edisgo/run/presets/uc2_flex_opf.yaml @@ -0,0 +1,43 @@ +_comment: | + UC2 — OPF with full flexibility: + Like UC1 but loads real egon_data time series (oedb) and runs a + powermodels OPF over flexibilities (heat pumps, EV, DSM, storage) + before the final reinforce. Cost delta = extra reinforcement needed + under optimal flex dispatch. + +_workflow: + - setup_grid: load ding0 topology, import generators + - base_reinforce: worst-case TS + reinforce + reset equipment_changes + - import_heat_pumps: from egon_data + - import_home_batteries: from egon_data + - import_dsm: from egon_data + - import_electromobility: from egon_data (dumb charging, flex bands) + - oedb_ts: real wind/solar + load time series (168 h, 2035) + - apply_heat_pump_strategy: uncontrolled (overwritten by OPF) + - reactive_power + - check_integrity + - optimize: pm_optimize with flex assets (SOC, opf v2) + - reinforce: final reinforcement + - save + +scenario: eGon2035 + +pipeline: + - setup_grid: {import_generators: true} + - base_reinforce + - import_heat_pumps + - import_home_batteries + - import_dsm + - import_electromobility: {charging_strategy: dumb} + - oedb_ts: + timeindex: {start: "2035-01-01", periods: 168, freq: h} + dispatchable: {other: 0.7} + - apply_heat_pump_strategy: {strategy: uncontrolled} + - reactive_power + - check_integrity + - optimize: + flexible: [heat_pumps, storage] + method: soc + opf_version: 2 + - reinforce + - save diff --git a/edisgo/run/presets/uc3_oedb_ts.yaml b/edisgo/run/presets/uc3_oedb_ts.yaml new file mode 100644 index 000000000..59c184cdd --- /dev/null +++ b/edisgo/run/presets/uc3_oedb_ts.yaml @@ -0,0 +1,36 @@ +_comment: | + UC3 — real-world time series without OPF: + Like UC1 but uses real egon_data time series (oedb) instead of + synthetic worst cases. No optimization, no eTraGo. Difference to + UC1 is the data source for the final TS; difference to UC2 is no + OPF. + +_workflow: + - setup_grid: load ding0 topology, import generators + - base_reinforce: worst-case TS + reinforce + reset equipment_changes + - import_heat_pumps: from egon_data + - import_home_batteries: from egon_data + - import_dsm: from egon_data + - import_electromobility: from egon_data (dumb charging) + - oedb_ts: real egon_data time series + - apply_heat_pump_strategy: uncontrolled + - reactive_power + - reinforce: final reinforcement + - save + +scenario: eGon2035 + +pipeline: + - setup_grid: {import_generators: true} + - base_reinforce + - import_heat_pumps + - import_home_batteries + - import_dsm + - import_electromobility: {charging_strategy: dumb} + - oedb_ts: + timeindex: {start: "2035-01-01", periods: 168, freq: h} + dispatchable: {other: 0.7} + - apply_heat_pump_strategy: {strategy: uncontrolled} + - reactive_power + - reinforce + - save diff --git a/edisgo/run/registry.py b/edisgo/run/registry.py new file mode 100644 index 000000000..8aed4f3f5 --- /dev/null +++ b/edisgo/run/registry.py @@ -0,0 +1,113 @@ +""" +Task registry for the eDisGo pipeline runner. + +This module holds the global, process-wide mapping of task names to task +functions. Tasks are registered via the :func:`register_task` decorator +and looked up by name at pipeline execution time by the runner. Keeping +the registry separate from both the runner and the task implementations +lets external projects add their own tasks without patching eDisGo — +just import ``register_task`` and decorate a function. + +Registered tasks all share the signature ``(edisgo, ctx, **params)`` +where ``edisgo`` is the current :class:`~edisgo.EDisGo` instance (or +``None`` before it has been created by the first task), ``ctx`` is a +:class:`~edisgo.run.context.RunContext`, and ``**params`` are the +parameters passed from the YAML/JSON step definition. A task may return +an updated ``edisgo`` object (e.g. ``setup_grid`` creates it, ``load_*`` +replaces it); otherwise the runner keeps using the same instance. +""" +from __future__ import annotations + +from typing import Callable + +_TASKS: dict[str, Callable] = {} + + +def register_task(name: str) -> Callable[[Callable], Callable]: + """ + Decorator to register a task function under the given name. + + The decorated function becomes addressable from YAML/JSON pipelines + as either a plain string ``name`` or a single-key mapping + ``name: {param: value, ...}``. The name must be unique globally — + re-registering raises :class:`ValueError` to prevent silent + overrides across plugins. + + Parameters + ---------- + name : str + Unique task name used in pipeline definitions. + + Returns + ------- + Callable + A decorator that registers ``fn`` and returns it unchanged. + + Raises + ------ + ValueError + If ``name`` is already registered. + + Examples + -------- + >>> @register_task("set_timeindex_weekly") + ... def task_weekly(edisgo, ctx, *, start): + ... import pandas as pd + ... edisgo.set_timeindex(pd.date_range(start, periods=168, freq="h")) + + """ + def deco(fn: Callable) -> Callable: + if name in _TASKS: + raise ValueError( + f"Task '{name}' is already registered " + f"(existing={_TASKS[name].__qualname__}, " + f"new={fn.__qualname__})." + ) + _TASKS[name] = fn + return fn + + return deco + + +def get_task(name: str) -> Callable: + """ + Look up a registered task function by name. + + Parameters + ---------- + name : str + Task name as used in pipeline definitions. + + Returns + ------- + Callable + The task function registered under ``name``. + + Raises + ------ + KeyError + If ``name`` is not registered. The error message lists all + known task names to aid typo debugging. + + """ + if name not in _TASKS: + raise KeyError( + f"Unknown task: '{name}'. Known tasks: {sorted(_TASKS)}" + ) + return _TASKS[name] + + +def known_tasks() -> list[str]: + """ + Return a sorted list of all registered task names. + + Useful for error messages, CLI completion, and tests that assert + core tasks exist. + + Returns + ------- + list of str + All registered task names in alphabetical order. + + """ + return sorted(_TASKS) diff --git a/edisgo/run/runner.py b/edisgo/run/runner.py new file mode 100644 index 000000000..63f30aa07 --- /dev/null +++ b/edisgo/run/runner.py @@ -0,0 +1,261 @@ +""" +Pipeline execution engine for the eDisGo runner. + +This module ties the other three pieces — :mod:`edisgo.run.config` +(loader), :mod:`edisgo.run.validator` (static checks), and +:mod:`edisgo.run.registry` (task lookup) — together into a linear +stage-by-stage executor. + +The execution model: + +1. Load and validate the config. +2. Build a :class:`~edisgo.run.context.RunContext`. +3. For each stage, if the stage declares ``load_from: X``, reload + the EDisGo object from stage ``X``'s save-artifact (topology + + results only; time series are dropped to let the new stage set + fresh ones). +4. For each step in the stage's pipeline, look up the task function + in the registry and call it with the current EDisGo object and + the context. A task may return a new EDisGo object (``setup_grid``, + ``load_from_base``) which then replaces the current one. +5. Repeat for all stages, finally return the EDisGo object. + +Two entry points are exposed: + +* :func:`run_edisgo` — starts from no EDisGo object; the first task + must create one (usually ``setup_grid``). +* :func:`_run_pipeline_on` — starts from an existing EDisGo instance; + used by :meth:`edisgo.EDisGo.run_pipeline`. +""" +from __future__ import annotations + +import logging + +from pathlib import Path +from typing import Any + +from edisgo.run import tasks as _tasks # noqa: F401 — triggers registration +from edisgo.run.config import load_config +from edisgo.run.context import RunContext +from edisgo.run.registry import get_task +from edisgo.run.validator import _split_step, validate + +logger = logging.getLogger("edisgo.run.runner") + + +def run_edisgo(config) -> Any: + """ + Run an eDisGo pipeline from a YAML/JSON config or dict. + + This is the standalone entry point. The pipeline's first task is + typically ``setup_grid`` or ``load_from_base`` to bootstrap the + :class:`~edisgo.EDisGo` instance. If you already have one, + prefer :meth:`edisgo.EDisGo.run_pipeline` instead. + + Parameters + ---------- + config : str, pathlib.Path, or dict + Path to a YAML/JSON pipeline config, or an in-memory dict of + the same shape. + + Returns + ------- + :class:`~edisgo.EDisGo` + The EDisGo instance after the last stage has run. For + multi-stage configs this is the object produced by the final + stage. + + """ + return _run_pipeline_on(None, config) + + +def _run_pipeline_on(edisgo, config): + """ + Internal runner shared by :func:`run_edisgo` and the EDisGo method. + + Parameters + ---------- + edisgo : edisgo.EDisGo or None + Existing EDisGo instance to operate on, or ``None`` to have + the first task create one. + config : str, pathlib.Path, or dict + Config to execute. Passed through to + :func:`edisgo.run.config.load_config`. + + Returns + ------- + edisgo.EDisGo + The final EDisGo instance. + + Raises + ------ + RuntimeError + If a stage declares ``load_from: X`` but ``X`` produced no + artifact (typically because validate() was skipped). + + """ + cfg = load_config(config) + validate(cfg) + ctx = _build_context(cfg) + + for stage in cfg["stages"]: + ctx.current_stage = stage["name"] + ctx.logger.info(f"=== stage '{stage['name']}' ===") + + load_from = stage.get("load_from") + if load_from is not None: + artifact = ctx.stage_artifacts.get(load_from) + if artifact is None: + raise RuntimeError( + f"Stage '{stage['name']}' wants to load from " + f"'{load_from}' but no artifact is registered." + ) + edisgo = _load_artifact(str(artifact)) + + params = stage.get("params", {}) or {} + for step in stage["pipeline"]: + name, step_params = _split_step(step) + step_params = _resolve_templating(step_params, params) + ctx.logger.info(f" -> task '{name}'") + task_fn = get_task(name) + result = task_fn(edisgo, ctx, **step_params) + if result is not None: + edisgo = result + + return edisgo + + +def _build_context(cfg: dict) -> RunContext: + """ + Build a :class:`~edisgo.run.context.RunContext` from a config. + + Wires ``scenario`` and ``results.directory`` into the context and + stores the full config under :attr:`RunContext.raw_config` so + tasks can read supplementary sections. + + Parameters + ---------- + cfg : dict + Normalized config. + + Returns + ------- + RunContext + Initialized context with no engine, no artifacts, empty flags. + + """ + results_cfg = cfg.get("results") or {} + results_dir = results_cfg.get("directory") + return RunContext( + scenario=cfg.get("scenario"), + results_dir=Path(results_dir) if results_dir else None, + raw_config=cfg, + ) + + +def _load_artifact(path: str): + """ + Reload an EDisGo instance from a save-artifact for a ``load_from``. + + Loads topology + results only; time series and flex data are + dropped so the consuming stage can set them fresh. Equipment + changes are reset so the next stage's reinforce accounts only + for its own scenario. + + Parameters + ---------- + path : str + Path to a directory or ``.zip`` produced by the ``save`` + task. + + Returns + ------- + edisgo.EDisGo + The restored EDisGo instance. + + """ + import pandas as pd + + from edisgo.edisgo import import_edisgo_from_files + + from_zip = path.endswith(".zip") + edisgo = import_edisgo_from_files( + edisgo_path=path, + import_topology=True, + import_timeseries=False, + import_results=True, + import_electromobility=False, + import_heat_pump=False, + import_dsm=False, + import_overlying_grid=False, + from_zip_archive=from_zip, + ) + edisgo.legacy_grids = False + edisgo.results.equipment_changes = pd.DataFrame() + return edisgo + + +def _resolve_templating(step_params: dict, stage_params: dict) -> dict: + """ + Substitute ``{{params.x}}`` placeholders in step parameters. + + Stage-level ``params:`` allows a preset to expose a few knobs that + individual step parameters can reference. Only simple + ``{{params.KEY}}`` expansions inside string values are supported + (no filters, no conditionals, no nested expressions) — deliberately + kept trivial to avoid a Jinja dependency. + + Parameters + ---------- + step_params : dict + Keyword arguments for a single step. + stage_params : dict + Stage-level ``params:`` dict. + + Returns + ------- + dict + ``step_params`` with template strings resolved. + + """ + if not stage_params or not step_params: + return step_params + out = {} + for k, v in step_params.items(): + if isinstance(v, str) and "{{" in v: + out[k] = _render_template(v, stage_params) + else: + out[k] = v + return out + + +def _render_template(s: str, stage_params: dict) -> str: + """ + Expand ``{{params.KEY}}`` references in a single string. + + Parameters + ---------- + s : str + Source string. + stage_params : dict + Mapping of stage-level parameters. + + Returns + ------- + str + Rendered string. Unknown keys are left in place (the original + placeholder remains) so downstream errors point at the + typo-ed key rather than silently turning into an empty + string. + + """ + import re + + def repl(match): + expr = match.group(1).strip() + if expr.startswith("params."): + key = expr.split(".", 1)[1] + return str(stage_params.get(key, match.group(0))) + return match.group(0) + + return re.sub(r"\{\{\s*([^}]+)\s*\}\}", repl, s) diff --git a/edisgo/run/tasks/__init__.py b/edisgo/run/tasks/__init__.py new file mode 100644 index 000000000..0d59ea02a --- /dev/null +++ b/edisgo/run/tasks/__init__.py @@ -0,0 +1,25 @@ +""" +Task implementations for the eDisGo pipeline runner. + +Importing this package as a side effect registers every task defined +in its submodules with :func:`edisgo.run.registry.register_task`, so +that the runner sees them at execution time. The submodules are: + +* :mod:`.grid` — ``setup_grid``, ``load_from_base`` +* :mod:`.timeseries` — ``worst_case_ts``, ``oedb_ts``, ``manual_ts``, + ``set_timeindex``, ``reactive_power`` +* :mod:`.flex` — flex imports + (``import_heat_pumps``, ``import_home_batteries``, ``import_dsm``, + ``import_electromobility``, ``import_generators``) and operating + strategies (``apply_charging_strategy``, + ``apply_heat_pump_strategy``) +* :mod:`.analysis` — ``check_integrity``, ``analyze``, ``reinforce``, + ``base_reinforce``, ``optimize`` +* :mod:`.io` — ``save``, ``load_charging_from_files`` + +Task signature convention: ``(edisgo, ctx, **params)``. A task may +mutate ``edisgo`` in place and/or return a new EDisGo instance (the +returned value, if non-None, replaces the current one in the runner's +loop). +""" +from edisgo.run.tasks import analysis, flex, grid, io, timeseries # noqa: F401 diff --git a/edisgo/run/tasks/io.py b/edisgo/run/tasks/io.py new file mode 100644 index 000000000..3f604914e --- /dev/null +++ b/edisgo/run/tasks/io.py @@ -0,0 +1,179 @@ +""" +Input/output tasks — persisting results and ingesting external files. + +* :func:`task_save` (``save``) — persist topology, time series, and + results to disk (directory or zip). Also publishes the artifact + path into ``ctx.stage_artifacts`` so a later stage can + ``load_from:``. +* :func:`task_load_charging_from_files` + (``load_charging_from_files``) — R4MU-specific placeholder for + integrating scenario charging stations from a directory of CSV / + GeoPackage files; implementation is deferred until needed. +""" +from __future__ import annotations + +import os + +from edisgo.run.registry import register_task + + +@register_task("save") +def task_save(edisgo, ctx, *, directory=None, save_topology=True, + save_timeseries=True, save_results=True, + save_electromobility=None, save_opf_results=False, + save_heatpump=None, save_overlying_grid=False, + save_dsm=None, archive=False, archive_type="zip", + reduce_memory=False, parameters=None): + """ + Save the current EDisGo state to disk. + + If ``directory`` is not given, the artifact is written under + ``ctx.results_dir / `` so every stage gets its own + subdirectory. When ``archive=True`` the result is a single zip; + the artifact path (including ``.zip``) is recorded in + ``ctx.stage_artifacts[]`` so a downstream stage can + declare ``load_from: ``. + + Flags drive smart defaults for the optional ``save_*`` switches: + if flex data is absent (per ``ctx.flags``), saving it is skipped. + + Parameters + ---------- + edisgo : edisgo.EDisGo + EDisGo instance to persist. + ctx : RunContext + Run context. Uses ``ctx.results_dir``, ``ctx.current_stage``, + and reads ``has_heat_pumps`` / ``has_dsm`` / + ``has_electromobility`` flags. + directory : str, optional + Absolute target directory. If omitted, derived from + ``ctx.results_dir / ctx.current_stage``. + save_topology : bool, optional + Write the topology CSVs. Default ``True``. + save_timeseries : bool, optional + Write time-series CSVs. Default ``True``. + save_results : bool, optional + Write the results CSVs (equipment changes, expansion costs, + etc.). Default ``True``. + save_electromobility : bool or None, optional + If ``None``, auto-enabled iff + ``ctx.flags['has_electromobility']`` is truthy. + save_opf_results : bool, optional + Write OPF results if present. + save_heatpump : bool or None, optional + If ``None``, auto-enabled iff ``ctx.flags['has_heat_pumps']`` + is truthy. + save_overlying_grid : bool, optional + Write overlying-grid (eTraGo) specs if present. + save_dsm : bool or None, optional + If ``None``, auto-enabled iff ``ctx.flags['has_dsm']`` is + truthy. + archive : bool, optional + Pack the directory into a single ``.zip`` archive. + archive_type : str, optional + Archive format (currently only ``"zip"``). + reduce_memory : bool, optional + Downcast float time-series to ``float32`` to save disk. + parameters : dict, optional + Fine-grained selection of which results fields to write, + e.g. ``{"grid_expansion_results": ["equipment_changes"]}``. + + Returns + ------- + edisgo.EDisGo + The unchanged EDisGo instance. + + Raises + ------ + ValueError + If no ``directory`` is given and ``ctx.results_dir`` is also + unset. + + """ + if directory is None: + if ctx.results_dir is None: + raise ValueError( + "Task 'save' needs a 'directory' parameter or " + "config.results.directory." + ) + stage = ctx.current_stage or "main" + directory = os.path.join(str(ctx.results_dir), stage) + + if save_heatpump is None: + save_heatpump = ctx.flags.get("has_heat_pumps", False) + if save_dsm is None: + save_dsm = ctx.flags.get("has_dsm", False) + if save_electromobility is None: + save_electromobility = ctx.flags.get("has_electromobility", False) + + kwargs = dict( + directory=directory, + save_topology=save_topology, + save_timeseries=save_timeseries, + save_results=save_results, + save_electromobility=save_electromobility, + save_opf_results=save_opf_results, + save_heatpump=save_heatpump, + save_overlying_grid=save_overlying_grid, + save_dsm=save_dsm, + ) + if archive: + kwargs["archive"] = True + kwargs["archive_type"] = archive_type + if reduce_memory: + kwargs["reduce_memory"] = True + if parameters is not None: + kwargs["parameters"] = parameters + + edisgo.save(**kwargs) + + saved_path = directory + (".zip" if archive else "") + if ctx.current_stage: + ctx.stage_artifacts[ctx.current_stage] = saved_path + ctx.flags["last_saved"] = saved_path + return edisgo + + +@register_task("load_charging_from_files") +def task_load_charging_from_files(edisgo, ctx, *, charging_dir, + use_case_to_sector=None, + mv_threshold_kw=100.0): + """ + Integrate scenario charging stations from files (R4MU workflow). + + PLACEHOLDER — the full implementation lives in eGo's + ``_run_edisgo_task_load_charging_from_files`` and needs to be + ported when R4MU is prioritised. The eGo version reads a + GeoPackage / CSV of charging locations, filters by the MV grid + district geometry, and integrates them into the topology via + :func:`find_nearest_bus` / ``integrate_component_based_on_geolocation`` + with a use-case-to-sector mapping and an MV/LV connection + threshold. + + Parameters + ---------- + edisgo : edisgo.EDisGo + EDisGo instance to modify in place. + ctx : RunContext + Run context. + charging_dir : str + Directory containing the charging-station source files. + use_case_to_sector : dict, optional + Maps raw use-case labels (``"home_detached"`` etc.) to + eDisGo sector names (``"home"``, ``"work"``, …). + mv_threshold_kw : float, optional + Capacity threshold above which stations connect to an MV + bus; below connect to LV. + + Raises + ------ + NotImplementedError + Always — port the eGo implementation before using. + + """ + raise NotImplementedError( + "Task 'load_charging_from_files' is a placeholder port from " + "eGo R4MU. Port the logic from eGo's " + "_run_edisgo_task_load_charging_from_files when R4MU is " + "needed." + ) diff --git a/edisgo/run/validator.py b/edisgo/run/validator.py new file mode 100644 index 000000000..3989b8398 --- /dev/null +++ b/edisgo/run/validator.py @@ -0,0 +1,201 @@ +""" +Static validator for pipeline configs. + +The validator enforces structural and ordering rules that the runner +would otherwise hit at execution time — often after 20 minutes of work. +Running these checks up-front turns "cryptic AttributeError after half +the pipeline" into a clear ``ValueError`` at startup. + +Checked rules: + +* every step maps to a known, registered task name; +* ``reactive_power`` comes after every time-series task in a stage, + never before — ``set_time_series_reactive_power_control`` overwrites + reactive power on the currently set active-power time series; +* ``analyze`` and ``reinforce`` require a time-series task earlier in + the stage (or a ``load_from:`` that brings a prepared grid); +* ``optimize`` requires both a time-series task and at least one flex + import earlier in the stage — OPF without flexibility is meaningless; +* flex imports (``import_heat_pumps``, …) require a loaded grid, i.e. + an earlier ``setup_grid`` / ``load_from_base`` / a stage-level + ``load_from:``; +* ``base_reinforce`` likewise requires a loaded grid; +* a stage that declares ``load_from: X`` can only run if stage ``X`` + ran earlier AND contains a ``save`` step. +""" +from __future__ import annotations + +from typing import Any + +from edisgo.run.registry import known_tasks + +_TS_TASKS = {"worst_case_ts", "oedb_ts", "manual_ts", "set_timeindex"} +_GRID_CREATING_TASKS = {"setup_grid", "load_from_base"} +_FLEX_IMPORTS = { + "import_heat_pumps", + "import_home_batteries", + "import_dsm", + "import_electromobility", +} + + +def validate(cfg: dict) -> None: + """ + Validate a normalized pipeline config against the ordering rules. + + This function does not return a value. On success it simply + returns; on any rule violation it raises :class:`ValueError` with + a message identifying the offending stage and task. + + Parameters + ---------- + cfg : dict + Normalized config as returned by + :func:`edisgo.run.config.load_config`. Must have a ``stages`` + list at the top level. + + Raises + ------ + ValueError + If the config has no stages, an unknown task name, a + structural problem (reactive before TS, reinforce without TS, + optimize without flex, flex import without grid, …), or a + stage references a ``load_from`` source that doesn't exist or + has no ``save`` step. + + """ + stages = cfg.get("stages") or [] + if not stages: + raise ValueError("Config has no stages to run.") + + available_artifacts: set[str] = set() + + for stage in stages: + name = stage["name"] + pipeline = stage.get("pipeline") or [] + load_from = stage.get("load_from") + + if load_from is not None and load_from not in available_artifacts: + raise ValueError( + f"Stage '{name}' requires 'load_from: {load_from}' but " + f"that stage has not run or did not save. Available: " + f"{sorted(available_artifacts)}" + ) + + grid_available = load_from is not None + ts_set = False + reactive_set = False + flex_imported = False + has_save = False + + for step in pipeline: + task_name, _params = _split_step(step) + if task_name not in known_tasks(): + raise ValueError( + f"Unknown task '{task_name}' in stage '{name}'. " + f"Known: {known_tasks()}" + ) + + if task_name in _GRID_CREATING_TASKS: + grid_available = True + if task_name in _TS_TASKS: + if reactive_set: + raise ValueError( + f"Stage '{name}': time-series task " + f"'{task_name}' comes after 'reactive_power' " + f"— reactive_power must be the last " + f"time-series-altering step." + ) + ts_set = True + if task_name == "reactive_power": + reactive_set = True + if task_name in _FLEX_IMPORTS: + flex_imported = True + if not grid_available: + raise ValueError( + f"Stage '{name}': task '{task_name}' requires " + f"a loaded grid (setup_grid or " + f"load_from_base) before it." + ) + if task_name in {"analyze", "reinforce"} and not ( + ts_set or load_from + ): + raise ValueError( + f"Stage '{name}': task '{task_name}' requires time " + f"series to be set (e.g. worst_case_ts or " + f"oedb_ts) before it." + ) + if task_name == "optimize": + if not ts_set and not load_from: + raise ValueError( + f"Stage '{name}': 'optimize' requires time " + f"series." + ) + if not flex_imported and not load_from: + raise ValueError( + f"Stage '{name}': 'optimize' requires at least " + f"one flex asset to be imported." + ) + if task_name == "base_reinforce" and not grid_available: + raise ValueError( + f"Stage '{name}': 'base_reinforce' requires a " + f"loaded grid before it." + ) + if task_name == "save": + has_save = True + + if has_save: + available_artifacts.add(name) + + +def _split_step(step: Any) -> tuple[str, dict]: + """ + Normalize a pipeline step into ``(task_name, params)``. + + Steps are allowed in two forms in YAML/JSON: + + * bare string — ``worst_case_ts`` → ``("worst_case_ts", {})`` + * single-key mapping — + ``import_electromobility: {charging_strategy: dumb}`` + → ``("import_electromobility", {"charging_strategy": "dumb"})`` + + ``None`` as the parameter value is treated as an empty dict so + that YAML's ``task:`` (with nothing after the colon) works. + + Parameters + ---------- + step : str or dict + Raw step as it appears in the pipeline list. + + Returns + ------- + tuple of (str, dict) + The task name and its keyword arguments. + + Raises + ------ + ValueError + If ``step`` is not a string or a single-key mapping, or if + the parameter value is not a mapping. + + """ + if isinstance(step, str): + return step, {} + if isinstance(step, dict): + if len(step) != 1: + raise ValueError( + f"Task step must be a string or single-key mapping, " + f"got: {step}" + ) + (name, params), = step.items() + if params is None: + params = {} + if not isinstance(params, dict): + raise ValueError( + f"Parameters for task '{name}' must be a mapping, " + f"got: {type(params).__name__}" + ) + return name, params + raise ValueError( + f"Task step must be string or mapping, got: {step!r}" + ) diff --git a/setup.py b/setup.py index 0ee46d6c9..605bbe700 100644 --- a/setup.py +++ b/setup.py @@ -100,6 +100,7 @@ def read(fname): "edisgo": [ os.path.join("config", "*.cfg"), os.path.join("equipment", "*.csv"), + os.path.join("run", "presets", "*.yaml"), ] }, ) diff --git a/tests/run/__init__.py b/tests/run/__init__.py new file mode 100644 index 000000000..baf15e08b --- /dev/null +++ b/tests/run/__init__.py @@ -0,0 +1 @@ +"""Tests for the :mod:`edisgo.run` pipeline runner.""" diff --git a/tests/run/test_config.py b/tests/run/test_config.py new file mode 100644 index 000000000..10b4ec474 --- /dev/null +++ b/tests/run/test_config.py @@ -0,0 +1,154 @@ +""" +Unit tests for :mod:`edisgo.run.config` — loader, merger, adapter. + +Covers YAML/JSON parity, ``extends`` resolution (preset-by-name and +relative paths), deep-merge semantics, stage normalization, and the +eGo-legacy adapter. +""" +import json + +import pytest +import yaml + +from edisgo.run.config import _deep_merge, load_config + + +def _write(tmp_path, name, data): + """ + Helper: write ``data`` to ``tmp_path/name`` as YAML or JSON. + + Parameters + ---------- + tmp_path : pathlib.Path + Pytest-provided temporary directory. + name : str + File name with extension (``.yaml``/``.yml``/``.json``). + data : dict + Payload. + + Returns + ------- + pathlib.Path + Path to the written file. + + """ + path = tmp_path / name + if name.endswith(".json"): + path.write_text(json.dumps(data)) + else: + path.write_text(yaml.safe_dump(data)) + return path + + +def test_load_flat_pipeline_normalized_to_stages(tmp_path): + """A flat ``pipeline:`` must normalize to a single 'main' stage.""" + p = _write(tmp_path, "cfg.yaml", { + "scenario": "eGon2035", + "pipeline": ["setup_grid", "worst_case_ts", "reinforce"], + }) + cfg = load_config(str(p)) + assert "pipeline" not in cfg + assert cfg["stages"] == [ + {"name": "main", + "pipeline": ["setup_grid", "worst_case_ts", "reinforce"]} + ] + + +def test_yaml_and_json_equivalent(tmp_path): + """YAML and JSON payloads with identical content must load equal.""" + data = { + "scenario": "eGon2035", + "pipeline": ["setup_grid", "worst_case_ts", "reinforce"], + } + yaml_path = _write(tmp_path, "cfg.yaml", data) + json_path = _write(tmp_path, "cfg.json", data) + assert load_config(str(yaml_path)) == load_config(str(json_path)) + + +def test_extends_merges_parent(tmp_path): + """Child config must deep-merge with its ``extends:`` parent.""" + parent = _write(tmp_path, "parent.yaml", { + "scenario": "eGon2035", + "grid": {"legacy_ding0_grids": False}, + "pipeline": ["setup_grid", "reinforce"], + }) + child = _write(tmp_path, "child.yaml", { + "extends": str(parent), + "grid": {"ding0_path": "/tmp/xyz"}, + }) + cfg = load_config(str(child)) + assert cfg["scenario"] == "eGon2035" + assert cfg["grid"] == { + "legacy_ding0_grids": False, "ding0_path": "/tmp/xyz" + } + assert cfg["stages"][0]["pipeline"] == ["setup_grid", "reinforce"] + + +def test_extends_preset_by_name(tmp_path): + """``extends: basic`` must resolve to the bundled basic preset.""" + child = _write(tmp_path, "child.yaml", { + "extends": "basic", + "grid": {"ding0_path": "/tmp/xyz"}, + }) + cfg = load_config(str(child)) + assert "stages" in cfg + assert cfg["grid"]["ding0_path"] == "/tmp/xyz" + + +def test_deep_merge_nested(): + """Nested dicts must be merged key-by-key, child wins on conflict.""" + base = {"a": {"b": 1, "c": 2}, "d": 4} + over = {"a": {"b": 99, "e": 5}} + merged = _deep_merge(base, over) + assert merged == {"a": {"b": 99, "c": 2, "e": 5}, "d": 4} + + +def test_both_pipeline_and_stages_rejected(tmp_path): + """Top-level ``pipeline`` and ``stages`` are mutually exclusive.""" + p = _write(tmp_path, "cfg.yaml", { + "pipeline": ["setup_grid"], + "stages": [{"name": "x", "pipeline": ["setup_grid"]}], + }) + with pytest.raises(ValueError, match="both"): + load_config(str(p)) + + +def test_duplicate_stage_names_rejected(tmp_path): + """Stage names must be unique; duplicates raise ValueError.""" + p = _write(tmp_path, "cfg.yaml", { + "stages": [ + {"name": "x", "pipeline": ["setup_grid"]}, + {"name": "x", "pipeline": ["reinforce"]}, + ], + }) + with pytest.raises(ValueError, match="Duplicate stage"): + load_config(str(p)) + + +def test_ego_legacy_adapter(tmp_path): + """An eGo ``scenario_setting_*.json`` must adapt to the new schema.""" + ego_cfg = { + "eGo": {"eDisGo": True}, + "eTraGo": {"scn_name": "eGon2035"}, + "eDisGo": { + "grid_path": "/some/path", + "results": "/tmp/results", + "tasks": [ + "1_setup_grid", + "base_reinforce", + "import_heat_pumps_from_db", + "worst_case_ts", + "5_grid_reinforcement", + ], + }, + "database": {"host": "localhost"}, + } + p = _write(tmp_path, "legacy.json", ego_cfg) + cfg = load_config(str(p)) + assert cfg["scenario"] == "eGon2035" + assert cfg["grid"]["ding0_path"] == "/some/path" + assert cfg["stages"][0]["pipeline"] == [ + "setup_grid", "base_reinforce", "import_heat_pumps", + "worst_case_ts", "reinforce", + ] + assert cfg["database"]["host"] == "localhost" diff --git a/tests/run/test_registry.py b/tests/run/test_registry.py new file mode 100644 index 000000000..a56070bf6 --- /dev/null +++ b/tests/run/test_registry.py @@ -0,0 +1,35 @@ +""" +Unit tests for :mod:`edisgo.run.registry`. + +Verifies that core tasks are discoverable, that ``get_task`` raises a +useful error on typos, and that duplicate registrations are rejected. +""" +import pytest + +from edisgo.run.registry import get_task, known_tasks, register_task + + +def test_known_tasks_contains_core(): + """All core task names must be registered on import.""" + tasks = known_tasks() + for core in ["setup_grid", "worst_case_ts", "reactive_power", + "reinforce", "analyze", "save"]: + assert core in tasks + + +def test_get_task_unknown_raises(): + """Unknown task names must surface as a descriptive KeyError.""" + with pytest.raises(KeyError, match="Unknown task"): + get_task("does_not_exist") + + +def test_register_task_duplicate_raises(): + """Registering the same task name twice is a bug — must raise.""" + @register_task("_test_task_for_dup_check") + def _a(edisgo, ctx): + """Marker task #1 — test fixture only.""" + + with pytest.raises(ValueError, match="already registered"): + @register_task("_test_task_for_dup_check") + def _b(edisgo, ctx): + """Marker task #2 — test fixture only, must not register.""" diff --git a/tests/run/test_runner.py b/tests/run/test_runner.py new file mode 100644 index 000000000..0ecb6d08e --- /dev/null +++ b/tests/run/test_runner.py @@ -0,0 +1,118 @@ +""" +End-to-end tests for the eDisGo pipeline runner. + +Uses the small test grid under ``tests/data/ding0_test_network_2`` +(exposed by :mod:`tests.conftest` as +``pytest.ding0_test_network_2_path``) to run full pipelines without +touching the database. Covers: + +* the standalone ``run_edisgo`` entry point with a flat pipeline, +* the instance method ``EDisGo.run_pipeline``, +* the stage mechanism with ``save`` + ``load_from``. +""" +import os + +import pytest + +from edisgo.run import run_edisgo + + +@pytest.fixture +def basic_cfg(tmp_path): + """ + Minimal end-to-end config fixture. + + Produces a config that loads the small ding0 test grid, sets + worst-case time series, fixes reactive power, checks integrity, + runs reinforcement, and saves — no database needed. + + Parameters + ---------- + tmp_path : pathlib.Path + Pytest-provided temp directory for the run's artifacts. + + Returns + ------- + dict + The config dict. + + """ + return { + "scenario": "eGon2035", + "grid": { + "ding0_path": pytest.ding0_test_network_2_path, + "legacy_ding0_grids": True, + }, + "results": {"directory": str(tmp_path)}, + "pipeline": [ + "setup_grid", + "worst_case_ts", + "reactive_power", + "check_integrity", + "reinforce", + "save", + ], + } + + +def test_runner_basic_end_to_end(basic_cfg): + """A flat-pipeline run must execute and persist the expected artifact.""" + edisgo = run_edisgo(basic_cfg) + assert edisgo is not None + assert edisgo.topology is not None + assert os.path.isdir(os.path.join(basic_cfg["results"]["directory"], + "main")) + + +def test_runner_method_on_edisgo(basic_cfg): + """``EDisGo.run_pipeline`` must operate on the existing instance.""" + from edisgo import EDisGo + + basic_cfg["pipeline"] = basic_cfg["pipeline"][1:] # skip setup_grid + edisgo = EDisGo( + ding0_grid=basic_cfg["grid"]["ding0_path"], + legacy_ding0_grids=True, + ) + edisgo = edisgo.run_pipeline(basic_cfg) + assert edisgo.topology is not None + + +def test_runner_two_stages_with_load_from(tmp_path): + """ + A two-stage run must save the first stage and reload it via + ``load_from`` in the second stage, producing both artifacts. + """ + cfg = { + "scenario": "eGon2035", + "grid": { + "ding0_path": pytest.ding0_test_network_2_path, + "legacy_ding0_grids": True, + }, + "results": {"directory": str(tmp_path)}, + "stages": [ + { + "name": "base", + "pipeline": [ + "setup_grid", + "worst_case_ts", + "reactive_power", + "reinforce", + {"save": {"archive": True}}, + ], + }, + { + "name": "scenario", + "load_from": "base", + "pipeline": [ + "worst_case_ts", + "reactive_power", + "reinforce", + "save", + ], + }, + ], + } + edisgo = run_edisgo(cfg) + assert edisgo.topology is not None + assert os.path.exists(os.path.join(str(tmp_path), "base.zip")) + assert os.path.isdir(os.path.join(str(tmp_path), "scenario")) diff --git a/tests/run/test_validator.py b/tests/run/test_validator.py new file mode 100644 index 000000000..4b86f40cf --- /dev/null +++ b/tests/run/test_validator.py @@ -0,0 +1,97 @@ +""" +Unit tests for :mod:`edisgo.run.validator`. + +Each test pins one ordering rule: reactive-before-TS, reinforce +without TS, optimize without flex, flex import without grid, and the +stage-level ``load_from`` constraints. +""" +import pytest + +from edisgo.run.validator import validate + + +def _wrap(pipeline): + """ + Wrap a flat pipeline into a single-stage config dict. + + Parameters + ---------- + pipeline : list + Ordered list of task names / single-key mappings. + + Returns + ------- + dict + Minimal config in the shape expected by :func:`validate`. + + """ + return {"stages": [{"name": "main", "pipeline": pipeline}]} + + +def test_valid_pipeline(): + """A well-formed pipeline must pass validation without raising.""" + validate(_wrap(["setup_grid", "worst_case_ts", "reactive_power", + "reinforce", "save"])) + + +def test_unknown_task_rejected(): + """Typo'd task names must be rejected.""" + with pytest.raises(ValueError, match="Unknown task"): + validate(_wrap(["setup_grid", "nonexistent_task"])) + + +def test_reactive_before_ts_rejected(): + """reactive_power before a TS task violates the ordering rule.""" + with pytest.raises(ValueError, match="reactive_power"): + validate(_wrap(["setup_grid", "reactive_power", "worst_case_ts"])) + + +def test_reinforce_without_ts_rejected(): + """reinforce without any prior time-series step must fail.""" + with pytest.raises(ValueError, match="time series"): + validate(_wrap(["setup_grid", "reinforce"])) + + +def test_optimize_without_flex_rejected(): + """optimize requires at least one flex asset to be imported.""" + with pytest.raises(ValueError, match="flex asset"): + validate(_wrap(["setup_grid", "worst_case_ts", "optimize"])) + + +def test_flex_import_before_grid_rejected(): + """Flex imports require a loaded grid — pre-loading is not enough.""" + with pytest.raises(ValueError, match="loaded grid"): + validate(_wrap(["import_heat_pumps", "worst_case_ts", "reinforce"])) + + +def test_stage_load_from_missing_rejected(): + """``load_from: X`` where X has not run must fail.""" + cfg = {"stages": [ + {"name": "a", "pipeline": ["setup_grid", "worst_case_ts", + "reinforce"]}, + {"name": "b", "load_from": "nonexistent", + "pipeline": ["reinforce"]}, + ]} + with pytest.raises(ValueError, match="load_from"): + validate(cfg) + + +def test_stage_load_from_requires_save_in_source(): + """A stage consumed by ``load_from`` must itself end with ``save``.""" + cfg = {"stages": [ + {"name": "a", "pipeline": ["setup_grid", "worst_case_ts", + "reinforce"]}, # no save + {"name": "b", "load_from": "a", "pipeline": ["reinforce"]}, + ]} + with pytest.raises(ValueError, match="load_from"): + validate(cfg) + + +def test_stage_load_from_with_save_ok(): + """Stage chain with a save in the source must validate successfully.""" + cfg = {"stages": [ + {"name": "a", "pipeline": ["setup_grid", "worst_case_ts", + "reinforce", "save"]}, + {"name": "b", "load_from": "a", "pipeline": ["reinforce", "save"]}, + ]} + validate(cfg) From 0c8792eea11a41e6422c691aeb9e65f20e2cee53 Mon Sep 17 00:00:00 2001 From: Jonas Danke Date: Wed, 13 May 2026 16:18:51 +0200 Subject: [PATCH 11/37] Add example yaml for full example --- edisgo/run/presets/uc4_example_MS.yaml | 56 ++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100644 edisgo/run/presets/uc4_example_MS.yaml diff --git a/edisgo/run/presets/uc4_example_MS.yaml b/edisgo/run/presets/uc4_example_MS.yaml new file mode 100644 index 000000000..a72573496 --- /dev/null +++ b/edisgo/run/presets/uc4_example_MS.yaml @@ -0,0 +1,56 @@ +_comment: | + UC3 — OPF with full flexibility: + Like UC1 but loads real egon_data time series (oedb) and runs a + powermodels OPF over flexibilities (heat pumps, EV, DSM, storage) + before the final reinforce. Cost delta = extra reinforcement needed + under optimal flex dispatch. + +_workflow: + - setup_grid: load ding0 topology, import generators + - base_reinforce: worst-case TS + reinforce + reset equipment_changes + - import_generators: from edon-data + - import_heat_pumps: from egon_data + - import_home_batteries: from egon_data + - import_dsm: from egon_data + - import_electromobility: from egon_data (dumb charging, flex bands) + - oedb_ts: real wind/solar + load time series (24 h, 2035) + - apply_heat_pump_strategy: uncontrolled (overwritten by OPF) + - reactive_power + - check_integrity + - optimize: pm_optimize with flex assets (SOC, opf v2) + - reinforce: final reinforcement + - save + +scenario: eGon2035 +grid: + ding0_path: "/home/gurobi/.ding0/2024-07-25T17:38:34_new_planning_new_edisgo/ding0_grids/32377" + legacy_ding0_grids: false + +database: + ssh: + enabled: false + +timeindex: {start: "2035-01-01", periods: 24, freq: h} + +results: + directory: results/uc4_example + +pipeline: + - setup_grid + - base_reinforce + - import_generators + - import_home_batteries + - import_heat_pumps + - import_dsm + - import_electromobility: {charging_strategy: dumb, flexibility_bands_ucs : ["home", "work", "public", "hpc"]} + - apply_heat_pump_strategy: {strategy: uncontrolled} + - oedb_ts: + dispatchable: {other: 0.7} + - reactive_power + - check_integrity + - optimize: + flexible: [heat_pumps, storage, charging_points, dsm] + method: soc + opf_version: 2 + - reinforce + - save From d3476eb9873fcf4626fa2c9b64bb6560d1a3f848 Mon Sep 17 00:00:00 2001 From: Jonas Danke Date: Wed, 13 May 2026 16:20:22 +0200 Subject: [PATCH 12/37] Add file for analysis-tasks, Add short cut for DSM --- edisgo/run/tasks/analysis.py | 321 +++++++++++++++++++++++++++++++++++ 1 file changed, 321 insertions(+) create mode 100644 edisgo/run/tasks/analysis.py diff --git a/edisgo/run/tasks/analysis.py b/edisgo/run/tasks/analysis.py new file mode 100644 index 000000000..f028bb31c --- /dev/null +++ b/edisgo/run/tasks/analysis.py @@ -0,0 +1,321 @@ +""" +Power-flow, reinforcement, and optimization tasks. + +The three analysis layers: + +* :func:`task_analyze` (``analyze``) — non-linear AC load flow over + the active time series; does not modify the topology. +* :func:`task_reinforce` (``reinforce``) — iterative reinforcement + that adds/upgrades equipment until all technical constraints are + met. Populates ``results.equipment_changes``. +* :func:`task_optimize` (``optimize``) — powermodels OPF over + flexibilities (heat pumps, EV, DSM, storage) to minimize + reinforcement need. + +In addition: + +* :func:`task_check_integrity` (``check_integrity``) — a cheap + sanity check before the expensive steps. +* :func:`task_base_reinforce` (``base_reinforce``) — two-phase helper: + worst-case TS → reinforce → reset ``equipment_changes``. Used to + produce a "base" grid whose subsequent reinforce costs reflect + only a scenario overlay. +""" +from __future__ import annotations + +import pandas as pd + +from edisgo.run.registry import register_task + + +@register_task("check_integrity") +def task_check_integrity(edisgo, ctx): + """ + Run EDisGo's integrity checks on the topology and time series. + + Catches bus mismatches, missing time series for components, and + similar structural problems. Raises if something is off — do not + swallow it silently. + + Parameters + ---------- + edisgo : edisgo.EDisGo + EDisGo instance to check. + ctx : RunContext + Run context (unused). + + Returns + ------- + edisgo.EDisGo + The unchanged EDisGo instance. + + """ + edisgo.check_integrity() + return edisgo + + +@register_task("analyze") +def task_analyze(edisgo, ctx, *, mode=None, timesteps=None, + raise_not_converged=False, troubleshooting_mode=None): + """ + Run AC power flow over the active time series. + + Parameters + ---------- + edisgo : edisgo.EDisGo + EDisGo instance to analyze. + ctx : RunContext + Run context. Stores the number of non-converged time steps + under ``ctx.flags['not_converged_steps']`` and warns if any. + mode : str, optional + ``None`` (default) runs the full grid; ``"mv"`` runs only the + medium-voltage level; ``"lv"`` runs only LV. + timesteps : pandas.DatetimeIndex, optional + Restrict the analysis to these time steps. + raise_not_converged : bool, optional + If ``True``, raise on non-convergence. Default ``False`` so + the pipeline can continue and ``reinforce`` can attempt to + resolve the issue. + troubleshooting_mode : str, optional + Extra diagnostic mode passed through to + :meth:`EDisGo.analyze`. + + Returns + ------- + edisgo.EDisGo + The analyzed EDisGo instance. + + """ + result = edisgo.analyze( + mode=mode, + timesteps=timesteps, + raise_not_converged=raise_not_converged, + troubleshooting_mode=troubleshooting_mode, + ) + if isinstance(result, tuple) and len(result) == 2: + converged, not_converged = result + ctx.flags["not_converged_steps"] = len(not_converged) + if len(not_converged) > 0: + ctx.logger.warning( + f"Power flow did not converge for {len(not_converged)} " + f"time steps." + ) + return edisgo + + +@register_task("reinforce") +def task_reinforce(edisgo, ctx, *, timesteps_pfa=None, reduced_analysis=False, + copy_grid=False, max_while_iterations=20, + split_voltage_band=True, mode=None, + without_generator_import=False, n_minus_one=False, + catch_convergence_problems=False): + """ + Run iterative grid reinforcement. + + Adds/upgrades lines and transformers until voltage and loading + constraints are met for all time steps. Results accumulate in + :attr:`EDisGo.results.equipment_changes` and + :attr:`~EDisGo.results.grid_expansion_costs`. + + Parameters + ---------- + edisgo : edisgo.EDisGo + EDisGo instance to reinforce. + ctx : RunContext + Run context (unused beyond logging). + timesteps_pfa : pandas.DatetimeIndex, optional + Restrict the reinforcement's analysis to these time steps. + reduced_analysis : bool, optional + If ``True``, use a cheaper convergence check during + reinforcement. + copy_grid : bool, optional + If ``True``, operate on a copy and return it as a new + instance (default ``False``). + max_while_iterations : int, optional + Cap on the outer iteration loop. + split_voltage_band : bool, optional + Split the allowed voltage deviation between MV and LV + (typical MV/LV coupling rule). + mode : str, optional + ``None``, ``"mv"``, ``"lv"``, or ``"mvlv"``. Restricts + reinforcement to a voltage level. + without_generator_import : bool, optional + Skip the implicit generator import step. + n_minus_one : bool, optional + Enable (N-1) contingency reinforcement. Expensive. + catch_convergence_problems : bool, optional + Wrap in the catch-convergence helper for troublesome grids. + + Returns + ------- + edisgo.EDisGo + The reinforced EDisGo instance. + + """ + edisgo.reinforce( + timesteps_pfa=timesteps_pfa, + reduced_analysis=reduced_analysis, + copy_grid=copy_grid, + max_while_iterations=max_while_iterations, + split_voltage_band=split_voltage_band, + mode=mode, + without_generator_import=without_generator_import, + n_minus_one=n_minus_one, + catch_convergence_problems=catch_convergence_problems, + ) + return edisgo + + +@register_task("base_reinforce") +def task_base_reinforce(edisgo, ctx, *, cases=None, + reset_equipment_changes=True, save_artifact=True): + """ + Produce a base-reinforced grid and reset the cost accumulator. + + This is the composite step ported from eGo's two-phase reinforce + workflow: + + 1. Set synthetic worst-case time series (``feed-in_case`` + + ``load_case``). + 2. Run :meth:`EDisGo.reinforce` to bring the grid to a neutral + baseline. + 3. Optionally save the resulting grid so downstream stages can + ``load_from: ...``. + 4. Clear :attr:`Results.equipment_changes` so the next reinforce + captures only scenario-specific deltas. + 5. Restore the prior time index so the next TS-setting task + starts from a clean state. + + Parameters + ---------- + edisgo : edisgo.EDisGo + EDisGo instance to base-reinforce. + ctx : RunContext + Run context. ``ctx.results_dir`` is the artifact destination. + Sets ``ctx.flags['base_reinforced'] = True`` and + ``ctx.stage_artifacts['__base_reinforce__']`` on save. + cases : list of str, optional + Which worst cases to set (subset of + ``{"load_case", "feed-in_case"}``). Default is both. + reset_equipment_changes : bool, optional + Clear the equipment-changes DataFrame after reinforcement. + save_artifact : bool, optional + Write a ``grid_data_base_reinforcement.zip`` next to the + other results. + + Returns + ------- + edisgo.EDisGo + The base-reinforced EDisGo instance. + + """ + import os + + prev_timeindex = edisgo.timeseries.timeindex + + edisgo.set_time_series_worst_case_analysis(cases=cases) + edisgo.reinforce() + + if save_artifact and ctx.results_dir is not None: + artifact_dir = os.path.join( + str(ctx.results_dir), "grid_data_base_reinforcement" + ) + edisgo.save( + directory=artifact_dir, + save_topology=True, + save_timeseries=False, + save_results=True, + archive=True, + archive_type="zip", + parameters={"grid_expansion_results": ["equipment_changes"]}, + ) + ctx.stage_artifacts["__base_reinforce__"] = artifact_dir + ".zip" + + if reset_equipment_changes: + edisgo.results.equipment_changes = pd.DataFrame() + + if len(prev_timeindex) > 0: + edisgo.set_timeindex(prev_timeindex) + + ctx.flags["base_reinforced"] = True + return edisgo + + +@register_task("optimize") +def task_optimize(edisgo, ctx, *, flexible=None, flexible_cps=None, + flexible_hps=None, flexible_loads=None, + flexible_storage_units=None, opf_version=2, method="soc", + warm_start=False, s_base=1): + """ + Run a powermodels optimal-power-flow (OPF) over flexibilities. + + If ``flexible`` is given (high-level shortcut), it expands to the + lower-level ``flexible_*`` lists automatically: + + * ``"heat_pumps"`` → all loads of type ``heat_pump`` + * ``"charging_points"`` → all loads of type ``charging_point`` + * ``"storage"`` → all storage-unit indices + * ``"loads"`` → all DSM-ready load indices + + Explicit ``flexible_*`` kwargs override the shortcut. + + Parameters + ---------- + edisgo : edisgo.EDisGo + EDisGo instance to optimize. + ctx : RunContext + Run context (unused). + flexible : list of str, optional + High-level selector, subset of ``{"heat_pumps", + "charging_points", "storage"}``. If ``None``, nothing is + auto-populated. + flexible_cps : list of str, optional + Explicit list of flexible charging-point names. + flexible_hps : list of str, optional + Explicit list of flexible heat-pump load names. + flexible_loads : list of str, optional + Explicit list of flexible DSM load names. + flexible_storage_units : list of str, optional + Explicit list of flexible storage-unit names. + opf_version : int, optional + Powermodels OPF formulation version (1 or 2, default 2). + method : str, optional + OPF relaxation method, e.g. ``"soc"`` (second-order cone). + warm_start : bool, optional + Reuse a previous solution as the starting point. + s_base : float, optional + Per-unit base power for normalization. + + Returns + ------- + edisgo.EDisGo + The optimized EDisGo instance. + + """ + flexible = flexible or [] + + if flexible_hps is None and "heat_pumps" in flexible: + flexible_hps = edisgo.topology.loads_df.loc[ + edisgo.topology.loads_df.type == "heat_pump" + ].index.tolist() + if flexible_cps is None and "charging_points" in flexible: + flexible_cps = edisgo.topology.loads_df.loc[ + edisgo.topology.loads_df.type == "charging_point" + ].index.tolist() + if flexible_storage_units is None and "storage" in flexible: + flexible_storage_units = edisgo.topology.storage_units_df.index.tolist() + if flexible_loads is not None and "dsm" in flexbile: + flexible_loads = edisgo.dsm.p_min.columns.values + + + edisgo.pm_optimize( + flexible_cps=flexible_cps or [], + flexible_hps=flexible_hps or [], + flexible_loads=flexible_loads or [], + flexible_storage_units=flexible_storage_units or [], + opf_version=opf_version, + method=method, + warm_start=warm_start, + s_base=s_base, + ) + return edisgo From 25ae93e58954b4f377c9bd3ee81b58e73efb9da5 Mon Sep 17 00:00:00 2001 From: Jonas Danke Date: Wed, 13 May 2026 16:21:10 +0200 Subject: [PATCH 13/37] Add file for flex-tasks, Add flexibility band generation, --- edisgo/run/tasks/flex.py | 282 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 282 insertions(+) create mode 100644 edisgo/run/tasks/flex.py diff --git a/edisgo/run/tasks/flex.py b/edisgo/run/tasks/flex.py new file mode 100644 index 000000000..fd914a2a0 --- /dev/null +++ b/edisgo/run/tasks/flex.py @@ -0,0 +1,282 @@ +""" +Flex-asset import and operation-strategy tasks. + +These tasks either pull flex assets (heat pumps, home batteries, DSM, +electromobility, generators) from egon_data / OEP into the topology, +or apply an operating strategy on assets already present. They must +run AFTER the grid is loaded (``setup_grid`` or ``load_from_base``) +and typically BEFORE the time-series step, so the time series can +cover the new assets. +""" +from __future__ import annotations + +from edisgo.run.registry import register_task + + +@register_task("import_heat_pumps") +def task_import_heat_pumps(edisgo, ctx, *, import_types=None, timeindex=None): + """ + Import heat pumps from egon_data into the topology. + + Parameters + ---------- + edisgo : edisgo.EDisGo + EDisGo instance to modify in place. + ctx : RunContext + Run context. Uses ``ctx.scenario`` and + ``ctx.ensure_engine()``. Sets + ``ctx.flags['has_heat_pumps']`` to the observed count. + import_types : list of str, optional + Subset of ``["individual_heat_pumps", "central_heat_pumps"]``; + default imports both. + timeindex : pandas.DatetimeIndex, optional + Restrict COP / heat-demand time series to this index. + + Returns + ------- + edisgo.EDisGo + The modified EDisGo instance. + + """ + edisgo.import_heat_pumps( + scenario=ctx.scenario, + engine=ctx.ensure_engine(), + timeindex=timeindex, + import_types=import_types, + ) + ctx.flags["has_heat_pumps"] = len( + edisgo.topology.loads_df.loc[ + edisgo.topology.loads_df.type == "heat_pump" + ] + ) > 0 + return edisgo + + +@register_task("import_home_batteries") +def task_import_home_batteries(edisgo, ctx): + """ + Import home batteries from egon_data into the topology. + + Parameters + ---------- + edisgo : edisgo.EDisGo + EDisGo instance to modify in place. + ctx : RunContext + Run context. Uses ``ctx.scenario`` and + ``ctx.ensure_engine()``. Sets + ``ctx.flags['has_home_batteries']``. + + Returns + ------- + edisgo.EDisGo + The modified EDisGo instance. + + """ + edisgo.import_home_batteries( + scenario=ctx.scenario, engine=ctx.ensure_engine() + ) + ctx.flags["has_home_batteries"] = ( + not edisgo.topology.storage_units_df.empty + ) + return edisgo + + +@register_task("import_dsm") +def task_import_dsm(edisgo, ctx, *, timeindex=None): + """ + Import demand-side-management potential from egon_data. + + Parameters + ---------- + edisgo : edisgo.EDisGo + EDisGo instance to modify in place. + ctx : RunContext + Run context. Uses ``ctx.scenario`` and + ``ctx.ensure_engine()``. Sets ``ctx.flags['has_dsm']``. + timeindex : pandas.DatetimeIndex, optional + Restrict DSM availability time series to this index. + + Returns + ------- + edisgo.EDisGo + The modified EDisGo instance. + + """ + edisgo.import_dsm( + scenario=ctx.scenario, + engine=ctx.ensure_engine(), + timeindex=timeindex, + ) + ctx.flags["has_dsm"] = ( + edisgo.dsm.p_max is not None and not edisgo.dsm.p_max.empty + ) + return edisgo + + +@register_task("import_electromobility") +def task_import_electromobility(edisgo, ctx, *, data_source="oedb", + charging_strategy="dumb", + flexibility_bands_ucs = None, + import_electromobility_data_kwds=None, + allocate_charging_demand_kwds=None): + """ + Import electromobility data (charging processes + parks). + + Optionally applies a charging strategy directly after import to + turn the raw charging processes into active-power time series on + the charging points. + + Parameters + ---------- + edisgo : edisgo.EDisGo + EDisGo instance to modify in place. + ctx : RunContext + Run context. Uses ``ctx.scenario`` and + ``ctx.ensure_engine()`` (for ``data_source='oedb'``). Sets + ``ctx.flags['has_electromobility'] = True``. + data_source : str, optional + ``"oedb"`` (egon_data) or ``"directory"`` (requires + ``import_electromobility_data_kwds={"charging_processes_dir": + ..., "potential_charging_points_dir": ...}``). + charging_strategy : str or None, optional + Charging strategy applied right after import. ``"dumb"`` + (uncontrolled, default), ``"reduced"``, ``"residual"``, or + ``None`` to skip. + flexibility_bands_ucs : str or list of str, optional + Charging-point use case(s) to compute flexibility bands for + via :meth:`Electromobility.get_flexibility_bands` after import + and charging-strategy application. Valid entries: + ``"home"``, ``"work"``, ``"public"``, ``"hpc"``. Pass a single + string for one use case or a list for multiple. ``None`` + (default) skips flexibility-band computation. + import_electromobility_data_kwds : dict, optional + Extra kwargs passed through to the underlying importer. + allocate_charging_demand_kwds : dict, optional + Extra kwargs for charging-demand allocation. + + Returns + ------- + edisgo.EDisGo + The modified EDisGo instance. + + """ + edisgo.import_electromobility( + data_source=data_source, + scenario=ctx.scenario, + engine=ctx.ensure_engine(), + import_electromobility_data_kwds=import_electromobility_data_kwds, + allocate_charging_demand_kwds=allocate_charging_demand_kwds, + ) + if charging_strategy: + edisgo.apply_charging_strategy(strategy=charging_strategy) + if flexibility_bands_ucs is not None: + edisgo.electromobility.get_flexibility_bands( + edisgo, + use_case=flexibility_bands_ucs, + ) + ctx.flags["has_electromobility"] = True + return edisgo + + +@register_task("apply_charging_strategy") +def task_apply_charging_strategy(edisgo, ctx, *, strategy="dumb", + charging_park_ids=None): + """ + Apply a charging strategy to the already-imported EV fleet. + + Standalone variant of the step that ``import_electromobility`` + does inline. Useful when you want to import once and then try + multiple strategies in different runs. + + Parameters + ---------- + edisgo : edisgo.EDisGo + EDisGo instance to modify in place. + ctx : RunContext + Run context. + strategy : str, optional + Strategy name (``"dumb"`` / ``"reduced"`` / ``"residual"``). + charging_park_ids : list of int, optional + Restrict the strategy to these charging-park IDs. + + Returns + ------- + edisgo.EDisGo + The modified EDisGo instance. + + """ + edisgo.apply_charging_strategy( + strategy=strategy, charging_park_ids=charging_park_ids + ) + return edisgo + + +@register_task("apply_heat_pump_strategy") +def task_apply_heat_pump_strategy(edisgo, ctx, *, strategy="uncontrolled", + heat_pump_names=None): + """ + Apply a heat-pump operating strategy. + + Skipped with an info-log if no heat pumps are present + (``ctx.flags['has_heat_pumps']`` is falsy), so pipelines can + safely include this step without a conditional guard. + + Parameters + ---------- + edisgo : edisgo.EDisGo + EDisGo instance to modify in place. + ctx : RunContext + Run context. + strategy : str, optional + Operating strategy (``"uncontrolled"``, ``"flexible"``, …). + heat_pump_names : list of str, optional + Restrict to specific heat-pump load names; default is all. + + Returns + ------- + edisgo.EDisGo + The modified EDisGo instance. + + """ + if not ctx.flags.get("has_heat_pumps"): + ctx.logger.info( + "Skipping 'apply_heat_pump_strategy': no heat pumps " + "present." + ) + return edisgo + edisgo.apply_heat_pump_operating_strategy( + strategy=strategy, heat_pump_names=heat_pump_names + ) + return edisgo + + +@register_task("import_generators") +def task_import_generators(edisgo, ctx, *, generator_scenario=None): + """ + Import future generators for the active scenario. + + Thin wrapper around :meth:`EDisGo.import_generators`. Mostly + useful when you want to split grid loading and generator import + into two separate pipeline steps. + + Parameters + ---------- + edisgo : edisgo.EDisGo + EDisGo instance to modify in place. + ctx : RunContext + Run context. ``ctx.scenario`` is used if + ``generator_scenario`` is not given. + generator_scenario : str, optional + Scenario name, e.g. ``"nep2035"`` or ``"ego100"``. Defaults + to ``ctx.scenario``. + + Returns + ------- + edisgo.EDisGo + The modified EDisGo instance. + + """ + edisgo.import_generators( + generator_scenario=generator_scenario or ctx.scenario + ) + return edisgo From 56dbf7264e721c575efbdd43df4e2226aaa38f2b Mon Sep 17 00:00:00 2001 From: Jonas Danke Date: Wed, 13 May 2026 16:22:18 +0200 Subject: [PATCH 14/37] Add file for grid-tasks, Add timeindex in setup task --- edisgo/run/tasks/grid.py | 172 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 172 insertions(+) create mode 100644 edisgo/run/tasks/grid.py diff --git a/edisgo/run/tasks/grid.py b/edisgo/run/tasks/grid.py new file mode 100644 index 000000000..b472f3244 --- /dev/null +++ b/edisgo/run/tasks/grid.py @@ -0,0 +1,172 @@ +""" +Grid loading tasks — bring an EDisGo instance into existence. + +Two ways to start a pipeline: + +* :func:`task_setup_grid` (``setup_grid``) — read a ding0 topology + from disk. This is the typical first step of every pipeline. +* :func:`task_load_from_base` (``load_from_base``) — reload a + previously saved EDisGo instance. Used to split a computation into + a slow "base" phase and one or more fast "scenario" phases that + reuse the base-reinforced grid. +""" +from __future__ import annotations + +from edisgo.run.registry import register_task + + +@register_task("setup_grid") +def task_setup_grid(edisgo, ctx, *, timeindex = None, ding0_path=None, legacy_ding0_grids=None, + import_generators=False, generator_scenario=None): + """ + Load a ding0 grid into an EDisGo instance. + + If the runner was started without an EDisGo object (via + :func:`edisgo.run.run_edisgo`) this task creates one from the + ding0 CSV directory. If an EDisGo object is already present (via + :meth:`edisgo.EDisGo.run_pipeline`), it imports the topology into + that existing instance. + + Parameters + ---------- + edisgo : edisgo.EDisGo or None + Current EDisGo instance, or ``None`` to create a fresh one. + ctx : RunContext + Run context. ``ctx.raw_config['grid']`` is consulted when + parameters are not passed explicitly. + ding0_path : str, optional + Path to the ding0 grid directory. Falls back to + ``ctx.raw_config['grid']['ding0_path']``. + legacy_ding0_grids : bool, optional + Whether to treat the ding0 directory as the legacy format. + Falls back to ``ctx.raw_config['grid']['legacy_ding0_grids']`` + and ultimately to ``False``. + import_generators : bool, optional + If ``True``, call :meth:`EDisGo.import_generators` after + loading the grid. + generator_scenario : str, optional + Generator scenario name passed to + :meth:`EDisGo.import_generators` (only if + ``import_generators=True``). + + Returns + ------- + edisgo.EDisGo + The EDisGo instance with the ding0 topology loaded. + + Raises + ------ + ValueError + If no ``ding0_path`` is given either as a task parameter or + under ``config.grid.ding0_path``. + + """ + from edisgo import EDisGo + + grid_cfg = ctx.raw_config.get("grid", {}) + ding0_path = ding0_path or grid_cfg.get("ding0_path") + if ding0_path is None: + raise ValueError( + "Task 'setup_grid' requires 'ding0_path' either as task " + "parameter or under config.grid.ding0_path." + ) + if legacy_ding0_grids is None: + legacy_ding0_grids = grid_cfg.get("legacy_ding0_grids", False) + + if edisgo is None: + edisgo = EDisGo( + ding0_grid=str(ding0_path), + legacy_ding0_grids=legacy_ding0_grids, + ) + else: + edisgo.import_ding0_grid( + path=str(ding0_path), legacy_ding0_grids=legacy_ding0_grids + ) + + if import_generators: + edisgo.import_generators(generator_scenario=generator_scenario) + + if timeindex is not None: + ti_df = pd.date_range( + start=timeindex["start"], + periods=timeindex["periods"], + freq=timeindex.get("freq", "h"), + ) + edisgo.set_timeindex(ti_df) + + ctx.flags["grid_loaded"] = True + return edisgo + + +@register_task("load_from_base") +def task_load_from_base(edisgo, ctx, *, path, reset_equipment_changes=True, + import_timeseries=False, import_results=False, + import_electromobility=False, import_heat_pump=False, + import_dsm=False, import_overlying_grid=False): + """ + Reload an EDisGo instance from a previously saved directory/zip. + + This is the two-phase R4MU workflow's entry point: stage 1 + produces a base-reinforced grid and saves it, stage 2 (or N) + starts from ``load_from_base`` to pick up that grid and apply + scenario-specific modifications. The cost of the scenario then + shows up cleanly in ``equipment_changes`` because we reset it on + load. + + Parameters + ---------- + edisgo : edisgo.EDisGo or None + Unused — the task always replaces whatever was there. + ctx : RunContext + Run context (logger only). + path : str + Directory or ``.zip`` produced by :func:`task_save`. + reset_equipment_changes : bool, optional + If ``True`` (default), clear + :attr:`Results.equipment_changes` so only the scenario's + reinforce is tracked. + import_timeseries : bool, optional + Whether to import the saved time series. Default: ``False`` + so the next stage sets its own. + import_results : bool, optional + Whether to import saved results. Default: ``False``. + import_electromobility : bool, optional + Whether to import saved electromobility data. + import_heat_pump : bool, optional + Whether to import saved heat-pump data. + import_dsm : bool, optional + Whether to import saved DSM data. + import_overlying_grid : bool, optional + Whether to import saved overlying-grid data (eTraGo + specifications). + + Returns + ------- + edisgo.EDisGo + The restored EDisGo instance. + + """ + import os + + import pandas as pd + + from edisgo.edisgo import import_edisgo_from_files + + path = str(path) + from_zip = path.endswith(".zip") or not os.path.isdir(path) + edisgo = import_edisgo_from_files( + edisgo_path=path, + import_topology=True, + import_timeseries=import_timeseries, + import_results=import_results, + import_electromobility=import_electromobility, + import_heat_pump=import_heat_pump, + import_dsm=import_dsm, + import_overlying_grid=import_overlying_grid, + from_zip_archive=from_zip, + ) + edisgo.legacy_grids = False + if reset_equipment_changes: + edisgo.results.equipment_changes = pd.DataFrame() + ctx.flags["grid_loaded"] = True + return edisgo From 579aba8c0a31e4e645bba2def97fe4550cfaab0e Mon Sep 17 00:00:00 2001 From: Jonas Danke Date: Wed, 13 May 2026 16:23:27 +0200 Subject: [PATCH 15/37] Add file for timeseries-tasks --- edisgo/run/tasks/timeseries.py | 298 +++++++++++++++++++++++++++++++++ 1 file changed, 298 insertions(+) create mode 100644 edisgo/run/tasks/timeseries.py diff --git a/edisgo/run/tasks/timeseries.py b/edisgo/run/tasks/timeseries.py new file mode 100644 index 000000000..f738aa056 --- /dev/null +++ b/edisgo/run/tasks/timeseries.py @@ -0,0 +1,298 @@ +""" +Time-series tasks — set active/reactive power profiles on EDisGo. + +Time series drive every downstream step: ``analyze``, ``reinforce`` +and ``optimize`` all operate on the time index and power time series +attached to the EDisGo object. The order inside a stage matters: + +1. Set the time index and active-power profiles with one of + :func:`task_worst_case_ts`, :func:`task_oedb_ts`, + :func:`task_manual_ts`, possibly :func:`task_set_timeindex`. +2. Finally call :func:`task_reactive_power` to fix reactive power + control — this MUST come last because it overwrites whatever + reactive power was set by the earlier steps. +""" +from __future__ import annotations + +import pandas as pd + +from edisgo.run.registry import register_task + + +@register_task("worst_case_ts") +def task_worst_case_ts(edisgo, ctx, *, cases=None, + generators_names=None, loads_names=None, + storage_units_names=None): + """ + Set synthetic worst-case active-power time series. + + Produces two snapshots (load case and feed-in case) that + represent the network's extremes. Useful for a coarse first + reinforce that does not require real load/generation data. + + Parameters + ---------- + edisgo : edisgo.EDisGo + EDisGo instance to modify in place. + ctx : RunContext + Run context. Sets ``ctx.flags['timeseries_set'] = True``. + cases : list of str, optional + Subset of ``{"load_case", "feed-in_case"}``. Default is both. + generators_names : list of str, optional + Restrict to these generator names; default is all. + loads_names : list of str, optional + Restrict to these load names; default is all. + storage_units_names : list of str, optional + Restrict to these storage units; default is all. + + Returns + ------- + edisgo.EDisGo + The modified EDisGo instance. + + """ + edisgo.set_time_series_worst_case_analysis( + cases=cases, + generators_names=generators_names, + loads_names=loads_names, + storage_units_names=storage_units_names, + ) + ctx.flags["timeseries_set"] = True + return edisgo + + +@register_task("set_timeindex") +def task_set_timeindex(edisgo, ctx, *, start, periods=None, end=None, + freq="h"): + """ + Set the time index on the EDisGo object. + + Useful as a stand-alone step when you want a specific hourly + range without immediately attaching time-series data (the + ``oedb_ts`` task already accepts a ``timeindex`` argument and + does this internally). + + Parameters + ---------- + edisgo : edisgo.EDisGo + EDisGo instance to modify in place. + ctx : RunContext + Run context. + start : str or pandas.Timestamp + First timestamp of the range. + periods : int, optional + Number of periods; mutually exclusive with ``end``. + end : str or pandas.Timestamp, optional + Last timestamp; mutually exclusive with ``periods``. + freq : str, optional + pandas frequency string, default hourly (``"h"``). + + Returns + ------- + edisgo.EDisGo + The modified EDisGo instance. + + Raises + ------ + ValueError + If neither ``periods`` nor ``end`` is provided. + + """ + if end is not None: + timeindex = pd.date_range(start=start, end=end, freq=freq) + else: + if periods is None: + raise ValueError( + "set_timeindex needs either 'periods' or 'end'." + ) + timeindex = pd.date_range(start=start, periods=periods, freq=freq) + edisgo.set_timeindex(timeindex) + return edisgo + + +@register_task("oedb_ts") +def task_oedb_ts(edisgo, ctx, *, timeindex=None, dispatchable=None, + fluctuating="oedb", conventional_loads="oedb", + charging_points_ts=None): + """ + Set active-power time series from egon_data (OEP) plus overrides. + + This is the "real data" path: wind and solar profiles come from + ``egon_era5_renewable_feedin``, conventional loads come from the + egon demand tables. Dispatchable generators (conventional, + etc.) are set via a per-technology-type profile since egon_data + does not dispatch them. Storage units default to zero if not + already set. + + Parameters + ---------- + edisgo : edisgo.EDisGo + EDisGo instance to modify in place. + ctx : RunContext + Run context. Uses ``ctx.scenario`` and + ``ctx.ensure_engine()`` when any source is ``"oedb"``. Sets + ``ctx.flags['timeseries_set'] = True``. + timeindex : dict, optional + ``{"start": ..., "periods": N, "freq": "h"}``. If present, a + matching :class:`~pandas.DatetimeIndex` is set before + importing data. + dispatchable : dict, optional + Per-technology scaling factors, e.g. ``{"other": 0.7}`` → + constant profile of 0.7 p.u. for all non-fluctuating + generators of type "other". + fluctuating : str or pandas.DataFrame, optional + How to populate wind/solar. ``"oedb"`` pulls egon_data, + ``"default"`` uses bundled standard profiles, or a DataFrame + with columns "solar" / "wind" is passed through. + conventional_loads : str, optional + Source for conventional loads (not heat pumps / charging + points). ``"oedb"`` or ``"demandlib"``. + charging_points_ts : pandas.DataFrame, optional + Explicit active-power profile for charging points; default + ``None`` leaves them untouched so + :func:`task_apply_charging_strategy` can set them. + + Returns + ------- + edisgo.EDisGo + The modified EDisGo instance. + + """ + if timeindex is not None: + ti_df = pd.date_range( + start=timeindex["start"], + periods=timeindex["periods"], + freq=timeindex.get("freq", "h"), + ) + edisgo.set_timeindex(ti_df) + + dispatchable_df = None + if dispatchable is not None: + ti = edisgo.timeseries.timeindex + dispatchable_df = pd.DataFrame(dispatchable, index=ti) + + conv_loads_names = None + if conventional_loads == "oedb": + conv_loads_names = edisgo.topology.loads_df.loc[ + ~edisgo.topology.loads_df.type.isin( + ["heat_pump", "charging_point"] + ) + ].index.tolist() + + edisgo.set_time_series_active_power_predefined( + fluctuating_generators_ts=fluctuating, + conventional_loads_ts=conventional_loads, + conventional_loads_names=conv_loads_names, + dispatchable_generators_ts=dispatchable_df, + charging_points_ts=charging_points_ts, + scenario=ctx.scenario, + engine=ctx.ensure_engine() if fluctuating == "oedb" + or conventional_loads == "oedb" else None, + ) + + su_names = edisgo.topology.storage_units_df.index + if len(su_names) > 0 and edisgo.timeseries.storage_units_active_power.empty: + edisgo.timeseries.storage_units_active_power = pd.DataFrame( + 0.0, index=edisgo.timeseries.timeindex, columns=su_names, + ) + ctx.flags["timeseries_set"] = True + return edisgo + + +@register_task("manual_ts") +def task_manual_ts(edisgo, ctx, *, + generators_active_power=None, + generators_reactive_power=None, + loads_active_power=None, + loads_reactive_power=None, + storage_units_active_power=None, + storage_units_reactive_power=None): + """ + Set active/reactive power time series from explicit DataFrames. + + Used when the caller already has the raw profiles (e.g. from a + coupled run) and wants to inject them directly. Any argument left + at ``None`` is not touched. + + Parameters + ---------- + edisgo : edisgo.EDisGo + EDisGo instance to modify in place. + ctx : RunContext + Run context. Sets ``ctx.flags['timeseries_set'] = True``. + generators_active_power : dict or pandas.DataFrame, optional + Generator active-power profile(s). Converted via + :class:`pandas.DataFrame`. + generators_reactive_power : dict or pandas.DataFrame, optional + Generator reactive-power profile(s). + loads_active_power : dict or pandas.DataFrame, optional + Load active-power profile(s). + loads_reactive_power : dict or pandas.DataFrame, optional + Load reactive-power profile(s). + storage_units_active_power : dict or pandas.DataFrame, optional + Storage-unit active-power profile(s). + storage_units_reactive_power : dict or pandas.DataFrame, optional + Storage-unit reactive-power profile(s). + + Returns + ------- + edisgo.EDisGo + The modified EDisGo instance. + + """ + def _as_df(obj): + return pd.DataFrame(obj) if obj is not None else None + + edisgo.set_time_series_manual( + generators_active_power=_as_df(generators_active_power), + generators_reactive_power=_as_df(generators_reactive_power), + loads_active_power=_as_df(loads_active_power), + loads_reactive_power=_as_df(loads_reactive_power), + storage_units_active_power=_as_df(storage_units_active_power), + storage_units_reactive_power=_as_df(storage_units_reactive_power), + ) + ctx.flags["timeseries_set"] = True + return edisgo + + +@register_task("reactive_power") +def task_reactive_power(edisgo, ctx, *, control="fixed_cosphi", + generators_parametrisation="default", + loads_parametrisation="default", + storage_units_parametrisation="default"): + """ + Apply reactive-power control on top of the active-power time series. + + This MUST be the last time-series-altering step before + ``analyze`` / ``reinforce`` / ``optimize``. The validator + enforces this ordering rule statically. + + Parameters + ---------- + edisgo : edisgo.EDisGo + EDisGo instance to modify in place. + ctx : RunContext + Run context. Sets ``ctx.flags['reactive_power_set'] = True``. + control : str, optional + Reactive-power control strategy; typically ``"fixed_cosphi"``. + generators_parametrisation : str or dict, optional + Per-generator parametrisation, ``"default"`` uses the config. + loads_parametrisation : str or dict, optional + Per-load parametrisation. + storage_units_parametrisation : str or dict, optional + Per-storage-unit parametrisation. + + Returns + ------- + edisgo.EDisGo + The modified EDisGo instance. + + """ + edisgo.set_time_series_reactive_power_control( + control=control, + generators_parametrisation=generators_parametrisation, + loads_parametrisation=loads_parametrisation, + storage_units_parametrisation=storage_units_parametrisation, + ) + ctx.flags["reactive_power_set"] = True + return edisgo From 65fa8fc0ba73c41eae0adb408dc39b2b26873f3a Mon Sep 17 00:00:00 2001 From: Moritz Schloesser Date: Tue, 19 May 2026 16:40:05 +0200 Subject: [PATCH 16/37] Edit uc4 example --- edisgo/run/presets/uc4_example_MS.yaml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/edisgo/run/presets/uc4_example_MS.yaml b/edisgo/run/presets/uc4_example_MS.yaml index a72573496..13e08b258 100644 --- a/edisgo/run/presets/uc4_example_MS.yaml +++ b/edisgo/run/presets/uc4_example_MS.yaml @@ -24,16 +24,21 @@ _workflow: scenario: eGon2035 grid: ding0_path: "/home/gurobi/.ding0/2024-07-25T17:38:34_new_planning_new_edisgo/ding0_grids/32377" + # grid path should be set in run file legacy_ding0_grids: false + # legacy parameter too database: ssh: enabled: false timeindex: {start: "2035-01-01", periods: 24, freq: h} +# only set in preset, not set in run-functions (eDisGo and eGo) results: directory: results/uc4_example +# actually all parameters + pipeline: - setup_grid @@ -42,6 +47,7 @@ pipeline: - import_home_batteries - import_heat_pumps - import_dsm + # where is decided, which electromobility use cases are used? - import_electromobility: {charging_strategy: dumb, flexibility_bands_ucs : ["home", "work", "public", "hpc"]} - apply_heat_pump_strategy: {strategy: uncontrolled} - oedb_ts: @@ -49,6 +55,7 @@ pipeline: - reactive_power - check_integrity - optimize: + # where is decided which flexibilities are used in the OPF? flexible: [heat_pumps, storage, charging_points, dsm] method: soc opf_version: 2 From 7ad649083bc70d1fc776b83f1fb6c6deb0e262f2 Mon Sep 17 00:00:00 2001 From: Moritz Schloesser Date: Wed, 20 May 2026 18:17:15 +0200 Subject: [PATCH 17/37] Wire overlying-grid data through pipeline runner Adds optional overlying_grid_data kwarg to run_edisgo() that is stashed on RunContext for downstream tasks instead of being passed as a keyword to every task (which broke task signatures). task_import_overlying_grid_data now: - accepts the standard (edisgo, ctx, *, ...) signature - reads overlying_grid_data from ctx, falls back to overlying_grid.path in the runner config - loads dispatchable + renewables_potential CSVs and applies them via set_time_series_active_power_predefined - shifts the year and reindexes overlying-grid attributes to the active edisgo timeindex so CSV-based input lines up with OEDB time series task_set_timeindex now reduces existing time-series data to the new index (via reduce_timeseries_data_to_given_timeindex) instead of silently leaving stale data behind. Renames the uc4_example_MS preset to uc4_example. --- .../{uc4_example_MS.yaml => uc4_example.yaml} | 9 +- edisgo/run/runner.py | 8 +- edisgo/run/tasks/io.py | 158 ++++++++++++++++-- edisgo/run/tasks/timeseries.py | 85 ++++++---- 4 files changed, 214 insertions(+), 46 deletions(-) rename edisgo/run/presets/{uc4_example_MS.yaml => uc4_example.yaml} (88%) diff --git a/edisgo/run/presets/uc4_example_MS.yaml b/edisgo/run/presets/uc4_example.yaml similarity index 88% rename from edisgo/run/presets/uc4_example_MS.yaml rename to edisgo/run/presets/uc4_example.yaml index 13e08b258..2d2650279 100644 --- a/edisgo/run/presets/uc4_example_MS.yaml +++ b/edisgo/run/presets/uc4_example.yaml @@ -23,7 +23,7 @@ _workflow: scenario: eGon2035 grid: - ding0_path: "/home/gurobi/.ding0/2024-07-25T17:38:34_new_planning_new_edisgo/ding0_grids/32377" + ding0_path: "/path/to/ding0_grid" # grid path should be set in run file legacy_ding0_grids: false # legacy parameter too @@ -48,10 +48,13 @@ pipeline: - import_heat_pumps - import_dsm # where is decided, which electromobility use cases are used? - - import_electromobility: {charging_strategy: dumb, flexibility_bands_ucs : ["home", "work", "public", "hpc"]} - - apply_heat_pump_strategy: {strategy: uncontrolled} + - import_electromobility: {charging_strategy: null, flexibility_bands_ucs : ["home", "work", "public", "hpc"]} - oedb_ts: dispatchable: {other: 0.7} + timeindex: {start: "2035-01-01", periods: 24, freq: h} + - apply_charging_strategy: {strategy: dumb} + - apply_heat_pump_strategy: {strategy: uncontrolled} + - import_overlying_grid_data - reactive_power - check_integrity - optimize: diff --git a/edisgo/run/runner.py b/edisgo/run/runner.py index 63f30aa07..2d08fd3bc 100644 --- a/edisgo/run/runner.py +++ b/edisgo/run/runner.py @@ -27,6 +27,7 @@ * :func:`_run_pipeline_on` — starts from an existing EDisGo instance; used by :meth:`edisgo.EDisGo.run_pipeline`. """ + from __future__ import annotations import logging @@ -43,7 +44,7 @@ logger = logging.getLogger("edisgo.run.runner") -def run_edisgo(config) -> Any: +def run_edisgo(config, overlying_grid_data=None) -> Any: """ Run an eDisGo pipeline from a YAML/JSON config or dict. @@ -66,10 +67,10 @@ def run_edisgo(config) -> Any: stage. """ - return _run_pipeline_on(None, config) + return _run_pipeline_on(None, config, overlying_grid_data=overlying_grid_data) -def _run_pipeline_on(edisgo, config): +def _run_pipeline_on(edisgo, config, overlying_grid_data=None): """ Internal runner shared by :func:`run_edisgo` and the EDisGo method. @@ -97,6 +98,7 @@ def _run_pipeline_on(edisgo, config): cfg = load_config(config) validate(cfg) ctx = _build_context(cfg) + ctx.overlying_grid_data = overlying_grid_data for stage in cfg["stages"]: ctx.current_stage = stage["name"] diff --git a/edisgo/run/tasks/io.py b/edisgo/run/tasks/io.py index 3f604914e..4e2e84d15 100644 --- a/edisgo/run/tasks/io.py +++ b/edisgo/run/tasks/io.py @@ -10,6 +10,7 @@ integrating scenario charging stations from a directory of CSV / GeoPackage files; implementation is deferred until needed. """ + from __future__ import annotations import os @@ -18,12 +19,24 @@ @register_task("save") -def task_save(edisgo, ctx, *, directory=None, save_topology=True, - save_timeseries=True, save_results=True, - save_electromobility=None, save_opf_results=False, - save_heatpump=None, save_overlying_grid=False, - save_dsm=None, archive=False, archive_type="zip", - reduce_memory=False, parameters=None): +def task_save( + edisgo, + ctx, + *, + directory=None, + save_topology=True, + save_timeseries=True, + save_results=True, + save_electromobility=None, + save_opf_results=False, + save_heatpump=None, + save_overlying_grid=False, + save_dsm=None, + archive=False, + archive_type="zip", + reduce_memory=False, + parameters=None, +): """ Save the current EDisGo state to disk. @@ -93,8 +106,7 @@ def task_save(edisgo, ctx, *, directory=None, save_topology=True, if directory is None: if ctx.results_dir is None: raise ValueError( - "Task 'save' needs a 'directory' parameter or " - "config.results.directory." + "Task 'save' needs a 'directory' parameter or config.results.directory." ) stage = ctx.current_stage or "main" directory = os.path.join(str(ctx.results_dir), stage) @@ -135,9 +147,9 @@ def task_save(edisgo, ctx, *, directory=None, save_topology=True, @register_task("load_charging_from_files") -def task_load_charging_from_files(edisgo, ctx, *, charging_dir, - use_case_to_sector=None, - mv_threshold_kw=100.0): +def task_load_charging_from_files( + edisgo, ctx, *, charging_dir, use_case_to_sector=None, mv_threshold_kw=100.0 +): """ Integrate scenario charging stations from files (R4MU workflow). @@ -177,3 +189,127 @@ def task_load_charging_from_files(edisgo, ctx, *, charging_dir, "_run_edisgo_task_load_charging_from_files when R4MU is " "needed." ) + + +@register_task("import_overlying_grid_data") +def task_import_overlying_grid_data(edisgo, ctx, *, overlying_grid_path=None): + """ + Import overlying grid data into the EDisGo instance. + + When ``overlying_grid_data`` is a dict of DataFrames (as returned by + ``get_etrago_results_per_bus``), the overlying-grid attributes and + dispatchable/fluctuating generator time series are set from it. + + When ``overlying_grid_path`` is a directory path, the overlying-grid + attributes are loaded from CSV files in that directory, and + ``dispatchable_generators_active_power.csv`` / + ``renewables_potential.csv`` are applied as generator time series + if present. + + Falls back to ``ctx.raw_config['eDisGo']['overlying_grid_source']`` + as the directory path when neither argument is given. + + Parameters + ---------- + edisgo : edisgo.EDisGo + EDisGo instance to modify in place. + ctx : RunContext + Run context. + overlying_grid_path : str, optional + Directory containing overlying-grid CSV files. + overlying_grid_data : dict, optional + Dict of DataFrames as returned by ``get_etrago_results_per_bus``. + + Returns + ------- + edisgo.EDisGo + The modified EDisGo instance. + + """ + import pandas as pd + + overlying_grid_data = getattr(ctx, "overlying_grid_data", None) + + if overlying_grid_data is not None: + # eTraGo results dict — set standard overlying-grid attributes + for attr in edisgo.overlying_grid._attributes: + if attr in overlying_grid_data: + setattr(edisgo.overlying_grid, attr, overlying_grid_data[attr]) + # set generator time series + edisgo.set_time_series_active_power_predefined( + dispatchable_generators_ts=overlying_grid_data.get( + "dispatchable_generators_active_power" + ), + fluctuating_generators_ts=overlying_grid_data.get("renewables_potential"), + ) + return edisgo + + # resolve path: explicit arg → runner config overlying_grid.path → skip + if overlying_grid_path is None: + overlying_grid_path = (ctx.raw_config.get("overlying_grid") or {}).get("path") + + if overlying_grid_path is None: + ctx.logger.warning( + "task 'import_overlying_grid_data': no overlying_grid_data or " + "overlying_grid_path provided — skipping." + ) + return edisgo + + # load overlying-grid attributes from CSV directory + edisgo.overlying_grid.from_csv(overlying_grid_path) + + # reindex overlying-grid attributes to match edisgo timeindex + # CSVs may use a different year — shift year then reindex + edisgo_ti = edisgo.timeseries.timeindex + if not edisgo_ti.empty: + for attr in edisgo.overlying_grid._attributes: + ts = getattr(edisgo.overlying_grid, attr) + if ts.empty: + continue + csv_year = ts.index[0].year + edisgo_year = edisgo_ti[0].year + if csv_year != edisgo_year: + ts.index = ts.index + pd.DateOffset(years=edisgo_year - csv_year) + if isinstance(ts, pd.Series): + setattr(edisgo.overlying_grid, attr, ts.reindex(edisgo_ti)) + else: + setattr(edisgo.overlying_grid, attr, ts.reindex(edisgo_ti)) + + # load dispatchable generator and renewables time series from the same dir + disp_path = os.path.join( + overlying_grid_path, "dispatchable_generators_active_power.csv" + ) + if os.path.isfile(disp_path): + disp_ts = pd.read_csv(disp_path, index_col=0, parse_dates=True) + if not edisgo_ti.empty: + csv_year = disp_ts.index[0].year + edisgo_year = edisgo_ti[0].year + if csv_year != edisgo_year: + disp_ts.index = disp_ts.index + pd.DateOffset( + years=edisgo_year - csv_year + ) + disp_ts = disp_ts.reindex(edisgo_ti) + else: + disp_ts = None + + pot_path = os.path.join(overlying_grid_path, "renewables_potential.csv") + if os.path.isfile(pot_path): + pot_ts = pd.read_csv(pot_path, index_col=0, parse_dates=True) + if not edisgo_ti.empty: + csv_year = pot_ts.index[0].year + edisgo_year = edisgo_ti[0].year + if csv_year != edisgo_year: + pot_ts.index = pot_ts.index + pd.DateOffset( + years=edisgo_year - csv_year + ) + pot_ts = pot_ts.reindex(edisgo_ti) + else: + pot_ts = None + + if disp_ts is not None or pot_ts is not None: + edisgo.set_time_series_active_power_predefined( + dispatchable_generators_ts=disp_ts, + fluctuating_generators_ts=pot_ts, + ) + + return edisgo diff --git a/edisgo/run/tasks/timeseries.py b/edisgo/run/tasks/timeseries.py index f738aa056..cf918c235 100644 --- a/edisgo/run/tasks/timeseries.py +++ b/edisgo/run/tasks/timeseries.py @@ -12,6 +12,7 @@ control — this MUST come last because it overwrites whatever reactive power was set by the earlier steps. """ + from __future__ import annotations import pandas as pd @@ -20,9 +21,15 @@ @register_task("worst_case_ts") -def task_worst_case_ts(edisgo, ctx, *, cases=None, - generators_names=None, loads_names=None, - storage_units_names=None): +def task_worst_case_ts( + edisgo, + ctx, + *, + cases=None, + generators_names=None, + loads_names=None, + storage_units_names=None, +): """ Set synthetic worst-case active-power time series. @@ -62,8 +69,7 @@ def task_worst_case_ts(edisgo, ctx, *, cases=None, @register_task("set_timeindex") -def task_set_timeindex(edisgo, ctx, *, start, periods=None, end=None, - freq="h"): +def task_set_timeindex(edisgo, ctx, *, start, periods=None, end=None, freq="h"): """ Set the time index on the EDisGo object. @@ -98,22 +104,32 @@ def task_set_timeindex(edisgo, ctx, *, start, periods=None, end=None, If neither ``periods`` nor ``end`` is provided. """ + from edisgo.tools.tools import reduce_timeseries_data_to_given_timeindex + if end is not None: timeindex = pd.date_range(start=start, end=end, freq=freq) else: if periods is None: - raise ValueError( - "set_timeindex needs either 'periods' or 'end'." - ) + raise ValueError("set_timeindex needs either 'periods' or 'end'.") timeindex = pd.date_range(start=start, periods=periods, freq=freq) - edisgo.set_timeindex(timeindex) + if edisgo.timeseries.timeindex.empty: + edisgo.set_timeindex(timeindex) + else: + reduce_timeseries_data_to_given_timeindex(edisgo, timeindex) return edisgo @register_task("oedb_ts") -def task_oedb_ts(edisgo, ctx, *, timeindex=None, dispatchable=None, - fluctuating="oedb", conventional_loads="oedb", - charging_points_ts=None): +def task_oedb_ts( + edisgo, + ctx, + *, + timeindex=None, + dispatchable=None, + fluctuating="oedb", + conventional_loads="oedb", + charging_points_ts=None, +): """ Set active-power time series from egon_data (OEP) plus overrides. @@ -174,9 +190,7 @@ def task_oedb_ts(edisgo, ctx, *, timeindex=None, dispatchable=None, conv_loads_names = None if conventional_loads == "oedb": conv_loads_names = edisgo.topology.loads_df.loc[ - ~edisgo.topology.loads_df.type.isin( - ["heat_pump", "charging_point"] - ) + ~edisgo.topology.loads_df.type.isin(["heat_pump", "charging_point"]) ].index.tolist() edisgo.set_time_series_active_power_predefined( @@ -186,27 +200,34 @@ def task_oedb_ts(edisgo, ctx, *, timeindex=None, dispatchable=None, dispatchable_generators_ts=dispatchable_df, charging_points_ts=charging_points_ts, scenario=ctx.scenario, - engine=ctx.ensure_engine() if fluctuating == "oedb" - or conventional_loads == "oedb" else None, + engine=ctx.ensure_engine() + if fluctuating == "oedb" or conventional_loads == "oedb" + else None, ) su_names = edisgo.topology.storage_units_df.index if len(su_names) > 0 and edisgo.timeseries.storage_units_active_power.empty: edisgo.timeseries.storage_units_active_power = pd.DataFrame( - 0.0, index=edisgo.timeseries.timeindex, columns=su_names, + 0.0, + index=edisgo.timeseries.timeindex, + columns=su_names, ) ctx.flags["timeseries_set"] = True return edisgo @register_task("manual_ts") -def task_manual_ts(edisgo, ctx, *, - generators_active_power=None, - generators_reactive_power=None, - loads_active_power=None, - loads_reactive_power=None, - storage_units_active_power=None, - storage_units_reactive_power=None): +def task_manual_ts( + edisgo, + ctx, + *, + generators_active_power=None, + generators_reactive_power=None, + loads_active_power=None, + loads_reactive_power=None, + storage_units_active_power=None, + storage_units_reactive_power=None, +): """ Set active/reactive power time series from explicit DataFrames. @@ -240,6 +261,7 @@ def task_manual_ts(edisgo, ctx, *, The modified EDisGo instance. """ + def _as_df(obj): return pd.DataFrame(obj) if obj is not None else None @@ -256,10 +278,15 @@ def _as_df(obj): @register_task("reactive_power") -def task_reactive_power(edisgo, ctx, *, control="fixed_cosphi", - generators_parametrisation="default", - loads_parametrisation="default", - storage_units_parametrisation="default"): +def task_reactive_power( + edisgo, + ctx, + *, + control="fixed_cosphi", + generators_parametrisation="default", + loads_parametrisation="default", + storage_units_parametrisation="default", +): """ Apply reactive-power control on top of the active-power time series. From 5fce2573c2dc66fdeb81d847ec39bf8bfc65e2a4 Mon Sep 17 00:00:00 2001 From: Moritz Schloesser Date: Thu, 21 May 2026 16:52:24 +0200 Subject: [PATCH 18/37] Redirect to installed julia version --- edisgo/opf/powermodels_opf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/edisgo/opf/powermodels_opf.py b/edisgo/opf/powermodels_opf.py index c26475141..6fbdd1e41 100644 --- a/edisgo/opf/powermodels_opf.py +++ b/edisgo/opf/powermodels_opf.py @@ -130,7 +130,7 @@ def _convert(o): logger.info("starting julia process") julia_process = subprocess.Popen( [ - "julia", + "/opt/julia-1.8.3/bin/julia", os.path.join(opf_dir, "eDisGo_OPF.jl/Main.jl"), pm["name"], solution_dir, From d6eddc27895a5a41eaf1e6a0a727de4512a415f0 Mon Sep 17 00:00:00 2001 From: Moritz Schloesser Date: Thu, 21 May 2026 16:26:21 +0200 Subject: [PATCH 19/37] Fix OPF result write-back, flex resolution and overlying-grid SOC reindex MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit powermodels_io.from_powermodels now slices the destination time index explicitly when writing OPF flex results back to the EDisGo timeseries (gen_nd, heatpumps, electromobility, dsm, storage). Using `loc[:, names]` overwrites every row of the underlying DataFrame even when the OPF only covered a subset of timesteps; restricting to `timeseries.timeindex` keeps untouched rows intact. task_optimize: - fixes typo `flexbile` → `flexible` in the dsm shortcut - corrects the dsm condition (was checking `is not None`, should populate when `None`) - materializes empty flex lists once instead of repeating `or []` at every call site task_import_overlying_grid_data reindexes the three SOC attributes (storage_units_soc, thermal_storage_units_{central,decentral}_soc) to timeindex + 1 extra step, because PowerModels expects the end-of-period SOC value; non-SOC overlying-grid attributes still reindex to the plain timeindex. uc4_example preset reworked: switches to opf_version 3 (HV constraints from overlying grid), drops base_reinforce + check_integrity from the pipeline, adds an explicit `overlying_grid.path` config slot, splits import_electromobility kwargs onto separate lines, and enables archive + save_opf_results on the final save step. --- edisgo/io/powermodels_io.py | 33 +++++++----- edisgo/run/presets/uc4_example.yaml | 47 ++++++++--------- edisgo/run/tasks/analysis.py | 79 +++++++++++++++++++++-------- edisgo/run/tasks/io.py | 14 +++-- 4 files changed, 108 insertions(+), 65 deletions(-) diff --git a/edisgo/io/powermodels_io.py b/edisgo/io/powermodels_io.py index e82cc4734..6bf21affb 100644 --- a/edisgo/io/powermodels_io.py +++ b/edisgo/io/powermodels_io.py @@ -250,10 +250,10 @@ def from_powermodels( Base value of apparent power for per unit system. Default: 1 MVA. """ - if type(pm_results) == str: + if isinstance(pm_results, str): with open(pm_results) as f: pm = json.loads(json.load(f)) - elif type(pm_results) == dict: + elif isinstance(pm_results, dict): pm = pm_results else: raise ValueError( @@ -306,17 +306,20 @@ def from_powermodels( ] results = pd.DataFrame(index=timesteps, columns=names, data=data) if (flex == "gen_nd") & (pm["nw"]["1"]["opf_version"] in [3, 4]): - edisgo_object.timeseries._generators_active_power.loc[:, names] = ( + ti = edisgo_object.timeseries.timeindex + edisgo_object.timeseries._generators_active_power.loc[ti, names] = ( edisgo_object.timeseries.generators_active_power.loc[:, names].values - results[names].values ) elif flex in ["heatpumps", "electromobility"]: - edisgo_object.timeseries._loads_active_power.loc[:, names] = results[ + ti = edisgo_object.timeseries.timeindex + edisgo_object.timeseries._loads_active_power.loc[ti, names] = results[ names ].values elif flex == "dsm": - edisgo_object.timeseries._loads_active_power.loc[:, names] = ( - edisgo_object.timeseries._loads_active_power.loc[:, names].values + ti = edisgo_object.timeseries.timeindex + edisgo_object.timeseries._loads_active_power.loc[ti, names] = ( + edisgo_object.timeseries._loads_active_power.loc[ti, names].values + results[names].values ) elif flex == "storage": @@ -328,8 +331,9 @@ def from_powermodels( data=results[names].values, ) else: + ti = edisgo_object.timeseries.timeindex edisgo_object.timeseries._storage_units_active_power.loc[ - :, names + ti, names ] = results[names].values except AttributeError: setattr( @@ -787,8 +791,8 @@ def _build_branch(edisgo_obj, psa_net, pm, flexible_storage_units, s_base): # only modify r, x and l values if min value is too small branches[par] = val.clip(lower=min_value) logger.warning( - f"Min value of {text} is too small. Lowest {100 * quant}% of {text} values will be set " - f"to {min_value} {unit}" + f"Min value of {text} is too small. Lowest {100 * quant}% of " + f"{text} values will be set to {min_value} {unit}" ) for branch_i in np.arange(len(branches.index)): @@ -933,8 +937,8 @@ def _build_load( pf, sign = _get_pf(edisgo_obj, pm, idx_bus, "charging_point") else: logger.warning( - f"No type specified for load {loads_df.index[load_i]}. Power factor and sign will" - "be set for conventional load." + f"No type specified for load {loads_df.index[load_i]}. " + "Power factor and sign will be set for conventional load." ) pf, sign = _get_pf(edisgo_obj, pm, idx_bus, "conventional_load") p_d = psa_net.loads_t.p_set[loads_df.index[load_i]] @@ -1219,9 +1223,10 @@ def _build_heatpump(psa_net, pm, edisgo_obj, s_base, flexible_hps): comparison = (heat_df2[hp_p_nom.index] > hp_cop * hp_p_nom.squeeze()).any() if comparison.any(): logger.warning( - "Heat demand is higher than rated heatpump power" - f" of heatpumps: {comparison.index[comparison.values].values}. Demand can not be covered if no sufficient" - " heat storage capacities are available." + "Heat demand is higher than rated heatpump power of heatpumps: " + f"{comparison.index[comparison.values].values}. " + "Demand can not be covered if no sufficient heat storage " + "capacities are available." ) for hp_i in np.arange(len(heat_df.index)): idx_bus = _mapping(psa_net, edisgo_obj, heat_df.bus.iloc[hp_i]) diff --git a/edisgo/run/presets/uc4_example.yaml b/edisgo/run/presets/uc4_example.yaml index 2d2650279..d3f9efd90 100644 --- a/edisgo/run/presets/uc4_example.yaml +++ b/edisgo/run/presets/uc4_example.yaml @@ -1,54 +1,51 @@ _comment: | - UC3 — OPF with full flexibility: - Like UC1 but loads real egon_data time series (oedb) and runs a - powermodels OPF over flexibilities (heat pumps, EV, DSM, storage) - before the final reinforce. Cost delta = extra reinforcement needed - under optimal flex dispatch. + UC4 — OPF with full flexibility: + Loads real egon_data time series (oedb) and runs a powermodels OPF + over flexibilities (heat pumps, EV, DSM, storage) with HV requirements + from overlying grid. opf_version 3 activates HV-constraints from + overlying_grid CSV directory. _workflow: - - setup_grid: load ding0 topology, import generators - - base_reinforce: worst-case TS + reinforce + reset equipment_changes - - import_generators: from edon-data - - import_heat_pumps: from egon_data + - setup_grid: load ding0 topology + - import_generators: from egon_data - import_home_batteries: from egon_data + - import_heat_pumps: from egon_data - import_dsm: from egon_data - import_electromobility: from egon_data (dumb charging, flex bands) - oedb_ts: real wind/solar + load time series (24 h, 2035) + - apply_charging_strategy: dumb - apply_heat_pump_strategy: uncontrolled (overwritten by OPF) - - reactive_power - - check_integrity - - optimize: pm_optimize with flex assets (SOC, opf v2) - - reinforce: final reinforcement - - save + - import_overlying_grid_data: HV constraints from CSV dir + - optimize: pm_optimize with flex assets (SOC, opf v3 = HV constraints) scenario: eGon2035 + grid: ding0_path: "/path/to/ding0_grid" - # grid path should be set in run file legacy_ding0_grids: false - # legacy parameter too database: ssh: enabled: false timeindex: {start: "2035-01-01", periods: 24, freq: h} -# only set in preset, not set in run-functions (eDisGo and eGo) + +overlying_grid: + path: "/path/to/overlying_grid_csv_dir" results: directory: results/uc4_example -# actually all parameters pipeline: - setup_grid - - base_reinforce - import_generators - import_home_batteries - import_heat_pumps - import_dsm - # where is decided, which electromobility use cases are used? - - import_electromobility: {charging_strategy: null, flexibility_bands_ucs : ["home", "work", "public", "hpc"]} + - import_electromobility: + charging_strategy: null + flexibility_bands_ucs: ["home", "work", "public", "hpc"] - oedb_ts: dispatchable: {other: 0.7} timeindex: {start: "2035-01-01", periods: 24, freq: h} @@ -56,11 +53,11 @@ pipeline: - apply_heat_pump_strategy: {strategy: uncontrolled} - import_overlying_grid_data - reactive_power - - check_integrity - optimize: - # where is decided which flexibilities are used in the OPF? flexible: [heat_pumps, storage, charging_points, dsm] method: soc - opf_version: 2 + opf_version: 3 - reinforce - - save + - save: + archive: true + save_opf_results: true diff --git a/edisgo/run/tasks/analysis.py b/edisgo/run/tasks/analysis.py index f028bb31c..becff93b9 100644 --- a/edisgo/run/tasks/analysis.py +++ b/edisgo/run/tasks/analysis.py @@ -21,6 +21,7 @@ produce a "base" grid whose subsequent reinforce costs reflect only a scenario overlay. """ + from __future__ import annotations import pandas as pd @@ -55,8 +56,15 @@ def task_check_integrity(edisgo, ctx): @register_task("analyze") -def task_analyze(edisgo, ctx, *, mode=None, timesteps=None, - raise_not_converged=False, troubleshooting_mode=None): +def task_analyze( + edisgo, + ctx, + *, + mode=None, + timesteps=None, + raise_not_converged=False, + troubleshooting_mode=None, +): """ Run AC power flow over the active time series. @@ -97,18 +105,26 @@ def task_analyze(edisgo, ctx, *, mode=None, timesteps=None, ctx.flags["not_converged_steps"] = len(not_converged) if len(not_converged) > 0: ctx.logger.warning( - f"Power flow did not converge for {len(not_converged)} " - f"time steps." + f"Power flow did not converge for {len(not_converged)} time steps." ) return edisgo @register_task("reinforce") -def task_reinforce(edisgo, ctx, *, timesteps_pfa=None, reduced_analysis=False, - copy_grid=False, max_while_iterations=20, - split_voltage_band=True, mode=None, - without_generator_import=False, n_minus_one=False, - catch_convergence_problems=False): +def task_reinforce( + edisgo, + ctx, + *, + timesteps_pfa=None, + reduced_analysis=False, + copy_grid=False, + max_while_iterations=20, + split_voltage_band=True, + mode=None, + without_generator_import=False, + n_minus_one=False, + catch_convergence_problems=False, +): """ Run iterative grid reinforcement. @@ -167,8 +183,9 @@ def task_reinforce(edisgo, ctx, *, timesteps_pfa=None, reduced_analysis=False, @register_task("base_reinforce") -def task_base_reinforce(edisgo, ctx, *, cases=None, - reset_equipment_changes=True, save_artifact=True): +def task_base_reinforce( + edisgo, ctx, *, cases=None, reset_equipment_changes=True, save_artifact=True +): """ Produce a base-reinforced grid and reset the cost accumulator. @@ -242,10 +259,20 @@ def task_base_reinforce(edisgo, ctx, *, cases=None, @register_task("optimize") -def task_optimize(edisgo, ctx, *, flexible=None, flexible_cps=None, - flexible_hps=None, flexible_loads=None, - flexible_storage_units=None, opf_version=2, method="soc", - warm_start=False, s_base=1): +def task_optimize( + edisgo, + ctx, + *, + flexible=None, + flexible_cps=None, + flexible_hps=None, + flexible_loads=None, + flexible_storage_units=None, + opf_version=2, + method="soc", + warm_start=False, + s_base=1, +): """ Run a powermodels optimal-power-flow (OPF) over flexibilities. @@ -304,15 +331,23 @@ def task_optimize(edisgo, ctx, *, flexible=None, flexible_cps=None, ].index.tolist() if flexible_storage_units is None and "storage" in flexible: flexible_storage_units = edisgo.topology.storage_units_df.index.tolist() - if flexible_loads is not None and "dsm" in flexbile: - flexible_loads = edisgo.dsm.p_min.columns.values - + if flexible_loads is None and "dsm" in flexible: + flexible_loads = edisgo.dsm.p_min.columns.values + + if flexible_cps is None: + flexible_cps = [] + if flexible_hps is None: + flexible_hps = [] + if flexible_loads is None: + flexible_loads = [] + if flexible_storage_units is None: + flexible_storage_units = [] edisgo.pm_optimize( - flexible_cps=flexible_cps or [], - flexible_hps=flexible_hps or [], - flexible_loads=flexible_loads or [], - flexible_storage_units=flexible_storage_units or [], + flexible_cps=flexible_cps, + flexible_hps=flexible_hps, + flexible_loads=flexible_loads, + flexible_storage_units=flexible_storage_units, opf_version=opf_version, method=method, warm_start=warm_start, diff --git a/edisgo/run/tasks/io.py b/edisgo/run/tasks/io.py index 4e2e84d15..c2b748cf0 100644 --- a/edisgo/run/tasks/io.py +++ b/edisgo/run/tasks/io.py @@ -262,6 +262,14 @@ def task_import_overlying_grid_data(edisgo, ctx, *, overlying_grid_path=None): # CSVs may use a different year — shift year then reindex edisgo_ti = edisgo.timeseries.timeindex if not edisgo_ti.empty: + # SOC needs one extra step at the end (end-of-period state) + ti_freq = edisgo_ti.freq or (edisgo_ti[1] - edisgo_ti[0]) + edisgo_ti_plus1 = edisgo_ti.union([edisgo_ti[-1] + ti_freq]) + soc_attrs = { + "storage_units_soc", + "thermal_storage_units_decentral_soc", + "thermal_storage_units_central_soc", + } for attr in edisgo.overlying_grid._attributes: ts = getattr(edisgo.overlying_grid, attr) if ts.empty: @@ -270,10 +278,8 @@ def task_import_overlying_grid_data(edisgo, ctx, *, overlying_grid_path=None): edisgo_year = edisgo_ti[0].year if csv_year != edisgo_year: ts.index = ts.index + pd.DateOffset(years=edisgo_year - csv_year) - if isinstance(ts, pd.Series): - setattr(edisgo.overlying_grid, attr, ts.reindex(edisgo_ti)) - else: - setattr(edisgo.overlying_grid, attr, ts.reindex(edisgo_ti)) + target_ti = edisgo_ti_plus1 if attr in soc_attrs else edisgo_ti + setattr(edisgo.overlying_grid, attr, ts.reindex(target_ti)) # load dispatchable generator and renewables time series from the same dir disp_path = os.path.join( From fc84a14bc5d3b8f524a028f8ed11aeb4fc3ca02b Mon Sep 17 00:00:00 2001 From: Moritz Schloesser Date: Thu, 21 May 2026 17:40:36 +0200 Subject: [PATCH 20/37] Add overly_grid boolean --- edisgo/run/tasks/io.py | 171 +++++++++++++++++++++-------------------- 1 file changed, 87 insertions(+), 84 deletions(-) diff --git a/edisgo/run/tasks/io.py b/edisgo/run/tasks/io.py index c2b748cf0..258148eb8 100644 --- a/edisgo/run/tasks/io.py +++ b/edisgo/run/tasks/io.py @@ -227,95 +227,98 @@ def task_import_overlying_grid_data(edisgo, ctx, *, overlying_grid_path=None): """ import pandas as pd - + overlying_grid_data = getattr(ctx, "overlying_grid_data", None) - if overlying_grid_data is not None: - # eTraGo results dict — set standard overlying-grid attributes - for attr in edisgo.overlying_grid._attributes: - if attr in overlying_grid_data: - setattr(edisgo.overlying_grid, attr, overlying_grid_data[attr]) - # set generator time series - edisgo.set_time_series_active_power_predefined( - dispatchable_generators_ts=overlying_grid_data.get( - "dispatchable_generators_active_power" - ), - fluctuating_generators_ts=overlying_grid_data.get("renewables_potential"), - ) - return edisgo + if overlying_grid: + + + if overlying_grid_data is not None: + # eTraGo results dict — set standard overlying-grid attributes + for attr in edisgo.overlying_grid._attributes: + if attr in overlying_grid_data: + setattr(edisgo.overlying_grid, attr, overlying_grid_data[attr]) + # set generator time series + edisgo.set_time_series_active_power_predefined( + dispatchable_generators_ts=overlying_grid_data.get( + "dispatchable_generators_active_power" + ), + fluctuating_generators_ts=overlying_grid_data.get("renewables_potential"), + ) + return edisgo - # resolve path: explicit arg → runner config overlying_grid.path → skip - if overlying_grid_path is None: - overlying_grid_path = (ctx.raw_config.get("overlying_grid") or {}).get("path") + # resolve path: explicit arg → runner config overlying_grid.path → skip + if overlying_grid_path is None: + overlying_grid_path = (ctx.raw_config.get("overlying_grid_path") or {}).get("path") - if overlying_grid_path is None: - ctx.logger.warning( - "task 'import_overlying_grid_data': no overlying_grid_data or " - "overlying_grid_path provided — skipping." - ) - return edisgo - - # load overlying-grid attributes from CSV directory - edisgo.overlying_grid.from_csv(overlying_grid_path) - - # reindex overlying-grid attributes to match edisgo timeindex - # CSVs may use a different year — shift year then reindex - edisgo_ti = edisgo.timeseries.timeindex - if not edisgo_ti.empty: - # SOC needs one extra step at the end (end-of-period state) - ti_freq = edisgo_ti.freq or (edisgo_ti[1] - edisgo_ti[0]) - edisgo_ti_plus1 = edisgo_ti.union([edisgo_ti[-1] + ti_freq]) - soc_attrs = { - "storage_units_soc", - "thermal_storage_units_decentral_soc", - "thermal_storage_units_central_soc", - } - for attr in edisgo.overlying_grid._attributes: - ts = getattr(edisgo.overlying_grid, attr) - if ts.empty: - continue - csv_year = ts.index[0].year - edisgo_year = edisgo_ti[0].year - if csv_year != edisgo_year: - ts.index = ts.index + pd.DateOffset(years=edisgo_year - csv_year) - target_ti = edisgo_ti_plus1 if attr in soc_attrs else edisgo_ti - setattr(edisgo.overlying_grid, attr, ts.reindex(target_ti)) - - # load dispatchable generator and renewables time series from the same dir - disp_path = os.path.join( - overlying_grid_path, "dispatchable_generators_active_power.csv" - ) - if os.path.isfile(disp_path): - disp_ts = pd.read_csv(disp_path, index_col=0, parse_dates=True) - if not edisgo_ti.empty: - csv_year = disp_ts.index[0].year - edisgo_year = edisgo_ti[0].year - if csv_year != edisgo_year: - disp_ts.index = disp_ts.index + pd.DateOffset( - years=edisgo_year - csv_year - ) - disp_ts = disp_ts.reindex(edisgo_ti) - else: - disp_ts = None - - pot_path = os.path.join(overlying_grid_path, "renewables_potential.csv") - if os.path.isfile(pot_path): - pot_ts = pd.read_csv(pot_path, index_col=0, parse_dates=True) + if overlying_grid_path is None: + ctx.logger.warning( + "task 'import_overlying_grid_data': no overlying_grid_data or " + "overlying_grid_path provided — skipping." + ) + return edisgo + + # load overlying-grid attributes from CSV directory + edisgo.overlying_grid.from_csv(overlying_grid_path) + + # reindex overlying-grid attributes to match edisgo timeindex + # CSVs may use a different year — shift year then reindex + edisgo_ti = edisgo.timeseries.timeindex if not edisgo_ti.empty: - csv_year = pot_ts.index[0].year - edisgo_year = edisgo_ti[0].year - if csv_year != edisgo_year: - pot_ts.index = pot_ts.index + pd.DateOffset( - years=edisgo_year - csv_year - ) - pot_ts = pot_ts.reindex(edisgo_ti) - else: - pot_ts = None - - if disp_ts is not None or pot_ts is not None: - edisgo.set_time_series_active_power_predefined( - dispatchable_generators_ts=disp_ts, - fluctuating_generators_ts=pot_ts, + # SOC needs one extra step at the end (end-of-period state) + ti_freq = edisgo_ti.freq or (edisgo_ti[1] - edisgo_ti[0]) + edisgo_ti_plus1 = edisgo_ti.union([edisgo_ti[-1] + ti_freq]) + soc_attrs = { + "storage_units_soc", + "thermal_storage_units_decentral_soc", + "thermal_storage_units_central_soc", + } + for attr in edisgo.overlying_grid._attributes: + ts = getattr(edisgo.overlying_grid, attr) + if ts.empty: + continue + csv_year = ts.index[0].year + edisgo_year = edisgo_ti[0].year + if csv_year != edisgo_year: + ts.index = ts.index + pd.DateOffset(years=edisgo_year - csv_year) + target_ti = edisgo_ti_plus1 if attr in soc_attrs else edisgo_ti + setattr(edisgo.overlying_grid, attr, ts.reindex(target_ti)) + + # load dispatchable generator and renewables time series from the same dir + disp_path = os.path.join( + overlying_grid_path, "dispatchable_generators_active_power.csv" ) + if os.path.isfile(disp_path): + disp_ts = pd.read_csv(disp_path, index_col=0, parse_dates=True) + if not edisgo_ti.empty: + csv_year = disp_ts.index[0].year + edisgo_year = edisgo_ti[0].year + if csv_year != edisgo_year: + disp_ts.index = disp_ts.index + pd.DateOffset( + years=edisgo_year - csv_year + ) + disp_ts = disp_ts.reindex(edisgo_ti) + else: + disp_ts = None + + pot_path = os.path.join(overlying_grid_path, "renewables_potential.csv") + if os.path.isfile(pot_path): + pot_ts = pd.read_csv(pot_path, index_col=0, parse_dates=True) + if not edisgo_ti.empty: + csv_year = pot_ts.index[0].year + edisgo_year = edisgo_ti[0].year + if csv_year != edisgo_year: + pot_ts.index = pot_ts.index + pd.DateOffset( + years=edisgo_year - csv_year + ) + pot_ts = pot_ts.reindex(edisgo_ti) + else: + pot_ts = None + + if disp_ts is not None or pot_ts is not None: + edisgo.set_time_series_active_power_predefined( + dispatchable_generators_ts=disp_ts, + fluctuating_generators_ts=pot_ts, + ) return edisgo From d4672e370d7af9c6e22468074d098234cab5d238 Mon Sep 17 00:00:00 2001 From: Jonas Danke Date: Fri, 22 May 2026 10:48:29 +0200 Subject: [PATCH 21/37] Add full-flex distribution OPF configuration without overlying-grid constraints --- edisgo/run/presets/flex_opf_full.yaml | 61 +++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) create mode 100644 edisgo/run/presets/flex_opf_full.yaml diff --git a/edisgo/run/presets/flex_opf_full.yaml b/edisgo/run/presets/flex_opf_full.yaml new file mode 100644 index 000000000..2ddb5cca7 --- /dev/null +++ b/edisgo/run/presets/flex_opf_full.yaml @@ -0,0 +1,61 @@ +_comment: | + Full-flex distribution OPF, no overlying-grid constraints. + Loads real egon_data time series (oedb) and runs a powermodels OPF + over the full flexibility set (heat pumps, EV, DSM, storage) on the + distribution grid alone. opf_version 2 — no HV requirements from an + overlying grid (use the variant with opf_version 3 for that). + Suitable for runs without eTraGo / without an overlying-grid CSV. + +_workflow: + - setup_grid: load ding0 topology + - import_generators: from egon_data + - import_home_batteries: from egon_data + - import_heat_pumps: from egon_data + - import_dsm: from egon_data + - import_electromobility: from egon_data (dumb charging, flex bands) + - oedb_ts: real wind/solar + load time series (24 h, 2035) + - apply_charging_strategy: dumb + - apply_heat_pump_strategy: uncontrolled (overwritten by OPF) + - optimize: pm_optimize with full flex set (SOC, opf v2) + - reinforce: final reinforcement under optimized dispatch + - save + +scenario: eGon2035 + +grid: + ding0_path: "/path/to/ding0_grid" + legacy_ding0_grids: false + +database: + ssh: + enabled: false + +timeindex: {start: "2035-01-01", periods: 24, freq: h} + +results: + directory: results/flex_opf_full + + +pipeline: + - setup_grid + - import_generators + - import_home_batteries + - import_heat_pumps + - import_dsm + - import_electromobility: + charging_strategy: null + flexibility_bands_ucs: ["home", "work", "public", "hpc"] + - oedb_ts: + dispatchable: {other: 0.7} + timeindex: {start: "2035-01-01", periods: 24, freq: h} + - apply_charging_strategy: {strategy: dumb} + - apply_heat_pump_strategy: {strategy: uncontrolled} + - reactive_power + - optimize: + flexible: [heat_pumps, storage, charging_points, dsm] + method: soc + opf_version: 2 + - reinforce + - save: + archive: true + save_opf_results: true From 8d6a947c72dad238af556eac8927284244e0c232 Mon Sep 17 00:00:00 2001 From: Moritz Schloesser Date: Wed, 27 May 2026 10:00:09 +0200 Subject: [PATCH 22/37] Add overlying grid enabler and source selection --- edisgo/run/config.py | 2 + edisgo/run/presets/uc4_example.yaml | 6 ++- edisgo/run/tasks/io.py | 82 +++++++++++++++++------------ 3 files changed, 55 insertions(+), 35 deletions(-) diff --git a/edisgo/run/config.py b/edisgo/run/config.py index 5c4ee8573..1dddb5690 100644 --- a/edisgo/run/config.py +++ b/edisgo/run/config.py @@ -399,6 +399,8 @@ def _adapt_ego_legacy(cfg: dict) -> dict: "grid": {"ding0_path": edisgo_cfg.get("grid_path")}, "results": {"directory": edisgo_cfg.get("results")}, "pipeline": mapped, + "overlying_grid": {"path": edisgo_cfg.get("overlying_grid_source")}, + "overlying_grid": {"selection": edisgo_cfg.get("overlying_grid")} } if "database" in cfg: adapted["database"] = cfg["database"] diff --git a/edisgo/run/presets/uc4_example.yaml b/edisgo/run/presets/uc4_example.yaml index d3f9efd90..b3cc3cfdc 100644 --- a/edisgo/run/presets/uc4_example.yaml +++ b/edisgo/run/presets/uc4_example.yaml @@ -31,7 +31,9 @@ database: timeindex: {start: "2035-01-01", periods: 24, freq: h} overlying_grid: - path: "/path/to/overlying_grid_csv_dir" + enabled: false # master switch — set true to activate import_overlying_grid_data + source: csv # "csv" (load from path) or "etrago" (consume overlying_grid_data kwarg) + path: "/path/to/overlying_grid_csv_dir" # required when source == csv; full leaf dir for ONE grid (like ding0_path) results: directory: results/uc4_example @@ -56,7 +58,7 @@ pipeline: - optimize: flexible: [heat_pumps, storage, charging_points, dsm] method: soc - opf_version: 3 + opf_version: 2 - reinforce - save: archive: true diff --git a/edisgo/run/tasks/io.py b/edisgo/run/tasks/io.py index 258148eb8..0bf51bf3a 100644 --- a/edisgo/run/tasks/io.py +++ b/edisgo/run/tasks/io.py @@ -196,29 +196,35 @@ def task_import_overlying_grid_data(edisgo, ctx, *, overlying_grid_path=None): """ Import overlying grid data into the EDisGo instance. - When ``overlying_grid_data`` is a dict of DataFrames (as returned by - ``get_etrago_results_per_bus``), the overlying-grid attributes and - dispatchable/fluctuating generator time series are set from it. + Behavior controlled by ``ctx.raw_config['overlying_grid']``: - When ``overlying_grid_path`` is a directory path, the overlying-grid - attributes are loaded from CSV files in that directory, and - ``dispatchable_generators_active_power.csv`` / - ``renewables_potential.csv`` are applied as generator time series - if present. + * ``enabled`` (bool) — master switch. Falsy → task no-ops. + * ``source`` (str) — ``"etrago"`` or ``"csv"``. - Falls back to ``ctx.raw_config['eDisGo']['overlying_grid_source']`` - as the directory path when neither argument is given. + ``source: etrago`` consumes ``ctx.overlying_grid_data`` (a dict of + DataFrames as returned by ``get_etrago_results_per_bus``), injected + via the ``overlying_grid_data=`` kwarg of + :func:`edisgo.run.run_edisgo`. Sets overlying-grid attributes and + dispatchable/fluctuating generator time series from it. + + ``source: csv`` loads overlying-grid attributes from CSVs in + ``overlying_grid.path`` (full directory path for ONE grid — same + leaf-dir convention as ``grid.ding0_path``; callers handling many + grids must compose the per-grid subdirectory themselves). + ``dispatchable_generators_active_power.csv`` and + ``renewables_potential.csv``, if present in that dir, are applied + as generator time series. Parameters ---------- edisgo : edisgo.EDisGo EDisGo instance to modify in place. ctx : RunContext - Run context. + Run context. Reads ``raw_config['overlying_grid']`` and + ``overlying_grid_data`` attribute. overlying_grid_path : str, optional - Directory containing overlying-grid CSV files. - overlying_grid_data : dict, optional - Dict of DataFrames as returned by ``get_etrago_results_per_bus``. + CSV directory override (takes precedence over + ``overlying_grid.path`` from the config) when ``source='csv'``. Returns ------- @@ -228,33 +234,38 @@ def task_import_overlying_grid_data(edisgo, ctx, *, overlying_grid_path=None): """ import pandas as pd - overlying_grid_data = getattr(ctx, "overlying_grid_data", None) + og_cfg = ctx.raw_config.get("overlying_grid") or {} + if not og_cfg.get("enabled"): + return edisgo - if overlying_grid: - + source = og_cfg.get("source") + overlying_grid_data = getattr(ctx, "overlying_grid_data", None) - if overlying_grid_data is not None: - # eTraGo results dict — set standard overlying-grid attributes - for attr in edisgo.overlying_grid._attributes: - if attr in overlying_grid_data: - setattr(edisgo.overlying_grid, attr, overlying_grid_data[attr]) - # set generator time series - edisgo.set_time_series_active_power_predefined( - dispatchable_generators_ts=overlying_grid_data.get( - "dispatchable_generators_active_power" - ), - fluctuating_generators_ts=overlying_grid_data.get("renewables_potential"), + if source == "etrago": + if overlying_grid_data is None: + ctx.logger.warning( + "task 'import_overlying_grid_data': source='etrago' but no " + "overlying_grid_data passed to run_edisgo — skipping." ) return edisgo + for attr in edisgo.overlying_grid._attributes: + if attr in overlying_grid_data: + setattr(edisgo.overlying_grid, attr, overlying_grid_data[attr]) + edisgo.set_time_series_active_power_predefined( + dispatchable_generators_ts=overlying_grid_data.get( + "dispatchable_generators_active_power" + ), + fluctuating_generators_ts=overlying_grid_data.get("renewables_potential"), + ) + return edisgo - # resolve path: explicit arg → runner config overlying_grid.path → skip + if source == "csv": if overlying_grid_path is None: - overlying_grid_path = (ctx.raw_config.get("overlying_grid_path") or {}).get("path") - + overlying_grid_path = og_cfg.get("path") if overlying_grid_path is None: ctx.logger.warning( - "task 'import_overlying_grid_data': no overlying_grid_data or " - "overlying_grid_path provided — skipping." + "task 'import_overlying_grid_data': source='csv' but no " + "overlying_grid.path configured — skipping." ) return edisgo @@ -320,5 +331,10 @@ def task_import_overlying_grid_data(edisgo, ctx, *, overlying_grid_path=None): dispatchable_generators_ts=disp_ts, fluctuating_generators_ts=pot_ts, ) + return edisgo + ctx.logger.warning( + f"task 'import_overlying_grid_data': unknown source={source!r} " + "(expected 'etrago' or 'csv') — skipping." + ) return edisgo From 04aad53141c9244b081341e731607abab87a76d8 Mon Sep 17 00:00:00 2001 From: Moritz Schloesser Date: Wed, 27 May 2026 11:29:08 +0200 Subject: [PATCH 23/37] Change opf_version --- edisgo/run/presets/uc4_example.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/edisgo/run/presets/uc4_example.yaml b/edisgo/run/presets/uc4_example.yaml index b3cc3cfdc..5ec024f6c 100644 --- a/edisgo/run/presets/uc4_example.yaml +++ b/edisgo/run/presets/uc4_example.yaml @@ -58,7 +58,7 @@ pipeline: - optimize: flexible: [heat_pumps, storage, charging_points, dsm] method: soc - opf_version: 2 + opf_version: 3 - reinforce - save: archive: true From f1be570aa2e54fdd31b30533b7e1638031a97bf3 Mon Sep 17 00:00:00 2001 From: ClaraBuettner Date: Wed, 27 May 2026 11:38:12 +0200 Subject: [PATCH 24/37] Add renewables_potential and dispatchable_generator timeseries to OverlyingGrid object --- edisgo/network/overlying_grid.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/edisgo/network/overlying_grid.py b/edisgo/network/overlying_grid.py index 241768a20..05edc2168 100644 --- a/edisgo/network/overlying_grid.py +++ b/edisgo/network/overlying_grid.py @@ -84,6 +84,17 @@ def __init__(self, **kwargs): "feedin_district_heating", pd.DataFrame(dtype="float64") ) + self.dispatchable_generators_active_power = kwargs.get( + "dispatchable_generators_active_power", pd.DataFrame(dtype="float64") + ) + + self.dispatchable_generators_reactive_power = kwargs.get( + "dispatchable_generators_reactive_power", pd.DataFrame(dtype="float64") + ) + + self.renewables_potential = kwargs.get( + "renewables_potential", pd.Series(dtype="float64") + ) @property def _attributes(self): return [ @@ -97,6 +108,9 @@ def _attributes(self): "heat_pump_central_active_power", "thermal_storage_units_central_soc", "feedin_district_heating", + "dispatchable_generators_active_power", + "dispatchable_generators_reactive_power", + "renewables_potential", ] def reduce_memory(self, attr_to_reduce=None, to_type="float32"): From e79dcebeab748417c2154ae93a7340c7e4ea28cf Mon Sep 17 00:00:00 2001 From: ClaraBuettner Date: Wed, 27 May 2026 11:40:52 +0200 Subject: [PATCH 25/37] Import overlying_grid_data only from csv files when it is not directly handed over --- edisgo/run/tasks/io.py | 107 ++++++++++++++++++++++++----------------- 1 file changed, 63 insertions(+), 44 deletions(-) diff --git a/edisgo/run/tasks/io.py b/edisgo/run/tasks/io.py index c2b748cf0..b03027795 100644 --- a/edisgo/run/tasks/io.py +++ b/edisgo/run/tasks/io.py @@ -235,14 +235,30 @@ def task_import_overlying_grid_data(edisgo, ctx, *, overlying_grid_path=None): for attr in edisgo.overlying_grid._attributes: if attr in overlying_grid_data: setattr(edisgo.overlying_grid, attr, overlying_grid_data[attr]) + + if not overlying_grid_data.get( + "dispatchable_generators_active_power" + ).empty: # set generator time series - edisgo.set_time_series_active_power_predefined( - dispatchable_generators_ts=overlying_grid_data.get( - "dispatchable_generators_active_power" - ), - fluctuating_generators_ts=overlying_grid_data.get("renewables_potential"), - ) - return edisgo + edisgo.set_time_series_active_power_predefined( + dispatchable_generators_ts=overlying_grid_data.get( + "dispatchable_generators_active_power" + ), + ) + if not overlying_grid_data.get("renewables_potential").empty: + pot_ts = overlying_grid_data.get("renewables_potential") + edisgo_ti = edisgo.timeseries.timeindex + csv_year = pot_ts.index[0].year + edisgo_year = edisgo_ti[0].year + if csv_year != edisgo_year: + pot_ts.index = pot_ts.index + pd.DateOffset( + years=edisgo_year - csv_year + ) + pot_ts = pot_ts.reindex(edisgo_ti) + + edisgo.set_time_series_active_power_predefined( + fluctuating_generators_ts=pot_ts, + ) # resolve path: explicit arg → runner config overlying_grid.path → skip if overlying_grid_path is None: @@ -255,8 +271,9 @@ def task_import_overlying_grid_data(edisgo, ctx, *, overlying_grid_path=None): ) return edisgo - # load overlying-grid attributes from CSV directory - edisgo.overlying_grid.from_csv(overlying_grid_path) + if overlying_grid_data is None: + # load overlying-grid attributes from CSV directory + edisgo.overlying_grid.from_csv(overlying_grid_path) # reindex overlying-grid attributes to match edisgo timeindex # CSVs may use a different year — shift year then reindex @@ -281,41 +298,43 @@ def task_import_overlying_grid_data(edisgo, ctx, *, overlying_grid_path=None): target_ti = edisgo_ti_plus1 if attr in soc_attrs else edisgo_ti setattr(edisgo.overlying_grid, attr, ts.reindex(target_ti)) - # load dispatchable generator and renewables time series from the same dir - disp_path = os.path.join( - overlying_grid_path, "dispatchable_generators_active_power.csv" - ) - if os.path.isfile(disp_path): - disp_ts = pd.read_csv(disp_path, index_col=0, parse_dates=True) - if not edisgo_ti.empty: - csv_year = disp_ts.index[0].year - edisgo_year = edisgo_ti[0].year - if csv_year != edisgo_year: - disp_ts.index = disp_ts.index + pd.DateOffset( - years=edisgo_year - csv_year - ) - disp_ts = disp_ts.reindex(edisgo_ti) - else: - disp_ts = None - - pot_path = os.path.join(overlying_grid_path, "renewables_potential.csv") - if os.path.isfile(pot_path): - pot_ts = pd.read_csv(pot_path, index_col=0, parse_dates=True) - if not edisgo_ti.empty: - csv_year = pot_ts.index[0].year - edisgo_year = edisgo_ti[0].year - if csv_year != edisgo_year: - pot_ts.index = pot_ts.index + pd.DateOffset( - years=edisgo_year - csv_year - ) - pot_ts = pot_ts.reindex(edisgo_ti) - else: - pot_ts = None - - if disp_ts is not None or pot_ts is not None: - edisgo.set_time_series_active_power_predefined( - dispatchable_generators_ts=disp_ts, - fluctuating_generators_ts=pot_ts, + if overlying_grid_data is None: + # load dispatchable generator and renewables time series from the same dir + disp_path = os.path.join( + overlying_grid_path, "dispatchable_generators_active_power.csv" ) + if os.path.isfile(disp_path): + disp_ts = pd.read_csv(disp_path, index_col=0, parse_dates=True) + if not edisgo_ti.empty: + csv_year = disp_ts.index[0].year + edisgo_year = edisgo_ti[0].year + if csv_year != edisgo_year: + disp_ts.index = disp_ts.index + pd.DateOffset( + years=edisgo_year - csv_year + ) + disp_ts = disp_ts.reindex(edisgo_ti) + else: + disp_ts = None + + pot_path = os.path.join(overlying_grid_path, "renewables_potential.csv") + if os.path.isfile(pot_path): + pot_ts = pd.read_csv(pot_path, index_col=0, parse_dates=True) + if not edisgo_ti.empty: + csv_year = pot_ts.index[0].year + edisgo_year = edisgo_ti[0].year + if csv_year != edisgo_year: + pot_ts.index = pot_ts.index + pd.DateOffset( + years=edisgo_year - csv_year + ) + pot_ts = pot_ts.reindex(edisgo_ti) + else: + pot_ts = None + + + if disp_ts is not None or pot_ts is not None: + edisgo.set_time_series_active_power_predefined( + dispatchable_generators_ts=disp_ts, + fluctuating_generators_ts=pot_ts, + ) return edisgo From f31cf31180fd15746affb4f50aba6e9645a22be9 Mon Sep 17 00:00:00 2001 From: ClaraBuettner Date: Wed, 27 May 2026 11:42:05 +0200 Subject: [PATCH 26/37] Select timesteps in overlying_grid_data that are relevant for eDisGo's optimization --- edisgo/io/powermodels_io.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/edisgo/io/powermodels_io.py b/edisgo/io/powermodels_io.py index 6bf21affb..30d753439 100644 --- a/edisgo/io/powermodels_io.py +++ b/edisgo/io/powermodels_io.py @@ -1015,8 +1015,17 @@ def _build_battery_storage( """ branches = pd.concat([psa_net.lines, psa_net.transformers]) if not edisgo_obj.overlying_grid.storage_units_soc.empty: + # Select relevant timesteps + timesteps = edisgo_obj.timeseries.timeindex.union( + [ + edisgo_obj.timeseries.timeindex[-1] + + edisgo_obj.timeseries.timeindex.freq + ] + ) + if edisgo_obj.overlying_grid.storage_units_soc.index[0].year==2011: + timesteps = timesteps.map(lambda t: t.replace(year=2011)) data = pd.concat( - [edisgo_obj.overlying_grid.storage_units_soc] + [edisgo_obj.overlying_grid.storage_units_soc.loc[timesteps]] * len(edisgo_obj.topology.storage_units_df), axis=1, ).values From 58d8f73e154736d5592e7595a8907259d4cfff37 Mon Sep 17 00:00:00 2001 From: ClaraBuettner Date: Wed, 27 May 2026 11:43:09 +0200 Subject: [PATCH 27/37] Workaround for generators_dispatch stored as pandas.DataFrame --- edisgo/io/powermodels_io.py | 28 ++++++++++++++++++++-------- 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/edisgo/io/powermodels_io.py b/edisgo/io/powermodels_io.py index 30d753439..4c8979d6e 100644 --- a/edisgo/io/powermodels_io.py +++ b/edisgo/io/powermodels_io.py @@ -1602,11 +1602,18 @@ def _build_hv_requirements( ) for i in np.arange(len(opf_flex)): - pm["HV_requirements"][str(i + 1)] = { - "P": hv_flex_dict[opf_flex[i]].iloc[0], - "name": opf_flex[i], - "count": count, - } + if type(hv_flex_dict[opf_flex[i]]) == pd.DataFrame: + pm["HV_requirements"][str(i + 1)] = { + "P": hv_flex_dict[opf_flex[i]].sum(axis=1).iloc[0], + "name": opf_flex[i], + "count": count, + } + else: + pm["HV_requirements"][str(i + 1)] = { + "P": hv_flex_dict[opf_flex[i]].iloc[0], + "name": opf_flex[i], + "count": count, + } def _build_timeseries( @@ -1932,9 +1939,14 @@ def _build_component_timeseries( if (kind == "HV_requirements") & (pm["opf_version"] in [3, 4]): for i in np.arange(len(opf_flex)): - pm_comp[(str(i + 1))] = { - "P": hv_flex_dict[opf_flex[i]].round(20).tolist(), - } + if type(hv_flex_dict[opf_flex[i]])==pd.DataFrame: + pm_comp[(str(i + 1))] = { + "P": hv_flex_dict[opf_flex[i]].sum(axis=1).round(20).tolist(), + } + else: + pm_comp[(str(i + 1))] = { + "P": hv_flex_dict[opf_flex[i]].round(20).tolist(), + } pm["time_series"][kind] = pm_comp From b367c8120e978adf89561d96b1f40e8395bbf52f Mon Sep 17 00:00:00 2001 From: ClaraBuettner Date: Wed, 27 May 2026 13:59:11 +0200 Subject: [PATCH 28/37] Workarround for pd.DataFrames in flex_opt data --- edisgo/io/powermodels_io.py | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/edisgo/io/powermodels_io.py b/edisgo/io/powermodels_io.py index 4c8979d6e..66bdd8464 100644 --- a/edisgo/io/powermodels_io.py +++ b/edisgo/io/powermodels_io.py @@ -365,13 +365,22 @@ def from_powermodels( # calculate relative error df2 = deepcopy(df) for flex in df2.columns: - abs_error = abs(df2[flex].values - hv_flex_dict[flex].values) - rel_error = [ - abs_error[i] / hv_flex_dict[flex].iloc[i] - if ((abs_error > 0.01)[i] & (hv_flex_dict[flex].iloc[i] != 0)) - else 0 - for i in range(len(abs_error)) - ] + if type(hv_flex_dict[flex]) == pd.Series: + abs_error = abs(df2[flex].values - hv_flex_dict[flex].values) + rel_error = [ + abs_error[i] / hv_flex_dict[flex].iloc[i] + if ((abs_error > 0.01)[i] & (hv_flex_dict[flex].iloc[i] != 0)) + else 0 + for i in range(len(abs_error)) + ] + else: + abs_error = abs(df2[flex].values - hv_flex_dict[flex].sum(axis=1).values) + rel_error = [ + abs_error[i] / hv_flex_dict[flex].sum(axis=1).iloc[i] + if ((abs_error > 0.01)[i] & (hv_flex_dict[flex].sum(axis=1).iloc[i] != 0)) + else 0 + for i in range(len(abs_error)) + ] df2[flex] = rel_error # write results to edisgo object edisgo_object.opf_results.overlying_grid = pd.DataFrame( From abf7f703c4cc54dc456163ac1a92c6c113f6b429 Mon Sep 17 00:00:00 2001 From: ClaraBuettner Date: Wed, 3 Jun 2026 15:18:05 +0200 Subject: [PATCH 29/37] Replace hard-coded year and select it from the timeindex instead --- edisgo/io/powermodels_io.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/edisgo/io/powermodels_io.py b/edisgo/io/powermodels_io.py index 66bdd8464..a0551ee64 100644 --- a/edisgo/io/powermodels_io.py +++ b/edisgo/io/powermodels_io.py @@ -1031,8 +1031,13 @@ def _build_battery_storage( + edisgo_obj.timeseries.timeindex.freq ] ) - if edisgo_obj.overlying_grid.storage_units_soc.index[0].year==2011: - timesteps = timesteps.map(lambda t: t.replace(year=2011)) + + # If the overlying grid data uses another year in the timeindex then + # edisgo.timindex, unify them + og_year = edisgo_obj.overlying_grid.storage_units_soc.index[0].year + if og_year != edisgo_obj.timeseries.timeindex[0].year: + timesteps = timesteps.map(lambda t: t.replace(year=og_year)) + data = pd.concat( [edisgo_obj.overlying_grid.storage_units_soc.loc[timesteps]] * len(edisgo_obj.topology.storage_units_df), From 76491a3a222e9ee8564c17ba9d36bfeddfe195b7 Mon Sep 17 00:00:00 2001 From: Jonas Danke Date: Wed, 3 Jun 2026 15:39:02 +0200 Subject: [PATCH 30/37] make ding0 grid path variable optional --- edisgo/run/tasks/grid.py | 40 ++++++++++++++++++++++++++++++++++------ 1 file changed, 34 insertions(+), 6 deletions(-) diff --git a/edisgo/run/tasks/grid.py b/edisgo/run/tasks/grid.py index b472f3244..b074b7e68 100644 --- a/edisgo/run/tasks/grid.py +++ b/edisgo/run/tasks/grid.py @@ -10,14 +10,25 @@ a slow "base" phase and one or more fast "scenario" phases that reuse the base-reinforced grid. """ + from __future__ import annotations +import pandas as pd + from edisgo.run.registry import register_task @register_task("setup_grid") -def task_setup_grid(edisgo, ctx, *, timeindex = None, ding0_path=None, legacy_ding0_grids=None, - import_generators=False, generator_scenario=None): +def task_setup_grid( + edisgo, + ctx, + *, + timeindex=None, + ding0_path=None, + legacy_ding0_grids=None, + import_generators=False, + generator_scenario=None, +): """ Load a ding0 grid into an EDisGo instance. @@ -99,10 +110,19 @@ def task_setup_grid(edisgo, ctx, *, timeindex = None, ding0_path=None, legacy_di @register_task("load_from_base") -def task_load_from_base(edisgo, ctx, *, path, reset_equipment_changes=True, - import_timeseries=False, import_results=False, - import_electromobility=False, import_heat_pump=False, - import_dsm=False, import_overlying_grid=False): +def task_load_from_base( + edisgo, + ctx, + *, + path=None, + reset_equipment_changes=True, + import_timeseries=False, + import_results=False, + import_electromobility=False, + import_heat_pump=False, + import_dsm=False, + import_overlying_grid=False, +): """ Reload an EDisGo instance from a previously saved directory/zip. @@ -152,6 +172,14 @@ def task_load_from_base(edisgo, ctx, *, path, reset_equipment_changes=True, from edisgo.edisgo import import_edisgo_from_files + if path is None: + grid_cfg = ctx.raw_config.get("grid", {}) or {} + path = grid_cfg.get("ding0_path") + if path is None: + raise ValueError( + "Task 'load_from_base' requires 'path' either as task " + "parameter or under config.grid.ding0_path." + ) path = str(path) from_zip = path.endswith(".zip") or not os.path.isdir(path) edisgo = import_edisgo_from_files( From 55376727bf275a69c9bcc5cb97f7da452b6c4f41 Mon Sep 17 00:00:00 2001 From: Moritz Schloesser Date: Wed, 10 Jun 2026 09:56:18 +0200 Subject: [PATCH 31/37] Revert "Redirect to installed julia version" This reverts commit 5fce2573c2dc66fdeb81d847ec39bf8bfc65e2a4. --- edisgo/opf/powermodels_opf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/edisgo/opf/powermodels_opf.py b/edisgo/opf/powermodels_opf.py index 6fbdd1e41..c26475141 100644 --- a/edisgo/opf/powermodels_opf.py +++ b/edisgo/opf/powermodels_opf.py @@ -130,7 +130,7 @@ def _convert(o): logger.info("starting julia process") julia_process = subprocess.Popen( [ - "/opt/julia-1.8.3/bin/julia", + "julia", os.path.join(opf_dir, "eDisGo_OPF.jl/Main.jl"), pm["name"], solution_dir, From 74d6bd1c025887f064e2cc46086461c7b7f6db04 Mon Sep 17 00:00:00 2001 From: Jonas Danke Date: Tue, 23 Jun 2026 15:47:15 +0200 Subject: [PATCH 32/37] fix: resolve 10 review findings in the run pipeline framework MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - timeseries.task_manual_ts: pass set_time_series_manual's real kwargs (generators_p/loads_p/... instead of *_active_power) — task was unusable. - config._adapt_ego_legacy: merge the duplicated 'overlying_grid' dict key (the 'path' entry was silently dropped). - tasks.io.import_overlying_grid_data: rewrite tangled control flow — guard .get(...).empty against None, drop the double from_csv, only warn on a genuinely unknown source, and infer the SOC extra step only when derivable (no IndexError on a single-timestamp timeindex). - validator: load_from no longer satisfies the time-series/flex prerequisite for analyze/reinforce/optimize, since _load_artifact reloads with import_timeseries=False and drops flex data (+ regression test). - powermodels_io: unify SOC year via DateOffset instead of replace(year=) to avoid a Feb-29 ValueError on leap-to-non-leap remapping. - context.ensure_engine: connect directly to a configured local postgres (psycopg2, pool_pre_ping) when SSH is disabled and host params are given. - config._resolve_extends: resolve a relative 'extends' against the including file before falling back to a bundled preset. --- edisgo/io/powermodels_io.py | 7 +- edisgo/run/config.py | 25 +++-- edisgo/run/context.py | 36 ++++++- edisgo/run/tasks/io.py | 166 +++++++++++++++------------------ edisgo/run/tasks/timeseries.py | 12 +-- edisgo/run/validator.py | 12 ++- tests/run/test_validator.py | 20 +++- 7 files changed, 161 insertions(+), 117 deletions(-) diff --git a/edisgo/io/powermodels_io.py b/edisgo/io/powermodels_io.py index 4438824ae..2f55104b3 100644 --- a/edisgo/io/powermodels_io.py +++ b/edisgo/io/powermodels_io.py @@ -1040,8 +1040,11 @@ def _build_battery_storage( # If the overlying grid data uses another year in the timeindex then # edisgo.timindex, unify them og_year = edisgo_obj.overlying_grid.storage_units_soc.index[0].year - if og_year != edisgo_obj.timeseries.timeindex[0].year: - timesteps = timesteps.map(lambda t: t.replace(year=og_year)) + year_diff = og_year - edisgo_obj.timeseries.timeindex[0].year + if year_diff != 0: + # Shift by whole years instead of Timestamp.replace(year=...), + # which raises on Feb 29 when the target year is not a leap year. + timesteps = timesteps + pd.DateOffset(years=year_diff) data = pd.concat( [edisgo_obj.overlying_grid.storage_units_soc.loc[timesteps]] diff --git a/edisgo/run/config.py b/edisgo/run/config.py index 1dddb5690..6f095133f 100644 --- a/edisgo/run/config.py +++ b/edisgo/run/config.py @@ -128,9 +128,10 @@ def _resolve_extends(cfg: dict, base_dir: Path) -> dict: Resolve an ``extends:`` reference and deep-merge parent into child. The parent is loaded recursively, so a chain of ``extends:`` works. - References are looked up as (1) a bundled preset name under - :mod:`edisgo.run.presets`, (2) a path relative to ``base_dir``. - The child's keys override the parent's on conflicts. + A relative reference is looked up as (1) a path relative to + ``base_dir``, (2) a bundled preset name under + :mod:`edisgo.run.presets`. The child's keys override the parent's on + conflicts. Parameters ---------- @@ -156,11 +157,15 @@ def _resolve_extends(cfg: dict, base_dir: Path) -> dict: return cfg ext_path = Path(ext).expanduser() if not ext_path.is_absolute(): - preset_path = _preset_path(str(ext_path)) - if preset_path is not None: - ext_path = preset_path + # Resolve relative to the including file first (least surprise: a + # local file next to the config wins), then fall back to a bundled + # preset of that name. + local_path = (base_dir / ext_path).resolve() + if local_path.is_file(): + ext_path = local_path else: - ext_path = (base_dir / ext_path).resolve() + preset_path = _preset_path(str(ext_path)) + ext_path = preset_path if preset_path is not None else local_path if not ext_path.is_file(): raise FileNotFoundError(f"extends: file not found: {ext_path}") parent = _read_file(ext_path) @@ -399,8 +404,10 @@ def _adapt_ego_legacy(cfg: dict) -> dict: "grid": {"ding0_path": edisgo_cfg.get("grid_path")}, "results": {"directory": edisgo_cfg.get("results")}, "pipeline": mapped, - "overlying_grid": {"path": edisgo_cfg.get("overlying_grid_source")}, - "overlying_grid": {"selection": edisgo_cfg.get("overlying_grid")} + "overlying_grid": { + "path": edisgo_cfg.get("overlying_grid_source"), + "selection": edisgo_cfg.get("overlying_grid"), + }, } if "database" in cfg: adapted["database"] = cfg["database"] diff --git a/edisgo/run/context.py b/edisgo/run/context.py index c2fbce234..f07effadf 100644 --- a/edisgo/run/context.py +++ b/edisgo/run/context.py @@ -105,11 +105,43 @@ def ensure_engine(self): "Task needs a database engine but no 'database' section " "is configured." ) + ssh_cfg = db_cfg.get("ssh") or {} + ssh_enabled = bool(ssh_cfg.get("enabled", False)) + + # Direct local database: when SSH is disabled and explicit + # connection parameters are given (host/port/user/password as + # passed by eGo), connect straight to that postgres via + # psycopg2. This avoids edisgo.io.db.engine(ssh=False), which + # is hard-wired to the remote OpenEnergyPlatform (oedialect) + # and can stall for hours on large queries. + host = db_cfg.get("host") + if not ssh_enabled and host: + from sqlalchemy import create_engine + + user = db_cfg.get("user") + password = db_cfg.get("password") + port = db_cfg.get("port") + name = db_cfg.get("database_name") or db_cfg.get("database") + self.logger.info( + f"ensure_engine: using local database " + f"{user}@{host}:{port}/{name} (no OEP, no SSH tunnel)." + ) + self.engine = create_engine( + f"postgresql+psycopg2://{user}:{password}@{host}:{port}/{name}", + connect_args={"connect_timeout": 10}, + # The engine is cached and reused across long-running tasks + # (e.g. electromobility can idle the connection for many + # minutes). pool_pre_ping detects connections the server/SSH + # tunnel dropped while idle and transparently reconnects, + # avoiding "server closed the connection unexpectedly". + pool_pre_ping=True, + ) + return self.engine + from edisgo.io.db import engine as egon_engine - ssh_cfg = db_cfg.get("ssh") or {} self.engine = egon_engine( path=db_cfg.get("credentials_path"), - ssh=bool(ssh_cfg.get("enabled", False)), + ssh=ssh_enabled, ) return self.engine diff --git a/edisgo/run/tasks/io.py b/edisgo/run/tasks/io.py index 470526a6b..77ad9f4c2 100644 --- a/edisgo/run/tasks/io.py +++ b/edisgo/run/tasks/io.py @@ -233,14 +233,53 @@ def task_import_overlying_grid_data(edisgo, ctx, *, overlying_grid_path=None): """ import pandas as pd - + og_cfg = ctx.raw_config.get("overlying_grid") or {} if not og_cfg.get("enabled"): return edisgo source = og_cfg.get("source") overlying_grid_data = getattr(ctx, "overlying_grid_data", None) + edisgo_ti = edisgo.timeseries.timeindex + soc_attrs = { + "storage_units_soc", + "thermal_storage_units_decentral_soc", + "thermal_storage_units_central_soc", + } + + def _to_edisgo_timeindex(ts, extra_step=False): + """ + Shift ``ts``'s index year to match the edisgo timeindex and reindex + onto it. ``extra_step`` appends one trailing step (for SOC series, + which carry an end-of-period state). Returns ``ts`` unchanged for + empty inputs or an empty edisgo timeindex. + """ + if ts is None or ts.empty or edisgo_ti.empty: + return ts + year_diff = edisgo_ti[0].year - ts.index[0].year + if year_diff != 0: + ts = ts.copy() + ts.index = ts.index + pd.DateOffset(years=year_diff) + target = edisgo_ti + if extra_step: + # Derive the step only when it can be inferred; a single-timestamp + # timeindex with no freq cannot, so fall back to no extra step. + freq = edisgo_ti.freq or ( + edisgo_ti[1] - edisgo_ti[0] if len(edisgo_ti) > 1 else None + ) + if freq is not None: + target = edisgo_ti.union([edisgo_ti[-1] + freq]) + return ts.reindex(target) + + if source not in ("etrago", "csv"): + ctx.logger.warning( + f"task 'import_overlying_grid_data': unknown source={source!r} " + "(expected 'etrago' or 'csv') — skipping." + ) + return edisgo + + # --- 1) load the overlying-grid attributes for the chosen source --- if source == "etrago": if overlying_grid_data is None: ctx.logger.warning( @@ -251,111 +290,54 @@ def task_import_overlying_grid_data(edisgo, ctx, *, overlying_grid_path=None): for attr in edisgo.overlying_grid._attributes: if attr in overlying_grid_data: setattr(edisgo.overlying_grid, attr, overlying_grid_data[attr]) - - if not overlying_grid_data.get( - "dispatchable_generators_active_power" - ).empty: - # set generator time series - edisgo.set_time_series_active_power_predefined( - dispatchable_generators_ts=overlying_grid_data.get( - "dispatchable_generators_active_power" - ), - ) - if not overlying_grid_data.get("renewables_potential").empty: - pot_ts = overlying_grid_data.get("renewables_potential") - edisgo_ti = edisgo.timeseries.timeindex - csv_year = pot_ts.index[0].year - edisgo_year = edisgo_ti[0].year - if csv_year != edisgo_year: - pot_ts.index = pot_ts.index + pd.DateOffset( - years=edisgo_year - csv_year - ) - pot_ts = pot_ts.reindex(edisgo_ti) - - edisgo.set_time_series_active_power_predefined( - fluctuating_generators_ts=pot_ts, - ) - - if source == "csv": - if overlying_grid_path is None: - overlying_grid_path = og_cfg.get("path") + else: # source == "csv" + overlying_grid_path = overlying_grid_path or og_cfg.get("path") if overlying_grid_path is None: ctx.logger.warning( "task 'import_overlying_grid_data': source='csv' but no " "overlying_grid.path configured — skipping." ) return edisgo - - # load overlying-grid attributes from CSV directory - edisgo.overlying_grid.from_csv(overlying_grid_path) - - if overlying_grid_data is None: - # load overlying-grid attributes from CSV directory edisgo.overlying_grid.from_csv(overlying_grid_path) - # reindex overlying-grid attributes to match edisgo timeindex - # CSVs may use a different year — shift year then reindex - edisgo_ti = edisgo.timeseries.timeindex - if not edisgo_ti.empty: - # SOC needs one extra step at the end (end-of-period state) - ti_freq = edisgo_ti.freq or (edisgo_ti[1] - edisgo_ti[0]) - edisgo_ti_plus1 = edisgo_ti.union([edisgo_ti[-1] + ti_freq]) - soc_attrs = { - "storage_units_soc", - "thermal_storage_units_decentral_soc", - "thermal_storage_units_central_soc", - } - for attr in edisgo.overlying_grid._attributes: - ts = getattr(edisgo.overlying_grid, attr) - if ts.empty: - continue - csv_year = ts.index[0].year - edisgo_year = edisgo_ti[0].year - if csv_year != edisgo_year: - ts.index = ts.index + pd.DateOffset(years=edisgo_year - csv_year) - target_ti = edisgo_ti_plus1 if attr in soc_attrs else edisgo_ti - setattr(edisgo.overlying_grid, attr, ts.reindex(target_ti)) - - if overlying_grid_data is None: - # load dispatchable generator and renewables time series from the same dir - disp_path = os.path.join( - overlying_grid_path, "dispatchable_generators_active_power.csv" + # --- 2) reindex the overlying-grid attributes onto the edisgo timeindex + # (data may use a different year; SOC series carry one extra end step) --- + for attr in edisgo.overlying_grid._attributes: + ts = getattr(edisgo.overlying_grid, attr) + if ts is None or ts.empty: + continue + setattr( + edisgo.overlying_grid, + attr, + _to_edisgo_timeindex(ts, extra_step=attr in soc_attrs), ) - if os.path.isfile(disp_path): - disp_ts = pd.read_csv(disp_path, index_col=0, parse_dates=True) - if not edisgo_ti.empty: - csv_year = disp_ts.index[0].year - edisgo_year = edisgo_ti[0].year - if csv_year != edisgo_year: - disp_ts.index = disp_ts.index + pd.DateOffset( - years=edisgo_year - csv_year - ) - disp_ts = disp_ts.reindex(edisgo_ti) - else: - disp_ts = None - - pot_path = os.path.join(overlying_grid_path, "renewables_potential.csv") - if os.path.isfile(pot_path): - pot_ts = pd.read_csv(pot_path, index_col=0, parse_dates=True) - if not edisgo_ti.empty: - csv_year = pot_ts.index[0].year - edisgo_year = edisgo_ti[0].year - if csv_year != edisgo_year: - pot_ts.index = pot_ts.index + pd.DateOffset( - years=edisgo_year - csv_year - ) - pot_ts = pot_ts.reindex(edisgo_ti) - else: - pot_ts = None + # --- 3) set dispatchable/fluctuating generator time series --- + if source == "etrago": + disp_ts = overlying_grid_data.get("dispatchable_generators_active_power") + pot_ts = overlying_grid_data.get("renewables_potential") + if disp_ts is not None and not disp_ts.empty: + edisgo.set_time_series_active_power_predefined( + dispatchable_generators_ts=disp_ts, + ) + if pot_ts is not None and not pot_ts.empty: + edisgo.set_time_series_active_power_predefined( + fluctuating_generators_ts=_to_edisgo_timeindex(pot_ts), + ) + else: # source == "csv": load the two generator-TS CSVs from the dir + def _load_generator_ts(filename): + path = os.path.join(overlying_grid_path, filename) + if not os.path.isfile(path): + return None + ts = pd.read_csv(path, index_col=0, parse_dates=True) + return _to_edisgo_timeindex(ts) + + disp_ts = _load_generator_ts("dispatchable_generators_active_power.csv") + pot_ts = _load_generator_ts("renewables_potential.csv") if disp_ts is not None or pot_ts is not None: edisgo.set_time_series_active_power_predefined( dispatchable_generators_ts=disp_ts, fluctuating_generators_ts=pot_ts, ) - ctx.logger.warning( - f"task 'import_overlying_grid_data': unknown source={source!r} " - "(expected 'etrago' or 'csv') — skipping." - ) return edisgo diff --git a/edisgo/run/tasks/timeseries.py b/edisgo/run/tasks/timeseries.py index cf918c235..1264a9b95 100644 --- a/edisgo/run/tasks/timeseries.py +++ b/edisgo/run/tasks/timeseries.py @@ -266,12 +266,12 @@ def _as_df(obj): return pd.DataFrame(obj) if obj is not None else None edisgo.set_time_series_manual( - generators_active_power=_as_df(generators_active_power), - generators_reactive_power=_as_df(generators_reactive_power), - loads_active_power=_as_df(loads_active_power), - loads_reactive_power=_as_df(loads_reactive_power), - storage_units_active_power=_as_df(storage_units_active_power), - storage_units_reactive_power=_as_df(storage_units_reactive_power), + generators_p=_as_df(generators_active_power), + generators_q=_as_df(generators_reactive_power), + loads_p=_as_df(loads_active_power), + loads_q=_as_df(loads_reactive_power), + storage_units_p=_as_df(storage_units_active_power), + storage_units_q=_as_df(storage_units_reactive_power), ) ctx.flags["timeseries_set"] = True return edisgo diff --git a/edisgo/run/validator.py b/edisgo/run/validator.py index 3989b8398..f8f998828 100644 --- a/edisgo/run/validator.py +++ b/edisgo/run/validator.py @@ -117,21 +117,23 @@ def validate(cfg: dict) -> None: f"a loaded grid (setup_grid or " f"load_from_base) before it." ) - if task_name in {"analyze", "reinforce"} and not ( - ts_set or load_from - ): + # load_from does NOT satisfy these: _load_artifact reloads the + # grid with import_timeseries=False and drops flex data, so a + # time-series (and, for optimize, a flex-import) task must run + # in the stage itself even after a load_from. + if task_name in {"analyze", "reinforce"} and not ts_set: raise ValueError( f"Stage '{name}': task '{task_name}' requires time " f"series to be set (e.g. worst_case_ts or " f"oedb_ts) before it." ) if task_name == "optimize": - if not ts_set and not load_from: + if not ts_set: raise ValueError( f"Stage '{name}': 'optimize' requires time " f"series." ) - if not flex_imported and not load_from: + if not flex_imported: raise ValueError( f"Stage '{name}': 'optimize' requires at least " f"one flex asset to be imported." diff --git a/tests/run/test_validator.py b/tests/run/test_validator.py index 4b86f40cf..1d375f10d 100644 --- a/tests/run/test_validator.py +++ b/tests/run/test_validator.py @@ -92,6 +92,24 @@ def test_stage_load_from_with_save_ok(): cfg = {"stages": [ {"name": "a", "pipeline": ["setup_grid", "worst_case_ts", "reinforce", "save"]}, - {"name": "b", "load_from": "a", "pipeline": ["reinforce", "save"]}, + # load_from reloads the grid with import_timeseries=False, so the + # consuming stage must set time series itself before reinforce. + {"name": "b", "load_from": "a", + "pipeline": ["worst_case_ts", "reinforce", "save"]}, ]} validate(cfg) + + +def test_stage_load_from_without_ts_rejected(): + """ + load_from does NOT satisfy the time-series prerequisite: the artifact is + reloaded with import_timeseries=False, so reinforce after a bare load_from + (no time-series task in the stage) must be rejected. + """ + cfg = {"stages": [ + {"name": "a", "pipeline": ["setup_grid", "worst_case_ts", + "reinforce", "save"]}, + {"name": "b", "load_from": "a", "pipeline": ["reinforce", "save"]}, + ]} + with pytest.raises(ValueError, match="requires time series"): + validate(cfg) From 031e7431f88d4286ed61f74118c23c3fa217cfd2 Mon Sep 17 00:00:00 2001 From: Jonas Danke Date: Wed, 1 Jul 2026 13:08:18 +0200 Subject: [PATCH 33/37] refactor: declarative task metadata for the validator + shared artifact loader - registry: register_task now records requires/provides/ts_altering metadata (TaskMeta) exposed via get_task_meta. Tasks declare their pre-/post- conditions (setup_grid provides 'grid', TS tasks provide 'timeseries', flex imports require 'grid' and provide 'flex', analyze/reinforce require 'timeseries', optimize requires 'timeseries'+'flex', base_reinforce requires 'grid'). - validator: check those metadata generically instead of maintaining parallel hard-coded task-name sets, so it stays in sync with the tasks. load_from provides only 'grid' (not timeseries/flex), matching _load_artifact. Compute the known-task set once instead of per step. - runner._load_artifact and tasks.grid.task_load_from_base now share one load_saved_edisgo() helper instead of duplicating the import_edisgo_from_files policy. --- edisgo/run/registry.py | 71 ++++++++++++++++++++++- edisgo/run/runner.py | 24 ++------ edisgo/run/tasks/analysis.py | 8 +-- edisgo/run/tasks/flex.py | 8 +-- edisgo/run/tasks/grid.py | 85 +++++++++++++++++++++------ edisgo/run/tasks/timeseries.py | 8 +-- edisgo/run/validator.py | 103 +++++++++++++++------------------ 7 files changed, 199 insertions(+), 108 deletions(-) diff --git a/edisgo/run/registry.py b/edisgo/run/registry.py index 8aed4f3f5..c7ffca338 100644 --- a/edisgo/run/registry.py +++ b/edisgo/run/registry.py @@ -18,12 +18,44 @@ """ from __future__ import annotations -from typing import Callable +from typing import Callable, NamedTuple _TASKS: dict[str, Callable] = {} -def register_task(name: str) -> Callable[[Callable], Callable]: +class TaskMeta(NamedTuple): + """ + Declarative metadata describing a task's pipeline pre-/post-conditions. + + Attributes + ---------- + requires : frozenset of str + Capabilities that must already be satisfied in the stage before + this task runs (e.g. ``{"grid"}``, ``{"timeseries"}``, ``{"flex"}``). + provides : frozenset of str + Capabilities this task establishes for later tasks in the stage. + ts_altering : bool + Whether the task sets/alters the active-power time series. Such + tasks must not appear after ``reactive_power``. The validator uses + this metadata so it stays in sync with the actual tasks instead of + maintaining a parallel hard-coded list. + """ + + requires: frozenset = frozenset() + provides: frozenset = frozenset() + ts_altering: bool = False + + +_META: dict[str, TaskMeta] = {} + + +def register_task( + name: str, + *, + requires=frozenset(), + provides=frozenset(), + ts_altering: bool = False, +) -> Callable[[Callable], Callable]: """ Decorator to register a task function under the given name. @@ -37,6 +69,14 @@ def register_task(name: str) -> Callable[[Callable], Callable]: ---------- name : str Unique task name used in pipeline definitions. + requires : iterable of str, optional + Capabilities the task needs (see :class:`TaskMeta`). Used by the + validator for static ordering checks. + provides : iterable of str, optional + Capabilities the task establishes for later tasks. + ts_altering : bool, optional + Whether the task alters the active-power time series (must precede + ``reactive_power``). Returns ------- @@ -50,7 +90,8 @@ def register_task(name: str) -> Callable[[Callable], Callable]: Examples -------- - >>> @register_task("set_timeindex_weekly") + >>> @register_task("set_timeindex_weekly", provides={"timeseries"}, + ... ts_altering=True) ... def task_weekly(edisgo, ctx, *, start): ... import pandas as pd ... edisgo.set_timeindex(pd.date_range(start, periods=168, freq="h")) @@ -64,11 +105,35 @@ def deco(fn: Callable) -> Callable: f"new={fn.__qualname__})." ) _TASKS[name] = fn + _META[name] = TaskMeta( + requires=frozenset(requires), + provides=frozenset(provides), + ts_altering=ts_altering, + ) return fn return deco +def get_task_meta(name: str) -> TaskMeta: + """ + Return the :class:`TaskMeta` for a registered task. + + Parameters + ---------- + name : str + Task name. + + Returns + ------- + TaskMeta + The task's declared metadata. Unregistered names yield an empty + :class:`TaskMeta` (no requirements, no provided capabilities). + + """ + return _META.get(name, TaskMeta()) + + def get_task(name: str) -> Callable: """ Look up a registered task function by name. diff --git a/edisgo/run/runner.py b/edisgo/run/runner.py index 2d08fd3bc..d3464010e 100644 --- a/edisgo/run/runner.py +++ b/edisgo/run/runner.py @@ -176,25 +176,11 @@ def _load_artifact(path: str): The restored EDisGo instance. """ - import pandas as pd - - from edisgo.edisgo import import_edisgo_from_files - - from_zip = path.endswith(".zip") - edisgo = import_edisgo_from_files( - edisgo_path=path, - import_topology=True, - import_timeseries=False, - import_results=True, - import_electromobility=False, - import_heat_pump=False, - import_dsm=False, - import_overlying_grid=False, - from_zip_archive=from_zip, - ) - edisgo.legacy_grids = False - edisgo.results.equipment_changes = pd.DataFrame() - return edisgo + from edisgo.run.tasks.grid import load_saved_edisgo + + # Topology + results only; time series and flex data are dropped so the + # consuming stage sets them fresh, and equipment_changes is reset. + return load_saved_edisgo(path, import_results=True) def _resolve_templating(step_params: dict, stage_params: dict) -> dict: diff --git a/edisgo/run/tasks/analysis.py b/edisgo/run/tasks/analysis.py index becff93b9..a0d0f4ebd 100644 --- a/edisgo/run/tasks/analysis.py +++ b/edisgo/run/tasks/analysis.py @@ -55,7 +55,7 @@ def task_check_integrity(edisgo, ctx): return edisgo -@register_task("analyze") +@register_task("analyze", requires={"timeseries"}) def task_analyze( edisgo, ctx, @@ -110,7 +110,7 @@ def task_analyze( return edisgo -@register_task("reinforce") +@register_task("reinforce", requires={"timeseries"}) def task_reinforce( edisgo, ctx, @@ -182,7 +182,7 @@ def task_reinforce( return edisgo -@register_task("base_reinforce") +@register_task("base_reinforce", requires={"grid"}) def task_base_reinforce( edisgo, ctx, *, cases=None, reset_equipment_changes=True, save_artifact=True ): @@ -258,7 +258,7 @@ def task_base_reinforce( return edisgo -@register_task("optimize") +@register_task("optimize", requires={"timeseries", "flex"}) def task_optimize( edisgo, ctx, diff --git a/edisgo/run/tasks/flex.py b/edisgo/run/tasks/flex.py index fd914a2a0..87ba4a35e 100644 --- a/edisgo/run/tasks/flex.py +++ b/edisgo/run/tasks/flex.py @@ -13,7 +13,7 @@ from edisgo.run.registry import register_task -@register_task("import_heat_pumps") +@register_task("import_heat_pumps", requires={"grid"}, provides={"flex"}) def task_import_heat_pumps(edisgo, ctx, *, import_types=None, timeindex=None): """ Import heat pumps from egon_data into the topology. @@ -52,7 +52,7 @@ def task_import_heat_pumps(edisgo, ctx, *, import_types=None, timeindex=None): return edisgo -@register_task("import_home_batteries") +@register_task("import_home_batteries", requires={"grid"}, provides={"flex"}) def task_import_home_batteries(edisgo, ctx): """ Import home batteries from egon_data into the topology. @@ -81,7 +81,7 @@ def task_import_home_batteries(edisgo, ctx): return edisgo -@register_task("import_dsm") +@register_task("import_dsm", requires={"grid"}, provides={"flex"}) def task_import_dsm(edisgo, ctx, *, timeindex=None): """ Import demand-side-management potential from egon_data. @@ -113,7 +113,7 @@ def task_import_dsm(edisgo, ctx, *, timeindex=None): return edisgo -@register_task("import_electromobility") +@register_task("import_electromobility", requires={"grid"}, provides={"flex"}) def task_import_electromobility(edisgo, ctx, *, data_source="oedb", charging_strategy="dumb", flexibility_bands_ucs = None, diff --git a/edisgo/run/tasks/grid.py b/edisgo/run/tasks/grid.py index b074b7e68..348438bf5 100644 --- a/edisgo/run/tasks/grid.py +++ b/edisgo/run/tasks/grid.py @@ -18,7 +18,7 @@ from edisgo.run.registry import register_task -@register_task("setup_grid") +@register_task("setup_grid", provides={"grid"}) def task_setup_grid( edisgo, ctx, @@ -109,7 +109,70 @@ def task_setup_grid( return edisgo -@register_task("load_from_base") +def load_saved_edisgo( + path, + *, + reset_equipment_changes=True, + import_timeseries=False, + import_results=False, + import_electromobility=False, + import_heat_pump=False, + import_dsm=False, + import_overlying_grid=False, +): + """ + Reload a previously saved EDisGo object from a directory or ``.zip``. + + Shared by the ``load_from_base`` task and the runner's stage-level + ``load_from`` handling so both load artifacts with the same policy. + Topology is always imported; time series and flex data default to off + (the consuming stage sets them fresh). ``legacy_grids`` is cleared and, + by default, ``results.equipment_changes`` is reset so a subsequent + reinforce reflects only the current scenario. + + Parameters + ---------- + path : str or pathlib.Path + Directory or ``.zip`` produced by the ``save`` task. + reset_equipment_changes : bool, optional + If ``True`` (default), clear ``results.equipment_changes``. + import_timeseries, import_results, import_electromobility, \ + import_heat_pump, import_dsm, import_overlying_grid : bool, optional + Which saved sub-datasets to import (all off by default except as + overridden by the caller). + + Returns + ------- + edisgo.EDisGo + The restored EDisGo instance. + + """ + import os + + import pandas as pd + + from edisgo.edisgo import import_edisgo_from_files + + path = str(path) + from_zip = path.endswith(".zip") or not os.path.isdir(path) + edisgo = import_edisgo_from_files( + edisgo_path=path, + import_topology=True, + import_timeseries=import_timeseries, + import_results=import_results, + import_electromobility=import_electromobility, + import_heat_pump=import_heat_pump, + import_dsm=import_dsm, + import_overlying_grid=import_overlying_grid, + from_zip_archive=from_zip, + ) + edisgo.legacy_grids = False + if reset_equipment_changes: + edisgo.results.equipment_changes = pd.DataFrame() + return edisgo + + +@register_task("load_from_base", provides={"grid"}) def task_load_from_base( edisgo, ctx, @@ -166,12 +229,6 @@ def task_load_from_base( The restored EDisGo instance. """ - import os - - import pandas as pd - - from edisgo.edisgo import import_edisgo_from_files - if path is None: grid_cfg = ctx.raw_config.get("grid", {}) or {} path = grid_cfg.get("ding0_path") @@ -180,21 +237,15 @@ def task_load_from_base( "Task 'load_from_base' requires 'path' either as task " "parameter or under config.grid.ding0_path." ) - path = str(path) - from_zip = path.endswith(".zip") or not os.path.isdir(path) - edisgo = import_edisgo_from_files( - edisgo_path=path, - import_topology=True, + edisgo = load_saved_edisgo( + path, + reset_equipment_changes=reset_equipment_changes, import_timeseries=import_timeseries, import_results=import_results, import_electromobility=import_electromobility, import_heat_pump=import_heat_pump, import_dsm=import_dsm, import_overlying_grid=import_overlying_grid, - from_zip_archive=from_zip, ) - edisgo.legacy_grids = False - if reset_equipment_changes: - edisgo.results.equipment_changes = pd.DataFrame() ctx.flags["grid_loaded"] = True return edisgo diff --git a/edisgo/run/tasks/timeseries.py b/edisgo/run/tasks/timeseries.py index 1264a9b95..d4a6c8a30 100644 --- a/edisgo/run/tasks/timeseries.py +++ b/edisgo/run/tasks/timeseries.py @@ -20,7 +20,7 @@ from edisgo.run.registry import register_task -@register_task("worst_case_ts") +@register_task("worst_case_ts", provides={"timeseries"}, ts_altering=True) def task_worst_case_ts( edisgo, ctx, @@ -68,7 +68,7 @@ def task_worst_case_ts( return edisgo -@register_task("set_timeindex") +@register_task("set_timeindex", provides={"timeseries"}, ts_altering=True) def task_set_timeindex(edisgo, ctx, *, start, periods=None, end=None, freq="h"): """ Set the time index on the EDisGo object. @@ -119,7 +119,7 @@ def task_set_timeindex(edisgo, ctx, *, start, periods=None, end=None, freq="h"): return edisgo -@register_task("oedb_ts") +@register_task("oedb_ts", provides={"timeseries"}, ts_altering=True) def task_oedb_ts( edisgo, ctx, @@ -216,7 +216,7 @@ def task_oedb_ts( return edisgo -@register_task("manual_ts") +@register_task("manual_ts", provides={"timeseries"}, ts_altering=True) def task_manual_ts( edisgo, ctx, diff --git a/edisgo/run/validator.py b/edisgo/run/validator.py index f8f998828..5ca548499 100644 --- a/edisgo/run/validator.py +++ b/edisgo/run/validator.py @@ -27,16 +27,21 @@ from typing import Any -from edisgo.run.registry import known_tasks - -_TS_TASKS = {"worst_case_ts", "oedb_ts", "manual_ts", "set_timeindex"} -_GRID_CREATING_TASKS = {"setup_grid", "load_from_base"} -_FLEX_IMPORTS = { - "import_heat_pumps", - "import_home_batteries", - "import_dsm", - "import_electromobility", +from edisgo.run.registry import get_task_meta, known_tasks + +# Human-readable message per required capability. The wording keeps the +# substrings the validator tests assert on ("loaded grid", "time series", +# "flex asset"). +_REQUIREMENT_MESSAGES = { + "grid": "requires a loaded grid (setup_grid or load_from_base) before it", + "timeseries": ( + "requires time series to be set (e.g. worst_case_ts or oedb_ts) " + "before it" + ), + "flex": "requires at least one flex asset to be imported", } +# Order in which a missing capability is reported when several are missing. +_REQUIREMENT_PRIORITY = ("grid", "timeseries", "flex") def validate(cfg: dict) -> None: @@ -68,6 +73,7 @@ def validate(cfg: dict) -> None: if not stages: raise ValueError("Config has no stages to run.") + known = set(known_tasks()) available_artifacts: set[str] = set() for stage in stages: @@ -82,67 +88,50 @@ def validate(cfg: dict) -> None: f"{sorted(available_artifacts)}" ) - grid_available = load_from is not None - ts_set = False + # Capabilities established so far in this stage. A stage-level + # load_from reloads the grid topology only — _load_artifact drops + # time series and flex data (import_timeseries=False) — so it + # provides "grid" but NOT "timeseries"/"flex". A task's requirements + # must therefore be satisfied by tasks run in this stage itself. + satisfied: set[str] = {"grid"} if load_from is not None else set() reactive_set = False - flex_imported = False has_save = False for step in pipeline: task_name, _params = _split_step(step) - if task_name not in known_tasks(): + if task_name not in known: raise ValueError( f"Unknown task '{task_name}' in stage '{name}'. " - f"Known: {known_tasks()}" + f"Known: {sorted(known)}" ) - if task_name in _GRID_CREATING_TASKS: - grid_available = True - if task_name in _TS_TASKS: - if reactive_set: - raise ValueError( - f"Stage '{name}': time-series task " - f"'{task_name}' comes after 'reactive_power' " - f"— reactive_power must be the last " - f"time-series-altering step." - ) - ts_set = True - if task_name == "reactive_power": - reactive_set = True - if task_name in _FLEX_IMPORTS: - flex_imported = True - if not grid_available: - raise ValueError( - f"Stage '{name}': task '{task_name}' requires " - f"a loaded grid (setup_grid or " - f"load_from_base) before it." - ) - # load_from does NOT satisfy these: _load_artifact reloads the - # grid with import_timeseries=False and drops flex data, so a - # time-series (and, for optimize, a flex-import) task must run - # in the stage itself even after a load_from. - if task_name in {"analyze", "reinforce"} and not ts_set: + meta = get_task_meta(task_name) + + # reactive_power must be the last time-series-altering step. + if meta.ts_altering and reactive_set: raise ValueError( - f"Stage '{name}': task '{task_name}' requires time " - f"series to be set (e.g. worst_case_ts or " - f"oedb_ts) before it." + f"Stage '{name}': time-series task '{task_name}' comes " + f"after 'reactive_power' — reactive_power must be the " + f"last time-series-altering step." + ) + + # Check declared requirements against what the stage provides. + missing = meta.requires - satisfied + if missing: + cap = next( + (c for c in _REQUIREMENT_PRIORITY if c in missing), + sorted(missing)[0], + ) + detail = _REQUIREMENT_MESSAGES.get( + cap, f"requires '{cap}' to be established before it" ) - if task_name == "optimize": - if not ts_set: - raise ValueError( - f"Stage '{name}': 'optimize' requires time " - f"series." - ) - if not flex_imported: - raise ValueError( - f"Stage '{name}': 'optimize' requires at least " - f"one flex asset to be imported." - ) - if task_name == "base_reinforce" and not grid_available: raise ValueError( - f"Stage '{name}': 'base_reinforce' requires a " - f"loaded grid before it." + f"Stage '{name}': task '{task_name}' {detail}." ) + + satisfied |= meta.provides + if task_name == "reactive_power": + reactive_set = True if task_name == "save": has_save = True From 9b696083ea30650f957cea2261333ea967628a34 Mon Sep 17 00:00:00 2001 From: Jonas Danke Date: Wed, 1 Jul 2026 13:08:33 +0200 Subject: [PATCH 34/37] refactor: shared time-series year-alignment helper + small cleanups - tools.align_series_to_timeindex: single helper that shifts a series' year (via DateOffset, leap-safe) and reindexes onto the edisgo time index, with an optional end-of-period step for SOC series. Used by both tasks.io.import_overlying_grid_data and powermodels_io (which now reindexes the SOC series instead of .loc, so a missing step yields NaN not KeyError). - context: declare overlying_grid_data as a RunContext field instead of a dynamically-set attribute; the task reads it directly. - EDisGo.run_pipeline: accept and forward overlying_grid_data (API symmetry with run_edisgo). - config._adapt_ego_legacy: deep-copy cfg['database'] before injecting ssh so the caller's config is not mutated. --- edisgo/edisgo.py | 10 ++++++-- edisgo/io/powermodels_io.py | 28 ++++++++------------- edisgo/run/config.py | 6 +++-- edisgo/run/context.py | 6 +++++ edisgo/run/tasks/io.py | 28 ++++----------------- edisgo/tools/tools.py | 49 +++++++++++++++++++++++++++++++++++++ 6 files changed, 82 insertions(+), 45 deletions(-) diff --git a/edisgo/edisgo.py b/edisgo/edisgo.py index 1775d3e25..0f0d258a3 100755 --- a/edisgo/edisgo.py +++ b/edisgo/edisgo.py @@ -243,7 +243,7 @@ def config(self): def config(self, kwargs): self._config = Config(**kwargs) - def run_pipeline(self, config): + def run_pipeline(self, config, overlying_grid_data=None): """ Run a YAML/JSON task pipeline on this EDisGo instance. @@ -253,6 +253,10 @@ def run_pipeline(self, config): ---------- config : str, :class:`pathlib.Path`, or dict Pipeline config as path to a YAML/JSON file or as a dict. + overlying_grid_data : dict, optional + Overlying-grid data (e.g. eTraGo results) consumed by the + ``import_overlying_grid_data`` task when + ``overlying_grid.source == "etrago"``. Returns ------- @@ -262,7 +266,9 @@ def run_pipeline(self, config): """ from edisgo.run import _run_pipeline_on - return _run_pipeline_on(self, config) + return _run_pipeline_on( + self, config, overlying_grid_data=overlying_grid_data + ) def import_ding0_grid(self, path, legacy_ding0_grids=True): """ diff --git a/edisgo/io/powermodels_io.py b/edisgo/io/powermodels_io.py index 2f55104b3..86a264f71 100644 --- a/edisgo/io/powermodels_io.py +++ b/edisgo/io/powermodels_io.py @@ -1029,26 +1029,18 @@ def _build_battery_storage( """ branches = pd.concat([psa_net.lines, psa_net.transformers]) if not edisgo_obj.overlying_grid.storage_units_soc.empty: - # Select relevant timesteps - timesteps = edisgo_obj.timeseries.timeindex.union( - [ - edisgo_obj.timeseries.timeindex[-1] - + edisgo_obj.timeseries.timeindex.freq - ] + # Align the SOC series (which may use another year) onto the edisgo + # time index plus one end-of-period step. Uses reindex, so a missing + # step yields NaN instead of a KeyError. + from edisgo.tools.tools import align_series_to_timeindex + + soc_aligned = align_series_to_timeindex( + edisgo_obj.overlying_grid.storage_units_soc, + edisgo_obj.timeseries.timeindex, + extra_step=True, ) - - # If the overlying grid data uses another year in the timeindex then - # edisgo.timindex, unify them - og_year = edisgo_obj.overlying_grid.storage_units_soc.index[0].year - year_diff = og_year - edisgo_obj.timeseries.timeindex[0].year - if year_diff != 0: - # Shift by whole years instead of Timestamp.replace(year=...), - # which raises on Feb 29 when the target year is not a leap year. - timesteps = timesteps + pd.DateOffset(years=year_diff) - data = pd.concat( - [edisgo_obj.overlying_grid.storage_units_soc.loc[timesteps]] - * len(edisgo_obj.topology.storage_units_df), + [soc_aligned] * len(edisgo_obj.topology.storage_units_df), axis=1, ).values else: diff --git a/edisgo/run/config.py b/edisgo/run/config.py index 6f095133f..ed50438ff 100644 --- a/edisgo/run/config.py +++ b/edisgo/run/config.py @@ -410,9 +410,11 @@ def _adapt_ego_legacy(cfg: dict) -> dict: }, } if "database" in cfg: - adapted["database"] = cfg["database"] + # Deep-copy so injecting ssh below does not mutate the caller's + # cfg["database"] (which is merged again in _deep_merge afterwards). + adapted["database"] = copy.deepcopy(cfg["database"]) if "ssh" in cfg: - adapted["database"]["ssh"] = cfg["ssh"] + adapted["database"]["ssh"] = copy.deepcopy(cfg["ssh"]) for side_key in ("eGo", "eTraGo", "ssh", "_comment", "_workflow"): cfg.pop(side_key, None) cfg.pop("eDisGo", None) diff --git a/edisgo/run/context.py b/edisgo/run/context.py index f07effadf..04b308d48 100644 --- a/edisgo/run/context.py +++ b/edisgo/run/context.py @@ -62,6 +62,11 @@ class RunContext: The fully resolved pipeline config (after ``extends``, ``external_config``, and eGo-legacy adaptation). Tasks can read supplementary keys like ``database.*`` from here. + overlying_grid_data : dict or None + Overlying-grid data (e.g. eTraGo results) injected via the + ``overlying_grid_data=`` argument of :func:`edisgo.run.run_edisgo`. + Consumed by the ``import_overlying_grid_data`` task when + ``overlying_grid.source == "etrago"``. """ @@ -75,6 +80,7 @@ class RunContext: stage_artifacts: dict[str, Path] = field(default_factory=dict) current_stage: str | None = None raw_config: dict[str, Any] = field(default_factory=dict) + overlying_grid_data: Any = None def ensure_engine(self): """ diff --git a/edisgo/run/tasks/io.py b/edisgo/run/tasks/io.py index 77ad9f4c2..6ffba2ee8 100644 --- a/edisgo/run/tasks/io.py +++ b/edisgo/run/tasks/io.py @@ -239,7 +239,7 @@ def task_import_overlying_grid_data(edisgo, ctx, *, overlying_grid_path=None): return edisgo source = og_cfg.get("source") - overlying_grid_data = getattr(ctx, "overlying_grid_data", None) + overlying_grid_data = ctx.overlying_grid_data edisgo_ti = edisgo.timeseries.timeindex soc_attrs = { @@ -248,29 +248,11 @@ def task_import_overlying_grid_data(edisgo, ctx, *, overlying_grid_path=None): "thermal_storage_units_central_soc", } + from edisgo.tools.tools import align_series_to_timeindex + def _to_edisgo_timeindex(ts, extra_step=False): - """ - Shift ``ts``'s index year to match the edisgo timeindex and reindex - onto it. ``extra_step`` appends one trailing step (for SOC series, - which carry an end-of-period state). Returns ``ts`` unchanged for - empty inputs or an empty edisgo timeindex. - """ - if ts is None or ts.empty or edisgo_ti.empty: - return ts - year_diff = edisgo_ti[0].year - ts.index[0].year - if year_diff != 0: - ts = ts.copy() - ts.index = ts.index + pd.DateOffset(years=year_diff) - target = edisgo_ti - if extra_step: - # Derive the step only when it can be inferred; a single-timestamp - # timeindex with no freq cannot, so fall back to no extra step. - freq = edisgo_ti.freq or ( - edisgo_ti[1] - edisgo_ti[0] if len(edisgo_ti) > 1 else None - ) - if freq is not None: - target = edisgo_ti.union([edisgo_ti[-1] + freq]) - return ts.reindex(target) + # bind the stage's edisgo time index to the shared aligner + return align_series_to_timeindex(ts, edisgo_ti, extra_step=extra_step) if source not in ("etrago", "csv"): ctx.logger.warning( diff --git a/edisgo/tools/tools.py b/edisgo/tools/tools.py index 29633f752..fd1860cba 100644 --- a/edisgo/tools/tools.py +++ b/edisgo/tools/tools.py @@ -38,6 +38,55 @@ logger = logging.getLogger(__name__) +def align_series_to_timeindex(ts, timeindex, extra_step=False): + """ + Align a time series to a target time index, tolerating a year mismatch. + + Data imported for the overlying grid (from CSV or eTraGo) may be indexed + in a different year than the EDisGo time index. This helper shifts the + series' index by whole years to match ``timeindex`` (using + :class:`pandas.DateOffset`, which — unlike ``Timestamp.replace(year=...)`` + — does not raise on a Feb-29 timestamp when the target year is not a leap + year) and reindexes onto it. Missing steps become ``NaN`` rather than + raising a ``KeyError``. + + Parameters + ---------- + ts : :pandas:`pandas.Series` or \ + :pandas:`pandas.DataFrame` or None + The time series to align. Returned unchanged if ``None``, empty, or + when ``timeindex`` is empty. + timeindex : :pandas:`pandas.DatetimeIndex` + Target time index to align to. + extra_step : bool, optional + If ``True``, append one trailing step to the target index (used for + state-of-charge series that carry an end-of-period value). The step + width is taken from ``timeindex.freq``, falling back to the spacing + of the first two entries; if neither is available (single-entry + index without freq) no extra step is added. + + Returns + ------- + Same type as ``ts`` + ``ts`` reindexed onto the (optionally extended) target index. + + """ + if ts is None or ts.empty or timeindex.empty: + return ts + year_diff = timeindex[0].year - ts.index[0].year + if year_diff != 0: + ts = ts.copy() + ts.index = ts.index + pd.DateOffset(years=year_diff) + target = timeindex + if extra_step: + freq = timeindex.freq or ( + timeindex[1] - timeindex[0] if len(timeindex) > 1 else None + ) + if freq is not None: + target = timeindex.union([timeindex[-1] + freq]) + return ts.reindex(target) + + def select_worstcase_snapshots(edisgo_obj): """ Select two worst-case snapshots from time series From 595754633f050493b6904b48e94bd4110290f583 Mon Sep 17 00:00:00 2001 From: Jonas Danke Date: Wed, 1 Jul 2026 13:08:33 +0200 Subject: [PATCH 35/37] test: add task-level tests for the run pipeline Cover the previously-untested task control flow (source of the review bugs): task_manual_ts applies its kwargs, import_overlying_grid_data handles the disabled/unknown/etrago-without-data/empty-etrago/csv-without-path branches without crashing, and every bundled preset passes the metadata-driven validator. --- tests/run/test_tasks.py | 104 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 104 insertions(+) create mode 100644 tests/run/test_tasks.py diff --git a/tests/run/test_tasks.py b/tests/run/test_tasks.py new file mode 100644 index 000000000..c6c12631e --- /dev/null +++ b/tests/run/test_tasks.py @@ -0,0 +1,104 @@ +""" +Unit tests for individual pipeline tasks in :mod:`edisgo.run.tasks`. + +These cover the task control flow that unit tests previously missed — the +task modules were the source of every bug found in the review. They run +without a database or SSH tunnel: a small self-constructed ding0 grid is +enough, and the DB-free branches of ``import_overlying_grid_data`` are +exercised directly. +""" +import glob +import os + +import pandas as pd +import pytest + +import edisgo.run as edisgo_run + +from edisgo.edisgo import EDisGo +from edisgo.run.config import load_config +from edisgo.run.context import RunContext +from edisgo.run.tasks.io import task_import_overlying_grid_data +from edisgo.run.tasks.timeseries import task_manual_ts +from edisgo.run.validator import validate + + +@pytest.fixture +def edisgo_obj(): + """Small ding0 grid with a 3-step time index, no DB access.""" + edisgo = EDisGo(ding0_grid=pytest.ding0_test_network_path) + edisgo.set_timeindex(pd.date_range("2011-01-01", periods=3, freq="h")) + return edisgo + + +class TestManualTs: + def test_manual_ts_applies_active_power(self, edisgo_obj): + """ + task_manual_ts must forward the eGo-style ``*_active_power`` args to + EDisGo.set_time_series_manual's real parameter names (regression: the + task used to pass unsupported kwargs and always raised TypeError). + """ + ti = edisgo_obj.timeseries.timeindex + gen = edisgo_obj.topology.generators_df.index[0] + df = pd.DataFrame({gen: [0.1, 0.2, 0.3]}, index=ti) + + ctx = RunContext() + result = task_manual_ts(edisgo_obj, ctx, generators_active_power=df) + + assert gen in result.timeseries.generators_active_power.columns + assert ctx.flags["timeseries_set"] is True + + +class TestImportOverlyingGridData: + def _ctx(self, og_cfg, overlying_grid_data=None): + return RunContext( + raw_config={"overlying_grid": og_cfg}, + overlying_grid_data=overlying_grid_data, + ) + + def test_disabled_returns_unchanged(self): + """enabled: false short-circuits before the grid is even touched.""" + sentinel = object() + ctx = self._ctx({"enabled": False}) + assert task_import_overlying_grid_data(sentinel, ctx) is sentinel + + def test_unknown_source_warns(self, edisgo_obj, caplog): + ctx = self._ctx({"enabled": True, "source": "bogus"}) + result = task_import_overlying_grid_data(edisgo_obj, ctx) + assert result is edisgo_obj + assert "unknown source" in caplog.text + + def test_etrago_without_data_warns(self, edisgo_obj, caplog): + ctx = self._ctx({"enabled": True, "source": "etrago"}, + overlying_grid_data=None) + result = task_import_overlying_grid_data(edisgo_obj, ctx) + assert result is edisgo_obj + assert "no" in caplog.text.lower() + + def test_etrago_empty_data_does_not_crash(self, edisgo_obj): + """ + A partial/empty etrago dict must not raise (regression: the task used + to call .empty on dict.get() results that were None). + """ + ctx = self._ctx({"enabled": True, "source": "etrago"}, + overlying_grid_data={}) + # must simply return without AttributeError + assert task_import_overlying_grid_data(edisgo_obj, ctx) is edisgo_obj + + def test_csv_without_path_warns(self, edisgo_obj, caplog): + ctx = self._ctx({"enabled": True, "source": "csv"}) + result = task_import_overlying_grid_data(edisgo_obj, ctx) + assert result is edisgo_obj + assert "path" in caplog.text.lower() + + +def test_all_bundled_presets_validate(): + """ + Every bundled preset must pass the (metadata-driven) validator — this + keeps the task requires/provides declarations in sync with real configs. + """ + presets_dir = os.path.join(os.path.dirname(edisgo_run.__file__), "presets") + presets = sorted(glob.glob(os.path.join(presets_dir, "*.yaml"))) + assert presets, "no bundled presets found" + for path in presets: + validate(load_config(path)) From 1565b26a85dc97e57ddc4901bd7cabd19c04df9b Mon Sep 17 00:00:00 2001 From: Jonas Danke Date: Thu, 2 Jul 2026 16:56:43 +0200 Subject: [PATCH 36/37] Add configurable database source (local egon-data / OEP) and fix SSH tunnel key Add engine_from_settings() mapping a scenario "database" section onto engine(): - source: "local" -> egon-data via SSH tunnel, using the optional config_path or the default ~/.ssh/egon-data.configuration.yaml (default_config_path(), overridable via EGON_DATA_CONFIG) - source: "oep" (or omitted) -> Open Energy Platform, as before engine(ssh=True) now falls back to the default config location when no path is given. RunContext.ensure_engine() is source-driven and delegates to engine_from_settings(); the legacy ssh.enabled flag and explicit direct-local database (host given) remain supported. Fix ssh_tunnel(): pass ssh_pkey as a string. Passing the pathlib.Path from credentials() made sshtunnel silently ignore the key and fall back to the default keys in ~/.ssh, failing gateway authentication. --- edisgo/io/db.py | 101 +++++++++++++++++++++++++++++++++++++++--- edisgo/run/context.py | 55 +++++++++++------------ 2 files changed, 121 insertions(+), 35 deletions(-) diff --git a/edisgo/io/db.py b/edisgo/io/db.py index fd12673af..daf1d0dce 100644 --- a/edisgo/io/db.py +++ b/edisgo/io/db.py @@ -36,6 +36,33 @@ logger = logging.getLogger(__name__) +#: Default location of the egon-data SSH tunnel configuration file. Used when +#: no explicit config path is passed and the connection mode is not forced. +#: Can be overridden through the ``EGON_DATA_CONFIG`` environment variable. +DEFAULT_EGON_DATA_CONFIG = "~/.ssh/egon-data.configuration.yaml" + + +def default_config_path() -> Path | None: + """ + Return the path to the egon-data SSH configuration file, or ``None``. + + The location is read from the ``EGON_DATA_CONFIG`` environment variable and + falls back to :data:`DEFAULT_EGON_DATA_CONFIG` + (``~/.ssh/egon-data.configuration.yaml``). ``None`` is returned when the + resolved path does not point to an existing file, which callers use as the + signal to fall back to the Open Energy Platform (OEP). + + Returns + ------- + pathlib.Path or None + Path to an existing egon-data configuration file, or ``None`` if none + was found. + + """ + raw = os.environ.get("EGON_DATA_CONFIG", DEFAULT_EGON_DATA_CONFIG) + path = Path(raw).expanduser() + return path if path.is_file() else None + def config_settings(path: Path | str) -> dict[str, dict[str, str | int | Path]]: """ @@ -155,7 +182,11 @@ def ssh_tunnel(cred: dict) -> str: server = SSHTunnelForwarder( ssh_address_or_host=(cred["SSH_HOST"], 22), ssh_username=cred["SSH_USER"], - ssh_pkey=cred["SSH_PKEY"], + # SSHTunnelForwarder only accepts a string path (or a loaded paramiko + # PKey) here. Passing the pathlib.Path produced by credentials() makes + # sshtunnel silently ignore the key and fall back to the default keys + # in ~/.ssh, which fails authentication against the gateway. + ssh_pkey=str(cred["SSH_PKEY"]), remote_bind_address=(cred["PGRES_HOST"], cred["PORT"]), ) server.start() @@ -172,12 +203,17 @@ def engine( Parameters ---------- path : str or pathlib.Path, optional (default=None) - Path to configuration YAML file of egon-data database. + Path to configuration YAML file of egon-data database. Only used when + ``ssh=True``. If None, the default location is used + (``EGON_DATA_CONFIG`` environment variable or + ``~/.ssh/egon-data.configuration.yaml``, see + :func:`default_config_path`). ssh : bool (default=False) - If False, connects to the remote Open Energy Platform database (using the - token, see parameter `token`). If True, establishes an ssh tunnel to a local - egon-data database using the connection information in the configuration YAML - given through `path`. + If False, connects to the remote Open Energy Platform database (using + the token, see parameter `token`). If True, establishes an ssh tunnel + to a local egon-data database using the connection information in the + configuration YAML given through `path` (or the default location if + `path` is None). token : str or pathlib.Path, optional (default=None) Token for database connection or path to text file containing token. If empty the default token file in the config folder OEP_TOKEN.txt @@ -239,6 +275,15 @@ def engine( echo=False, ) + if path is None: + path = default_config_path() + if path is None: + raise ValueError( + "SSH connection requested but no egon-data configuration file " + "was found (checked the EGON_DATA_CONFIG environment variable " + f"and the default location {DEFAULT_EGON_DATA_CONFIG})." + ) + cred = credentials(path=path) local_port = ssh_tunnel(cred) @@ -250,6 +295,50 @@ def engine( ) +def engine_from_settings(database: dict | None = None) -> Engine: + """ + Build a database engine from a scenario ``database`` settings section. + + This maps the data source configured in the scenario JSON onto + :func:`engine`. Recognised keys of `database`: + + * ``source`` — ``"local"`` connects to a local egon-data database through + an SSH tunnel; ``"oep"`` (or a missing/empty value) connects to the + remote Open Energy Platform (OEP), i.e. the previous default behaviour. + * ``config_path`` — optional path to the egon-data configuration YAML. + Only relevant for ``source="local"``. If omitted, the default location + is used (``EGON_DATA_CONFIG`` environment variable or + ``~/.ssh/egon-data.configuration.yaml``, see :func:`default_config_path`). + + Parameters + ---------- + database : dict or None + The ``database`` section of the scenario configuration. If None or + empty, an OEP engine is returned. + + Returns + ------- + :sqlalchemy:`sqlalchemy.Engine` + Database engine. + + """ + database = database or {} + source = str(database.get("source") or "oep").lower() + + if source in ("local", "ssh", "egon-data", "egon_data"): + # config_path may be given explicitly; otherwise engine() falls back to + # the default location (~/.ssh/egon-data.configuration.yaml). + config_path = database.get("config_path") or database.get("credentials_path") + logger.info( + f"engine_from_settings: source='local', using egon-data database " + f"via SSH tunnel (config {config_path or 'default (~/.ssh/...)'})." + ) + return engine(path=config_path, ssh=True) + + logger.info("engine_from_settings: source='oep', connecting to the OEP.") + return engine(ssh=False) + + @contextmanager def session_scope_egon_data(engine: Engine): """Provide a transactional scope around a series of operations.""" diff --git a/edisgo/run/context.py b/edisgo/run/context.py index 04b308d48..e41a07fc9 100644 --- a/edisgo/run/context.py +++ b/edisgo/run/context.py @@ -86,42 +86,37 @@ def ensure_engine(self): """ Return a database engine, creating it on first call. - Reads the ``database`` section of :attr:`raw_config` and calls - :func:`edisgo.io.db.engine`. Caches the engine on the context - so subsequent calls reuse the same connection. + The data source is chosen from the ``database`` section of + :attr:`raw_config`: + + * ``source: "local"`` — egon-data database via SSH tunnel, using + ``config_path`` if given, otherwise the default location + (``~/.ssh/egon-data.configuration.yaml``). + * ``source: "oep"`` or no ``database`` section — remote Open Energy + Platform (previous default behaviour). + + A legacy explicit direct-local database (``host`` given with SSH + disabled) is still honoured for backward compatibility. The engine is + cached on the context so subsequent calls reuse the same connection. Returns ------- sqlalchemy.engine.Engine The active database engine. - Raises - ------ - RuntimeError - If the config has no ``database`` section — indicates the - pipeline wants to reach the database without configuring - it. - """ if self.engine is not None: return self.engine - db_cfg = self.raw_config.get("database") - if not db_cfg: - raise RuntimeError( - "Task needs a database engine but no 'database' section " - "is configured." - ) + db_cfg = self.raw_config.get("database") or {} + source = str(db_cfg.get("source") or "").lower() + + # Legacy explicit direct local database: SSH disabled and explicit + # connection parameters given (host/port/user/password as passed by + # eGo). Connect straight to that postgres via psycopg2. ssh_cfg = db_cfg.get("ssh") or {} ssh_enabled = bool(ssh_cfg.get("enabled", False)) - - # Direct local database: when SSH is disabled and explicit - # connection parameters are given (host/port/user/password as - # passed by eGo), connect straight to that postgres via - # psycopg2. This avoids edisgo.io.db.engine(ssh=False), which - # is hard-wired to the remote OpenEnergyPlatform (oedialect) - # and can stall for hours on large queries. host = db_cfg.get("host") - if not ssh_enabled and host: + if source not in ("local", "oep") and host and not ssh_enabled: from sqlalchemy import create_engine user = db_cfg.get("user") @@ -144,10 +139,12 @@ def ensure_engine(self): ) return self.engine - from edisgo.io.db import engine as egon_engine + # Source-driven engine: source="local" -> egon-data via SSH tunnel + # (config_path or ~/.ssh default), source="oep"/absent -> OEP. + from edisgo.io.db import engine_from_settings - self.engine = egon_engine( - path=db_cfg.get("credentials_path"), - ssh=ssh_enabled, - ) + # A legacy ssh.enabled flag maps to source "local". + if not source and ssh_enabled: + db_cfg = {**db_cfg, "source": "local"} + self.engine = engine_from_settings(db_cfg) return self.engine From 289dbeefbd6abb290b7a3a96b54e91ccebf665f3 Mon Sep 17 00:00:00 2001 From: Jonas Danke Date: Thu, 2 Jul 2026 16:56:43 +0200 Subject: [PATCH 37/37] Pin paramiko < 4.0 for SSH tunnel support The SSH tunnel to the local egon-data database (via sshtunnel/paramiko) requires paramiko < 4.0; newer versions break key handling / the tunnel. --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 26d01c3f8..a2a4422b3 100644 --- a/setup.py +++ b/setup.py @@ -51,6 +51,7 @@ def read(fname): # sqlalchemy leads to new errors.. should be fixed at some point "numpy ==1.26.4", "pandas >= 1.4.0, < 2.2.0", + "paramiko < 4.0", "plotly < 6.0", "pydot < 4.1.0", "pypower < 5.2.0",