diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 708c4a1af2..41448fac8c 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -3,6 +3,7 @@ Change Log [upcoming release] - 2025-..-.. ------------------------------- +- [CHANGED] from_jao: converter now uses fuzzy matching with difflib module to find matching buses instead of as well as table column names if they are inconsistent between files. Added mapping to jao.rst - [FIXED] julia implementation, now using juliacall - [CHANGED] diagnostics restructured for better extensibility - [FIXED] implausible impedance test results never showing in report diff --git a/doc/converter/jao-converter.csv b/doc/converter/jao-converter.csv new file mode 100644 index 0000000000..31aa095ca5 --- /dev/null +++ b/doc/converter/jao-converter.csv @@ -0,0 +1,29 @@ +pandapower table,pandapower column,JAO sheet,JAO column (typical, variants),Transformation/notes +net.line,from_bus,"Lines, Tielines",Substation_1 / Full Name + Voltage_level(kV),Matched to an existing bus by name and VN, fuzzy name normalization applied. +net.line,to_bus,"Lines, Tielines",Substation_2 / Full Name + Voltage_level(kV),Matched to an existing bus by name and VN, fuzzy name normalization applied. +net.line,length_km,"Lines, Tielines",Electrical Parameters / Length_(km),"If 0 or NaN, set to 1 km (with log warning)." +net.line,r_ohm_per_km,"Lines, Tielines","Electrical Parameters / Resistance_R(Ω) or (""resistance r ohm"")",Total R divided by Length (km). +net.line,x_ohm_per_km,"Lines, Tielines","Electrical Parameters / Reactance_X(Ω) or (""reactance x ohm"")",Total X divided by Length (km). +net.line,c_nf_per_km,"Lines, Tielines","Susceptance_B(μS) (or ""Susceptance_B (µS)"")",B_total/Length (km) is passed into c_nf_per_km (units μS/km) +net.line,g_us_per_km,-,-,Not set (no JAO line conductance column used), defaults to 0 in pandapower. +net.line,max_i_ka,"Lines, Tielines",Maximum Current Imax (A) / Fixed,"Converted A to kA, if missing, filled with max_i_ka_fillna." +net.line,parallel,-,-,Not provided by JAO - defaults to 1. +net.line,df,-,-,Not provided by JAO - defaults to 1. +net.line,in_service,-,-,Not provided by JAO - defaults to True. +net.bus,vn_kv,"Lines, Tielines","Voltage_level(kV) (Voltage_level (kV), Voltage_level [kV])",Numeric conversion +net.bus,in_service,-,-,"Not provided by JAO, defaults to True." +net.transformer,hv_bus,Transformers + Buses,Location / Full Name + Primary/Secondary Voltage,"High-voltage side bus index. Location matched to existing bus at required VN, if no exact VN, nearest VN at location is used or a new bus is created (per converter rules)." +net.transformer,lv_bus,Transformers + Buses,Location / Full Name + Primary/Secondary Voltage,"Low-voltage side bus index, analogous to hv_bus. Ensured hv_bus != lv_bus (LV duplicated if necessary)." +net.transformer,sn_mva,Transformers,Maximum Current Imax (A) primary (Fixed/Max) + VN,Rated apparent power [MVA]. Computed: sn_mva = √3 · Imax_primary(A) · vn_hv_kv(kV) / 1e3. Falls back to “Max” or global fill if Fixed is missing. +net.transformer,vn_hv_kv,Transformers,Voltage level Primary/Secondary,"Rated HV voltage [kV], vn_hv_kv = max(Primary, Secondary)." +net.transformer,vn_lv_kv,Transformers,Voltage level Primary/Secondary,"Rated LV voltage [kV], vn_lv_kv = min(Primary, Secondary)." +net.transformer,vk_percent,Transformers,resistance r ohm, reactance x ohm,"Short-circuit voltage [%], compute per-unit r_k = R_ohm/z_base, x_k = X_ohm/z_base with z_base = vn_lv_kv²/s_n, vk% = sign(x_k)·√(r_k² + x_k²)·100." +net.transformer,vkr_percent,Transformers,resistance r ohm,Real component of short-circuit voltage [%], vkr% = r_k·100 (same base as vk%). +net.transformer,pfe_kw,Transformers,conductance g (μS),"Iron losses [kW], pfe_kw = g0 · sn_mva · 1e3, where g0 is per-unit conductance from G (μS)." +net.transformer,i0_percent,Transformers,"susceptance b (μS), conductance g (μS)","No-load current [%], i0% = 100 · √(b0² + g0²) · (system sn_mva) / sn_mva, b0,g0 from B,G (μS) on transformer base." +net.transformer,shift_degree,Transformers,theta degree,"Phase shift angle [deg], default 0 if missing." +net.transformer,in_service,-,-,"Not provided by JAO, defaults to True." +net.transformer,oltc,-,-,Optional in create_transformer_from_parameters. Defaults to False +net.transformer,power_station_unit,-,-,"Not provided by JAO, defaults to False." +net.transformer,leakage_resistance_ratio_hv,-,-,Ratio of transformer short-circuit resistance allocated on HV side (default 0.5). +net.transformer,leakage_reactance_ratio_hv,-,-,Ratio of transformer short-circuit reactance allocated on HV side (default 0.5). diff --git a/doc/converter/jao.rst b/doc/converter/jao.rst index 1eccc42145..7b4a42cf75 100644 --- a/doc/converter/jao.rst +++ b/doc/converter/jao.rst @@ -3,6 +3,15 @@ JAO Static Grid Model Converter Function The ``from_jao`` function allows users to convert the Static Grid Model provided by JAO (Joint Allocation Office) into a pandapower network by reading and processing the provided Excel and HTML files. +The table below shows the mapping of the JAO columns to pandapower parameters necessary for executing a balanced power flow calculation. + +.. tabularcolumns:: |p{0.15\linewidth}|p{0.15\linewidth}|p{0.10\linewidth}|p{0.25\linewidth}|p{0.35\linewidth}| +.. csv-table:: JAO - pandapower mapping + :file: jao-converter.csv + :delim: , + :quote: " + :header-rows: 1 + :widths: 15, 15, 10, 25, 35 Function Overview ----------------- diff --git a/pandapower/converter/jao/__init__.py b/pandapower/converter/jao/__init__.py index 2aae42afad..9c5f46cbb0 100644 --- a/pandapower/converter/jao/__init__.py +++ b/pandapower/converter/jao/__init__.py @@ -1 +1,7 @@ -from .from_jao import from_jao \ No newline at end of file +# -*- coding: utf-8 -*- + +# Copyright (c) 2016-2026 by University of Kassel and Fraunhofer Institute for Energy Economics +# and Energy System Technology (IEE), Kassel. All rights reserved. + +from pandapower.converter.jao.from_jao import ( + from_jao, get_grid_groups, drop_islanded_grid_groups) diff --git a/pandapower/converter/jao/_correction.py b/pandapower/converter/jao/_correction.py new file mode 100644 index 0000000000..b8b00876e6 --- /dev/null +++ b/pandapower/converter/jao/_correction.py @@ -0,0 +1,441 @@ +# -*- coding: utf-8 -*- + +# Copyright (c) 2016-2026 by University of Kassel and Fraunhofer Institute for Energy Economics +# and Energy System Technology (IEE), Kassel. All rights reserved. + +""" +Data correction and name normalization for the JAO static grid model. + +The published data is maintained by hand and contains many inconsistencies: differing +capitalisation and spelling of the same substation across sheets, decimal commas, ``\\xa0`` +placeholders, unicode variants of Ω/µ, tap strings wrapped in ``<...>`` etc. This module + + * harmonises the column headers and coerces the electrical parameters to floats + (:func:`data_correction`), and + * derives a list of ``(old_name -> new_name)`` rename rules that unify the many spellings of + the same location before the network is built (:func:`generate_rename_locnames`). +""" + +import re +import difflib +import logging +import unicodedata +from collections import defaultdict + +import pandas as pd +from pandas.api.types import is_object_dtype + +from pandapower.converter.jao import _schema +from pandapower.converter.jao._schema import SheetSchema + +logger = logging.getLogger(__name__) + +# Tokens that must be dropped from a transformer location before matching it to a bus name. +_TRAFO_STOPWORDS = {"tr", "pst", "trafo", "kv"} +_TRAFO_BLOCK_EXACT = {"", "LIPST", "EHPST", "TFO"} +# Split a transformer location on whitespace and the "-A"/"-TD"/"-PF" unit tags only. +# Note: we deliberately do NOT split on "/", so that a slashed location such as "Hamburg/Nord" +# stays intact and can match the identically named bus. +_TRAFO_SPLIT = re.compile(r"[ ]+|-A[0-9]+|-TD[0-9]+|-PF[0-9]+") + + +def multi_str_repl(text: str, replacements: list[tuple[str, str]]) -> str: + """Apply a list of ``(old, new)`` string replacements in order.""" + for old, new in replacements: + text = text.replace(old, new) + return text + + +# ================================================================================================== +# Name normalization / matching helpers +# ================================================================================================== + +def _strip_accents(s: str) -> str: + nfkd = unicodedata.normalize("NFKD", str(s)) + return "".join(ch for ch in nfkd if not unicodedata.combining(ch)) + + +def _simplify_name(name: object) -> str: + """Reduce a location name to a canonical uppercase base token. + + Removes trailing markers like ``(2)`` and directional suffixes (``/W``, ``-West``, ...), + voltage texts (``220 kV``) and unifies delimiters, e.g. ``"Doerpen/W/W (2)" -> "DOERPEN"``. + """ + trailing = re.compile(r"(?ix)(?:\s*\(\d+\) | (?:[/\-]\s*|\s+)(?:W(?:EST)?|E|N|S))\s*$") + volt = re.compile(r"\b\d{2,4}\s*k?V\b", re.IGNORECASE) + delims = re.compile(r"[/_.-]") + + s = _strip_accents(str(name)).encode("ascii", "ignore").decode() + s = volt.sub("", s).rstrip() + while True: # remove trailing markers/directions until stable + s2 = trailing.sub("", s) + if s2 == s: + break + s = s2 + s = delims.sub(" ", s) + return " ".join(s.split()).upper() + + +def _canonical_bus_key(s: object) -> str: + """Canonical key used to detect bus names that differ only in case/accents/punctuation.""" + s = _strip_accents(str(s).casefold()) + s = re.sub(r"[()\[\]{}]", " ", s) + s = re.sub(r"[-_/.,;:]+", " ", s) + return re.sub(r"\s+", "", s).upper() + + +def normalize_trafo_name(name: object) -> tuple[str, str]: + """Tokenize a transformer location into ``(joined, longest)`` matching candidates. + + Drops stopwords, blocked tokens and tokens that contain digits, then returns the remaining + tokens joined by spaces and the single longest token. ``"/"`` is preserved inside tokens so + a slashed location survives (see :data:`_TRAFO_SPLIT`). + """ + parts = [p.strip().replace(" ", "") for p in _TRAFO_SPLIT.split(str(name).strip())] + + def keep(tok: str) -> bool: + return (tok not in _TRAFO_BLOCK_EXACT + and tok.lower() not in _TRAFO_STOPWORDS + and not any(ch.isdigit() for ch in tok)) + + filtered = [p for p in parts if keep(p)] + joined = " ".join(filtered).strip() + longest = max(filtered, key=len) if filtered else "" + return joined, longest + + +def _suggest_closest(query: str, candidates: list[str]) -> str: + if not query: + return "" + matches = difflib.get_close_matches(query, candidates, n=1, cutoff=0.6) + return matches[0] if matches else "" + + +def collect_bus_location_names(data: dict[str, pd.DataFrame]) -> set[str]: + """All unique substation ``Full_name`` strings found in the Lines/Tielines sheets.""" + names = [] + for key in ("Lines", "Tielines"): + if key not in data: + continue + schema = SheetSchema(data[key]) + for side in ("Substation_1", "Substation_2"): + col = schema.fullname_col(side) + if col is not None: + names.append(schema.series(col)) + if not names: + return set() + return set(pd.concat(names, ignore_index=True).dropna()) + + +def _find_problematic_bus_name_variants(data: dict[str, pd.DataFrame]) -> pd.DataFrame: + """Bus names that are equal up to case/accents/punctuation, mapped to one representative. + + The representative is the most frequent spelling (ties broken by length, then lexically). + Returns rows ``[original, suggested, reason="same_canonical_variant"]``. + """ + cols = ["original", "suggested", "reason"] + all_names = [] + for key in ("Lines", "Tielines"): + if key not in data: + continue + schema = SheetSchema(data[key]) + for side in ("Substation_1", "Substation_2"): + col = schema.fullname_col(side) + if col is not None: + all_names.append(schema.series(col)) + if not all_names: + return pd.DataFrame(columns=cols) + s = pd.concat(all_names, ignore_index=True).dropna() + if s.empty: + return pd.DataFrame(columns=cols) + + df = pd.DataFrame({"original": s}) + df["canonical"] = df["original"].map(_canonical_bus_key) + freq = df["original"].value_counts() + + rows = [] + for _, sub in df.groupby("canonical"): + uniques = sub["original"].unique() + if len(uniques) <= 1: + continue + representative = sorted(uniques, key=lambda x: (-freq.get(x, 0), -len(x), x))[0] + for orig in uniques: + if orig != representative: + rows.append({"original": orig, "suggested": representative, + "reason": "same_canonical_variant"}) + return pd.DataFrame(rows, columns=cols) + + +def _find_unmatched_transformer_locations(data: dict[str, pd.DataFrame]) -> pd.DataFrame: + """Transformer locations that do not match any bus name after normalization. + + Returns rows ``[original, joined, longest, suggested, reason]``. + """ + cols = ["original", "joined", "longest", "suggested", "reason"] + if "Transformers" not in data: + return pd.DataFrame(columns=cols) + + bus_names = collect_bus_location_names(data) + bus_names_lower = {b.lower(): b for b in bus_names} + trafo_names = SheetSchema(data["Transformers"]).transformer_location_series() + + rows = [] + for original in trafo_names: + joined, longest = normalize_trafo_name(original) + if (joined in bus_names or longest in bus_names + or joined.lower() in bus_names_lower or longest.lower() in bus_names_lower): + continue + rows.append({ + "original": original, + "joined": joined, + "longest": longest, + "suggested": _suggest_closest(joined or longest, list(bus_names)), + "reason": "no_match_after_normalization", + }) + return pd.DataFrame(rows, columns=cols) + + +def report_problematic_names(data: dict[str, pd.DataFrame]) -> pd.DataFrame: + """Combine the transformer-location and bus-variant reports into one DataFrame.""" + cols = ["original", "joined", "longest", "suggested", "reason"] + trafo = _find_unmatched_transformer_locations(data) + bus_vars = _find_problematic_bus_name_variants(data).reindex(columns=cols) + combined = pd.concat([trafo.reindex(columns=cols), bus_vars], + ignore_index=True).drop_duplicates() + if not combined.empty: + logger.debug("Problematic JAO names after normalization:\n%s", combined.to_string()) + return combined + + +def generate_rename_locnames(data: dict[str, pd.DataFrame], + combined: pd.DataFrame | None = None) -> list[tuple[str, str]]: + """Build ``(old -> new)`` rename rules that unify the many spellings of each location. + + Sources: bus-name variants that differ only in formatting, unmatched transformer locations + (mapped to their closest existing bus name, with PST/TR prefix heuristics) and conservative + suffix-removal proposals derived from the bus names themselves. Ambiguous rules (one source + mapping to several targets) are dropped, and the result is de-duplicated while preserving + order. + """ + if combined is None: + combined = report_problematic_names(data) + + bus_names = collect_bus_location_names(data) + bus_names_lower = {b.lower(): b for b in bus_names} + + def resolve_to_bus(name: str) -> str: + return bus_names_lower.get(name.lower(), name) if name else name + + renames: list[tuple[str, str]] = [] + + def add(old: str, new: str) -> None: + if not old or not new: + return + target = resolve_to_bus(new) + if old != target: + renames.append((old, target)) + + def add_prefix_variants(target_base: str) -> None: + for prefix in ("PST", "TR"): + add(f"{prefix}{target_base.replace(' ', '')}", f"{prefix} {target_base}") + add(f"{prefix}{target_base.upper().replace(' ', '')}", + f"{prefix} {target_base.upper()}") + + if "reason" in combined.columns: + # 1) bus-name formatting variants: original -> representative + for _, row in combined.loc[combined["reason"] == "same_canonical_variant"].iterrows(): + add(str(row["original"]).strip(), str(row["suggested"]).strip()) + + # 2) unmatched transformer locations -> suggestion / fallback + for _, row in combined.loc[ + combined["reason"] == "no_match_after_normalization"].iterrows(): + orig = str(row.get("original", "")).strip() + joined = str(row.get("joined", "")).strip() + longest = str(row.get("longest", "")).strip() + sugg = str(row.get("suggested", "")).strip() + + if sugg and (sugg in bus_names or sugg.lower() in bus_names_lower): + add(orig, sugg) + if joined and joined != sugg: + add(joined, sugg) + if longest and longest != sugg: + add(longest, sugg) + else: + for val in (joined, longest): + if val and (val in bus_names or val.lower() in bus_names_lower): + add(orig, resolve_to_bus(val)) + break + + if sugg: + add_prefix_variants(resolve_to_bus(sugg)) + for token in (joined, longest): + if token and sugg and token not in bus_names: + add(token, sugg) + + # 3) case harmonization straight from the transformer locations + if "Transformers" in data: + schema = SheetSchema(data["Transformers"]) + if schema.fullname_col(None) is not None: + for original in schema.transformer_location_series(): + for tok in set(normalize_trafo_name(original)): + if tok and tok.lower() in bus_names_lower: + target = bus_names_lower[tok.lower()] + if tok != target: + add(tok, target) + add_prefix_variants(target) + + # 4) conservative suffix-removal proposals from the bus names themselves + for name in bus_names: + base = _simplify_name(name) + if base and name != base: + add(name, base) + + # drop ambiguous rules (one source -> several targets) and de-duplicate, keeping order + targets_by_old = defaultdict(set) + for old, new in renames: + targets_by_old[old].add(new) + + out: list[tuple[str, str]] = [] + seen = set() + for pair in renames: + if len(targets_by_old[pair[0]]) == 1 and pair not in seen: + out.append(pair) + seen.add(pair) + return out + + +# ================================================================================================== +# Data correction +# ================================================================================================== + +def _correct_line_columns(df: pd.DataFrame) -> None: + """Harmonise known Lines/Tielines header variants in place.""" + voltage = "Voltage_level(kV)" + replace_map = { + "Full Name": "Full_name", + "Short Name": "Short_name", + "Susceptance_B (µS)": "Susceptance_B(μS)", + "Voltage_level (kV)": voltage, + "Voltage_level [kV]": voltage, + } + cols = df.columns.to_frame(index=False) + cols.iloc[:, 1] = cols.iloc[:, 1].replace(replace_map) + cols.loc[cols.iloc[:, 1].isin([voltage, "Comment"]), cols.columns[0]] = None + cols.loc[cols.iloc[:, 0].astype(str).str.startswith("Unnamed:"), cols.columns[0]] = None + cols.loc[cols.iloc[:, 1] == _schema.LENGTH, cols.columns[0]] = _schema.ELECTRICAL_PARAMETERS + df.columns = pd.MultiIndex.from_frame(cols) + + +def _correct_line_numerics(df: pd.DataFrame, schema: SheetSchema, + max_i_ka_fillna: float) -> None: + """Coerce Imax and the R/X/B/length columns of a Lines/Tielines sheet to floats.""" + if _schema.LINE_IMAX in df.columns: + fill = max_i_ka_fillna * 1e3 + df[_schema.LINE_IMAX] = pd.to_numeric( + df[_schema.LINE_IMAX] + .replace({"\xa0": fill, "-": fill, " ": fill}) + .astype(str).str.replace(",", ".", regex=False), + errors="coerce") + + for col in ((_schema.ELECTRICAL_PARAMETERS, _schema.LENGTH), + (_schema.ELECTRICAL_PARAMETERS, _schema.RESISTANCE), + (_schema.ELECTRICAL_PARAMETERS, _schema.REACTANCE)): + if col in df.columns: + df[col] = _schema.numeric_column(df, col) + + for col in (schema.resistance_col(), schema.reactance_col(), schema.susceptance_col()): + if col is not None: + df.iloc[:, list(df.columns).index(col)] = _schema.numeric_column(df, col) + + +def _correct_transformer_taps(df: pd.DataFrame) -> None: + """Clean the tap and phase-shifter columns of the Transformers sheet in place.""" + taps = df.loc[:, _schema.TAPS].fillna("").astype(str).str.replace(" ", "", regex=False) + nonnull = taps.apply(len).astype(bool) + nonnull_taps = taps.loc[nonnull] + surrounded = nonnull_taps.str.startswith("<") & nonnull_taps.str.endswith(">") + nonnull_taps.loc[surrounded] = nonnull_taps.loc[surrounded].str[1:-1] + slash_sep = (~nonnull_taps.str.contains(";")) & nonnull_taps.str.contains("/") + nonnull_taps.loc[slash_sep] = nonnull_taps.loc[slash_sep].str.replace("/", ";", regex=False) + nonnull_taps.loc[nonnull_taps == "0"] = "0;0" + df.loc[nonnull, _schema.TAPS] = nonnull_taps + df.loc[~nonnull, _schema.TAPS] = "0;0" + + # phase shifters sometimes carry two "/"-separated values; keep the second one + for col in ("Phase Regulation δu (%)", "Angle Regulation δu (%)"): + tup = (_schema.PHASE_SHIFT_PROPERTIES, col) + if tup in df.columns and is_object_dtype(df.loc[:, tup]): + double = df.index[df.loc[:, tup].str.contains("/").fillna(False).astype(bool)] + df.loc[double, tup] = df.loc[double, tup].str.split("/", expand=True)[1].str.replace( + ",", ".", regex=False).astype(float).values + + +def data_correction(data: dict[str, pd.DataFrame], html_str: str | None, + max_i_ka_fillna: float) -> str | None: + """Correct the Excel sheets in place and apply the rename rules to the HTML string. + + Returns the (possibly corrected) HTML string. + """ + combined = report_problematic_names(data) + rename_locnames = generate_rename_locnames(data, combined) + + # keep only high-similarity renames that do not map between two already existing bus names + bus_names = collect_bus_location_names(data) + filtered = [] + for old, new in rename_locnames: + if old in bus_names and new in bus_names: + continue + if difflib.SequenceMatcher(None, old.lower(), new.lower()).ratio() < 0.8: + continue + filtered.append((old, new)) + rename_locnames = filtered + + for key in ("Lines", "Tielines"): + if key not in data: + continue + df = data[key] + _correct_line_columns(df) + # the schema must be built AFTER the column headers were corrected above + schema = SheetSchema(df) + _ensure_line_tso_column(df) + _correct_line_numerics(df, schema, max_i_ka_fillna) + + # unify capitalisation/spelling of the location names + loc_cols = [(None, "NE_name"), + schema.fullname_col("Substation_1"), + schema.fullname_col("Substation_2")] + for col in loc_cols: + if col is not None and col in df.columns: + df.loc[:, col] = df.loc[:, col].astype(str).str.strip().apply( + multi_str_repl, replacements=rename_locnames) + + html_str = multi_str_repl(html_str, rename_locnames) + + if "Transformers" in data: + df = data["Transformers"] + loc_name = ("Location", "Full Name") + if loc_name in df.columns: + df.loc[:, loc_name] = df.loc[:, loc_name].astype(str).str.strip().apply( + multi_str_repl, replacements=rename_locnames) + _correct_transformer_taps(df) + + return html_str + + +def _ensure_line_tso_column(df: pd.DataFrame) -> None: + """Ensure a generic ``(None, "TSO")`` column exists, joining ``TSO 1``/``TSO 2`` if needed.""" + if (None, "TSO") in df.columns: + return + + def first_present(candidates: list[tuple]) -> tuple | None: + return next((t for t in candidates if t in df.columns), None) + + t1 = first_present([(None, "TSO 1"), (None, "TSO1")]) + t2 = first_present([(None, "TSO 2"), (None, "TSO2")]) + if t1 and t2: + df[(None, "TSO")] = df.loc[:, t1].astype(str).str.strip() + "/" + \ + df.loc[:, t2].astype(str).str.strip() + elif t1: + df[(None, "TSO")] = df.loc[:, t1] + elif t2: + df[(None, "TSO")] = df.loc[:, t2] diff --git a/pandapower/converter/jao/_elements.py b/pandapower/converter/jao/_elements.py new file mode 100644 index 0000000000..fdc7dde87e --- /dev/null +++ b/pandapower/converter/jao/_elements.py @@ -0,0 +1,468 @@ +# -*- coding: utf-8 -*- + +# Copyright (c) 2016-2026 by University of Kassel and Fraunhofer Institute for Energy Economics +# and Energy System Technology (IEE), Kassel. All rights reserved. + +""" +Creation of buses, lines and transformers from the corrected JAO sheets. + +Buses are derived from the substations named in the Lines/Tielines sheets. Lines and tielines +are then created between those buses. Transformers are matched to their location by name and +voltage; where a suitable bus is missing (or its voltage deviates too much) new buses are +created. +""" + +import difflib +import logging + +import numpy as np +import pandas as pd +from pandas.api.types import is_integer_dtype + +from pandapower.create import create_buses, create_lines_from_parameters, \ + create_transformers_from_parameters +from pandapower.io_utils import pandapowerNet +from pandapower.converter.jao import _schema +from pandapower.converter.jao._schema import SheetSchema +from pandapower.converter.jao._correction import normalize_trafo_name + +logger = logging.getLogger(__name__) + + +# ================================================================================================== +# Small shared helpers +# ================================================================================================== + +def get_bus_idx(net: pandapowerNet) -> pd.Series: + """Series mapping ``(name, vn_kv) -> bus index``.""" + return net.bus[["name", "vn_kv"]].rename_axis("index").reset_index().set_index( + ["name", "vn_kv"])["index"] + + +def _drop_duplicates_and_join_tso(bus_df: pd.DataFrame) -> pd.DataFrame: + """Keep one bus per ``(name, vn_kv)``, joining the TSO strings of merged duplicates.""" + bus_df = bus_df.drop_duplicates(ignore_index=True) + bus_df = bus_df.groupby(["name", "vn_kv"], as_index=False).agg({"TSO": lambda x: "/".join(x)}) + if bus_df.duplicated(["name", "vn_kv"]).any(): + raise AssertionError("bus_df contains duplicate names with identical vn_kv") + return bus_df + + +def _get_float_column(df: pd.DataFrame, col_tuple: tuple, fill: float = 0) -> pd.Series: + series = df.loc[:, col_tuple] + series.loc[series == "\xa0"] = fill + return series.astype(float).fillna(fill) + + +# ================================================================================================== +# Buses and lines +# ================================================================================================== + +def create_buses_from_line_data(net: pandapowerNet, data: dict[str, pd.DataFrame]) -> None: + """Create the buses named as substations in the Lines and Tielines sheets.""" + parts = [] + for key in ("Lines", "Tielines"): + if key not in data: + continue + df = data[key] + schema = SheetSchema(df) + vn_col = schema.voltage_col() + if vn_col is None: + raise KeyError(f"{key}: no Voltage_level column found (fuzzy).") + vn_kv = _schema.numeric_column(df, vn_col) + for side in ("Substation_1", "Substation_2"): + full = schema.fullname_col(side) + if full is None: + continue + parts.append(pd.DataFrame({ + "name": schema.series(full).to_numpy(), + "vn_kv": vn_kv, + "TSO": schema.tso_series_for_side(side).to_numpy(), + })) + + bus_df = pd.concat(parts, ignore_index=True) if parts else \ + pd.DataFrame({"name": [], "vn_kv": [], "TSO": []}) + bus_df = _drop_duplicates_and_join_tso(bus_df) + new_bus_idx = create_buses(net, len(bus_df), vn_kv=bus_df.vn_kv, name=bus_df.name, + zone=bus_df.TSO) + if not np.array_equal(new_bus_idx, bus_df.index): + raise AssertionError("Created bus indices do not match the expected bus_df index.") + + +def create_lines(net: pandapowerNet, data: dict[str, pd.DataFrame], + max_i_ka_fillna: float) -> None: + """Create lines and tielines between the previously created buses.""" + bus_idx = get_bus_idx(net) + + for key in ("Lines", "Tielines"): + if key not in data: + continue + df = data[key] + schema = SheetSchema(df) + + vn_col = schema.voltage_col() + if vn_col is None: + raise KeyError(f"{key}: Voltage_level column not found (fuzzy).") + s1 = schema.fullname_col("Substation_1") + s2 = schema.fullname_col("Substation_2") + if s1 is None or s2 is None: + raise KeyError(f"{key}: Substation_1/2 Full_name column not found (fuzzy).") + + vn_kvs = df.loc[:, vn_col].to_numpy() + name1 = df[s1].astype(str).str.strip() + name2 = df[s2].astype(str).str.strip() + valid = (~pd.isna(vn_kvs) + & ~name1.isin(["", "NAN"]) & ~name2.isin(["", "NAN"])) + n_invalid = len(df) - int(valid.sum()) + if n_invalid: + logger.warning(f"{n_invalid} {key.lower()} were dropped due to missing or invalid data.") + + df = df[valid].copy() + vn_kvs = vn_kvs[valid.to_numpy()] + # rebuild the schema on the filtered rows so all column accessors align in length + schema = SheetSchema(df) + + length_km = df[(_schema.ELECTRICAL_PARAMETERS, _schema.LENGTH)].to_numpy() + zero_length = np.isclose(length_km, 0) + no_length = np.isnan(length_km) + if zero_length.any() or no_length.any(): + logger.warning( + f"According to the data, {int(zero_length.sum())} {key.lower()} have zero length " + f"and {int(no_length.sum())} have no length; both are set to 1 km.") + length_km[zero_length | no_length] = 1 + + from_bus = bus_idx.loc[list(zip(df[s1].astype(str), vn_kvs))].to_numpy() + to_bus = bus_idx.loc[list(zip(df[s2].astype(str), vn_kvs))].to_numpy() + + r_ohm = _rxb_values(df, schema.resistance_col(), + (_schema.ELECTRICAL_PARAMETERS, _schema.RESISTANCE)) + x_ohm = _rxb_values(df, schema.reactance_col(), + (_schema.ELECTRICAL_PARAMETERS, _schema.REACTANCE)) + b_us = _rxb_values(df, schema.susceptance_col(), None) + + if _schema.LINE_IMAX in df.columns: + i_ka = df[_schema.LINE_IMAX].fillna(max_i_ka_fillna * 1e3).to_numpy() / 1e3 + else: + i_ka = np.full(len(df), max_i_ka_fillna) + + create_lines_from_parameters( + net, from_bus, to_bus, length_km, + r_ohm / length_km, x_ohm / length_km, b_us / length_km, i_ka, + name=schema.string_values("NE name", tokens=["ne", "name"], default=None), + EIC_Code=schema.string_values("EIC code", tokens=["eic", "code"], default=None), + TSO=schema.line_tso_array(), + Comment=schema.string_values("comment", tokens=["comment"], default=""), + Tieline=(key == "Tielines"), + ) + + +def _rxb_values(df: pd.DataFrame, fuzzy_col: tuple | None, + fallback_col: tuple | None) -> np.ndarray: + """R/X/B values, preferring the fuzzy-matched column and falling back to the exact tuple.""" + if fuzzy_col is not None: + return df.iloc[:, list(df.columns).index(fuzzy_col)].to_numpy() + if fallback_col is not None and fallback_col in df.columns: + return df[fallback_col].to_numpy() + return np.zeros(len(df)) + + +# ================================================================================================== +# Transformers +# ================================================================================================== + +def create_transformers_and_buses(net: pandapowerNet, data: dict[str, pd.DataFrame], + strict: bool = True, **kwargs) -> None: + """Create transformers, matching them to buses and creating buses where necessary. + + Transformers whose location cannot be matched to a bus raise a ``ValueError`` when ``strict`` + is True (the default), or are skipped with a warning when ``strict`` is False. + """ + df = data["Transformers"] + schema = SheetSchema(df) + + bus_idx = get_bus_idx(net) + vn_hv_kv, vn_lv_kv = _get_transformer_voltages(schema, bus_idx) + trafo_connections, keep = _allocate_trafos_to_buses( + net, schema, bus_idx, vn_hv_kv, vn_lv_kv, strict=strict, **kwargs) + + # restrict every per-transformer input to the transformers that were actually allocated + if not keep.all(): + df = df.loc[keep].copy() + schema = SheetSchema(df) + vn_hv_kv = vn_hv_kv[keep] + vn_lv_kv = vn_lv_kv[keep] + + max_fixed = pd.to_numeric(df.loc[:, ("Maximum Current Imax (A) primary", "Fixed")], + errors="coerce") + max_max = pd.to_numeric(df.loc[:, ("Maximum Current Imax (A) primary", "Max")], + errors="coerce") + max_i_a = np.asarray(max_fixed.fillna(max_max), dtype=float) + + vn_hv_arr = vn_hv_kv.astype(float) + vn_lv_arr = vn_lv_kv.astype(float) + sn_mva = np.sqrt(3.0) * max_i_a * vn_hv_arr / 1e3 + z_pu = vn_lv_arr ** 2 / sn_mva + + r_ohm = schema.numeric_values("resistance r ohm", tokens=["resistance"]) + x_ohm = schema.numeric_values("reactance x ohm", tokens=["reactance"]) + b_us = schema.numeric_values("susceptance b us", tokens=["susceptance", "b", "us"]) + g_us = schema.numeric_values("conductance g us", tokens=["conductance", "g", "us"]) + + rk = r_ohm / z_pu + xk = x_ohm / z_pu + b0 = b_us * 1e-6 * z_pu + g0 = g_us * 1e-6 * z_pu + zk = np.sqrt(rk ** 2 + xk ** 2) + vk_percent = np.sign(xk) * zk * 100 + vkr_percent = rk * 100 + pfe_kw = g0 * sn_mva * 1e3 + i0_percent = 100 * np.sqrt(b0 ** 2 + g0 ** 2) * net.sn_mva / sn_mva + + taps = df.loc[:, _schema.TAPS].str.split(";", expand=True).astype(int).set_axis( + ["tap_min", "tap_max"], axis=1) + du = _get_float_column(df, (_schema.PHASE_SHIFT_PROPERTIES, "Phase Regulation δu (%)")) + dphi = _get_float_column(df, (_schema.PHASE_SHIFT_PROPERTIES, "Angle Regulation δu (%)")) + phase_shifter = np.isclose(du, 0) & (~np.isclose(dphi, 0)) + + comment = pd.Series(schema.string_values("comment", tokens=["comment"], default="")).replace( + "\xa0", "").to_numpy() + + create_transformers_from_parameters( + net, + trafo_connections.hv_bus.values, + trafo_connections.lv_bus.values, + sn_mva, vn_hv_kv, vn_lv_kv, vkr_percent, vk_percent, pfe_kw, i0_percent, + shift_degree=schema.numeric_values("theta degree", tokens=["theta"]), + tap_pos=0, tap_neutral=0, tap_side="lv", + tap_min=taps["tap_min"].values, tap_max=taps["tap_max"].values, + tap_phase_shifter=phase_shifter, + tap_step_percent=du, tap_step_degree=dphi, + name=schema.transformer_location_series().values, + EIC_Code=schema.string_values("eic code", tokens=["eic", "code"], default=None), + TSO=schema.transformer_tso_series().values, + Comment=comment, + ) + + +def _get_transformer_voltages(schema: SheetSchema, + bus_idx: pd.Series) -> tuple[np.ndarray, np.ndarray]: + col_p, col_s = schema.transformer_voltage_cols() + if col_p is None or col_s is None: + raise KeyError("Transformers: primary/secondary Voltage_level not found (fuzzy).") + vn_p = _schema.to_numeric(schema.df.loc[:, col_p]) + vn_s = _schema.to_numeric(schema.df.loc[:, col_s]) + vn_hv_kv = np.maximum(vn_p, vn_s) + vn_lv_kv = np.minimum(vn_p, vn_s) + + # keep the voltages integer if the bus index uses integer voltages + try: + if is_integer_dtype(list(bus_idx.index.dtypes)[1]): + vn_hv_kv = vn_hv_kv.astype(int) + vn_lv_kv = vn_lv_kv.astype(int) + except Exception: + pass + return vn_hv_kv, vn_lv_kv + + +def _find_trafo_locations(trafo_bus_names: pd.Series, + bus_location_names: set[str]) -> tuple[pd.Series, pd.Series]: + """Resolve each transformer location string to an existing bus location name. + + Each name is tokenized (keeping ``/`` intact, see :func:`normalize_trafo_name`) into a + joined candidate and its longest token. Exact matches are preferred; otherwise a close + difflib match (cutoff 0.8) is used. + + Returns ``(location_names, unmatched)`` where ``unmatched`` is a boolean Series flagging the + transformers for which no suitable bus location name was found. The caller decides whether an + unmatched transformer is a hard error or is skipped. The location string of an unmatched + transformer is left at its longest token (unused unless the caller skips it). + """ + normalized = trafo_bus_names.map(normalize_trafo_name) + joined = normalized.map(lambda t: t[0]) + longest = normalized.map(lambda t: t[1]) + + joined_hit = joined.isin(bus_location_names) + longest_hit = longest.isin(bus_location_names) + unmatched = ~(joined_hit | longest_hit) + + if unmatched.any(): + candidates = list(bus_location_names) + for i in unmatched[unmatched].index: + query = joined.at[i] or longest.at[i] + if not query: + continue + match = difflib.get_close_matches(query, candidates, n=1, cutoff=0.8) + if match: + joined.at[i] = match[0] + joined_hit.at[i] = True + unmatched.at[i] = False + + location_names = longest.copy() + location_names.loc[joined_hit] = joined.loc[joined_hit] + return location_names, unmatched + + +def _report_unmatched_trafos(trafo_bus_names: pd.Series, unmatched: pd.Series) -> str: + """Build a human-readable, one-per-line list of the unmatched transformer locations.""" + items = trafo_bus_names.loc[unmatched] + listing = "\n".join(f" - {name}" for name in items) + return (f"For {int(unmatched.sum())} transformers, no suitable bus location names were found " + f"(missing buses or inconsistent naming). Affected transformers:\n{listing}") + + +def _allocate_trafos_to_buses(net: pandapowerNet, schema: SheetSchema, bus_idx: pd.Series, + vn_hv_kv: np.ndarray, vn_lv_kv: np.ndarray, + rel_deviation_threshold_for_trafo_bus_creation: float = 0.2, + log_rel_vn_deviation: float = 0.12, strict: bool = True, + **kwargs) -> tuple[pd.DataFrame, np.ndarray]: + """Allocate transformers to bus pairs by location and voltage, creating buses when needed. + + For each side, transformers are connected to the bus at their location with the matching + voltage. If only a differing voltage exists, a new bus is created when the relative deviation + exceeds ``rel_deviation_threshold_for_trafo_bus_creation`` (a warning is logged above + ``log_rel_vn_deviation``). Transformers whose HV and LV side would land on the same bus get a + duplicated LV bus (suffix ``" (2)"``). + + Transformers whose location cannot be matched to any bus raise a ``ValueError`` (listing all + of them) when ``strict`` is True, or are dropped with a warning when ``strict`` is False. + + Returns ``(trafo_connections, keep)`` where ``keep`` is a boolean array over the original + transformer rows telling the caller which transformers were kept (all True when strict). + """ + if rel_deviation_threshold_for_trafo_bus_creation < log_rel_vn_deviation: + logger.warning( + f"Given parameters violate {rel_deviation_threshold_for_trafo_bus_creation=} >= " + f"{log_rel_vn_deviation=}. Therefore, " + f"rel_deviation_threshold_for_trafo_bus_creation={log_rel_vn_deviation} is assumed.") + rel_deviation_threshold_for_trafo_bus_creation = log_rel_vn_deviation + + bus_location_names = set(net.bus.name) + trafo_bus_names = schema.transformer_location_series() + trafo_location_names, unmatched = _find_trafo_locations(trafo_bus_names, bus_location_names) + + # a transformer without a (valid) voltage on either side cannot be placed on a bus either + no_voltage = pd.Series( + np.isnan(vn_hv_kv.astype(float)) | np.isnan(vn_lv_kv.astype(float)), + index=unmatched.index) + unmatched = unmatched | no_voltage + + # TSO per transformer, kept aligned to trafo_connections (used when new buses are created) + trafo_tso = schema.transformer_tso_series().reset_index(drop=True) + + keep = ~unmatched.to_numpy() + if unmatched.any(): + message = _report_unmatched_trafos(trafo_bus_names, unmatched) + if strict: + raise ValueError( + message + "\n\nPass strict=False to from_jao() to skip these transformers and " + "still perform the conversion (the resulting network will be incomplete).") + logger.warning(message + "\n\nThese transformers are skipped (strict=False).") + # drop the unmatched transformers from every per-transformer input + trafo_location_names = trafo_location_names.loc[keep].reset_index(drop=True) + trafo_bus_names = trafo_bus_names.loc[keep].reset_index(drop=True) + trafo_tso = trafo_tso.loc[keep].reset_index(drop=True) + vn_hv_kv = vn_hv_kv[keep] + vn_lv_kv = vn_lv_kv[keep] + + # use a clean 0..N-1 index so trafo_connections, trafo_tso and the vn arrays stay aligned + trafo_location_names = trafo_location_names.reset_index(drop=True) + empties = -1 * np.ones(len(vn_hv_kv), dtype=int) + trafo_connections = pd.DataFrame({ + "name": trafo_location_names, + "hv_bus": empties, + "lv_bus": empties, + "vn_hv_kv": vn_hv_kv, + "vn_lv_kv": vn_lv_kv, + "vn_hv_kv_next_bus": vn_hv_kv, + "vn_lv_kv_next_bus": vn_lv_kv, + "hv_rel_deviation": np.zeros(len(vn_hv_kv)), + "lv_rel_deviation": np.zeros(len(vn_hv_kv)), + }) + trafo_connections[["hv_bus", "lv_bus"]] = \ + trafo_connections[["hv_bus", "lv_bus"]].astype(np.int64) + + for side in ("hv", "lv"): + bus_col = f"{side}_bus" + trafo_vn_col = f"vn_{side}_kv" + next_col = f"vn_{side}_kv_next_bus" + rel_dev_col = f"{side}_rel_deviation" + has_dev_col = f"trafo_{side}_to_bus_deviation" + + name_vn = pd.Series(tuple(zip(trafo_location_names, trafo_connections[trafo_vn_col]))) + isin = name_vn.isin(bus_idx.index) + trafo_connections[has_dev_col] = ~isin + trafo_connections.loc[isin, bus_col] = bus_idx.loc[name_vn.loc[isin]].values + + # for locations without the exact voltage, take the nearest available voltage + next_vn = np.array([ + bus_idx.loc[row.name].index.values[ + (pd.Series(bus_idx.loc[row.name].index) - getattr(row, trafo_vn_col)) + .abs().idxmin()] + for row in trafo_connections.loc[~isin, ["name", trafo_vn_col]].itertuples()]) + trafo_connections.loc[~isin, next_col] = next_vn + rel_dev = np.abs(next_vn - trafo_connections.loc[~isin, trafo_vn_col].values) / next_vn + trafo_connections.loc[~isin, rel_dev_col] = rel_dev + trafo_connections.loc[~isin, bus_col] = bus_idx.loc[list(zip( + trafo_connections.loc[~isin, "name"], + trafo_connections.loc[~isin, next_col]))].values + + need_bus = trafo_connections[rel_dev_col] > rel_deviation_threshold_for_trafo_bus_creation + if need_bus.any(): + new_bus_data = pd.DataFrame({ + "vn_kv": trafo_connections.loc[need_bus, trafo_vn_col].values, + "name": trafo_connections.loc[need_bus, "name"].values, + "TSO": trafo_tso.loc[need_bus].values, + }) + new_bus_dd = _drop_duplicates_and_join_tso(new_bus_data) + new_bus_idx = create_buses(net, len(new_bus_dd), vn_kv=new_bus_dd.vn_kv, + name=new_bus_dd.name, zone=new_bus_dd.TSO) + trafo_connections.loc[need_bus, bus_col] = net.bus.loc[ + new_bus_idx, ["name", "vn_kv"]].reset_index().set_index(["name", "vn_kv"]).loc[ + list(new_bus_data[["name", "vn_kv"]].itertuples(index=False, name=None))].values + trafo_connections.loc[need_bus, next_col] = \ + trafo_connections.loc[need_bus, trafo_vn_col].values + trafo_connections.loc[need_bus, rel_dev_col] = 0 + trafo_connections.loc[need_bus, has_dev_col] = False + + _duplicate_same_bus_connections(net, trafo_connections, trafo_location_names) + + for side in ("hv", "lv"): + rel_dev_col = f"{side}_rel_deviation" + has_dev_col = f"trafo_{side}_to_bus_deviation" + next_col = f"vn_{side}_kv_next_bus" + trafo_vn_col = f"vn_{side}_kv" + need_logging = trafo_connections.loc[trafo_connections[has_dev_col], + rel_dev_col] > log_rel_vn_deviation + if n := int(need_logging.sum()): + idx_max = trafo_connections[rel_dev_col].idxmax() + logger.warning( + f"For {n} transformers ({side} side), only locations with relative deviation > " + f"{log_rel_vn_deviation} were found. Max deviation " + f"{trafo_connections[rel_dev_col].max()} at " + f"{trafo_connections.at[idx_max, trafo_vn_col]} kV vs bus " + f"{trafo_connections.at[idx_max, next_col]} kV.") + + assert (trafo_connections.hv_bus > -1).all() + assert (trafo_connections.lv_bus > -1).all() + assert (trafo_connections.hv_bus != trafo_connections.lv_bus).all() + return trafo_connections, keep + + +def _duplicate_same_bus_connections(net: pandapowerNet, trafo_connections: pd.DataFrame, + trafo_location_names: pd.Series) -> None: + """Give transformers whose HV and LV side share a bus a duplicated LV bus (suffix " (2)").""" + same_bus = trafo_connections.hv_bus == trafo_connections.lv_bus + duplicated = net.bus.loc[trafo_connections.loc[same_bus, "lv_bus"]].copy() + if duplicated.empty: + return + duplicated["name"] += " (2)" + start = net.bus.index.max() + 1 + duplicated.index = list(range(start, start + len(duplicated))) + trafo_connections.loc[same_bus, "lv_bus"] = duplicated.index + net.bus = pd.concat([net.bus, duplicated]) + + tr_names = trafo_location_names.loc[same_bus] + are_psts = tr_names.str.contains("PST") + logger.info( + f"{len(duplicated)} additional buses created to avoid same-bus transformers. " + f"Of {len(tr_names)} transformers, {int(are_psts.sum())} contain 'PST'.") diff --git a/pandapower/converter/jao/_geodata.py b/pandapower/converter/jao/_geodata.py new file mode 100644 index 0000000000..5dd1fdc5b1 --- /dev/null +++ b/pandapower/converter/jao/_geodata.py @@ -0,0 +1,230 @@ +# -*- coding: utf-8 -*- + +# Copyright (c) 2016-2026 by University of Kassel and Fraunhofer Institute for Energy Economics +# and Energy System Technology (IEE), Kassel. All rights reserved. + +""" +Geodata extraction from the JAO HTML map and its transfer onto buses and lines. + +The published HTML embeds the line geometry inside a leaflet ``htmlwidget`` JSON blob. +:func:`parse_html_str` extracts the line endpoints; :func:`add_bus_geo` assigns a coordinate to +each bus from the geometry of its connected lines. +""" + +import re +import json +import logging + +import numpy as np +import pandas as pd + +from pandapower.io_utils import pandapowerNet +from pandapower.plotting import set_line_geodata_from_bus_geodata + +logger = logging.getLogger(__name__) + +# The leaflet widget id changes with every JAO release, so it must not be hard-coded. Match any +# hex id and pick the JSON block that actually carries the map data (an "addPolylines" call). +_WIDGET_SCRIPT = re.compile( + r']*type="application/json"[^>]*data-for="htmlwidget-[0-9a-f]+"[^>]*>(.*?)', + re.DOTALL) + + +def parse_html_str(html_str: str) -> pd.DataFrame: + """Parse the embedded leaflet JSON and return per-line endpoint coordinates. + + Returns a tidy DataFrame with columns ``[EIC_Code, name, bus, geo_dim, value]`` where ``bus`` + is ``from``/``to`` and ``geo_dim`` is ``lng``/``lat``. + + Raises ``json.JSONDecodeError``/``KeyError`` if no map widget is found and ``AssertionError`` + if the EIC and geometry lists have different lengths. + """ + geo_data = _extract_widget_calls(html_str) + methods_pos = pd.Series({item["method"]: i for i, item in enumerate(geo_data)}) + polylines = geo_data[methods_pos.at["addPolylines"]]["args"] + + if len(polylines[6]) != len(polylines[0]): + raise AssertionError("The lists of EIC Code data and geo data are not of the same length.") + + eic_start = "EIC Code: " + line_eic = [tip[tip.find(eic_start) + len(eic_start):] for tip in polylines[6]] + line_name = [_filter_name(tip) for tip in polylines[6]] + line_geo_data = pd.concat( + [_lng_lat_to_df(polylines[0][i][0][0], line_eic[i], line_name[i]) + for i in range(len(polylines[0]))], + ignore_index=True) + + for col in ("EIC_Code", "name"): + line_geo_data[col] = line_geo_data[col].str.strip() + return line_geo_data + + +def _extract_widget_calls(html_str: str) -> list: + """Return the ``x.calls`` list of the leaflet map widget from the HTML. + + Scans every ``application/json`` htmlwidget script and returns the calls of the first one + that contains an ``addPolylines`` method (the map layer). + """ + for match in _WIDGET_SCRIPT.finditer(html_str): + try: + calls = json.loads(match.group(1))["x"]["calls"] + except (json.JSONDecodeError, KeyError, TypeError): + continue + if any(item.get("method") == "addPolylines" for item in calls): + return calls + raise json.JSONDecodeError("No leaflet map widget with polyline data found.", html_str, 0) + + +def _filter_name(tooltip: str) -> str: + name_start, name_end = "NE name: ", "" + pos0 = tooltip.find(name_start) + len(name_start) + pos1 = tooltip.find(name_end, pos0) + if pos0 < len(name_start) or pos1 < pos0: + raise AssertionError("Could not parse the NE name from a map tooltip.") + return tooltip[pos0:pos1] + + +def _lng_lat_to_df(coords: dict, line_eic: str, line_name: str) -> pd.DataFrame: + """Turn a ``{"lng": [..], "lat": [..]}`` endpoint dict into four tidy rows.""" + return pd.DataFrame([ + [line_eic, line_name, "from", "lng", coords["lng"][0]], + [line_eic, line_name, "to", "lng", coords["lng"][1]], + [line_eic, line_name, "from", "lat", coords["lat"][0]], + [line_eic, line_name, "to", "lat", coords["lat"][1]], + ], columns=["EIC_Code", "name", "bus", "geo_dim", "value"]) + + +def add_bus_geo(net: pandapowerNet, line_geo_data: pd.DataFrame) -> None: + """Assign a coordinate to every bus from the geometry of its connected lines. + + Coordinates are looked up primarily by EIC code and fall back to the line name where the EIC + is duplicated or missing. Ambiguous cases are reduced by rounding and by picking the most + frequent coordinate. The result is written as GeoJSON strings into ``net.bus.geo``. + """ + slicer = pd.IndexSlice + lgd_eic_bus = line_geo_data.pivot_table(values="value", index=["EIC_Code", "bus"], + columns="geo_dim") + lgd_name_bus = line_geo_data.pivot_table(values="value", index=["name", "bus"], + columns="geo_dim") + lgd_bus = pd.concat([ + lgd_eic_bus.set_axis(_extend_index(lgd_eic_bus, "EIC_Code")), + lgd_name_bus.set_axis(_extend_index(lgd_name_bus, "name")), + ]) + dupl_eics = net.line.EIC_Code.loc[net.line.EIC_Code.duplicated()] + dupl_names = net.line.name.loc[net.line.name.duplicated()] + + def geo_json(bus_geo: pd.Series) -> str: + return f'{{"coordinates": [{bus_geo.at["lng"]}, {bus_geo.at["lat"]}], "type": "Point"}}' + + def bus_geo(bus: int) -> str | None: + from_excerpt = net.line.loc[net.line.from_bus == bus, ["EIC_Code", "name", "Tieline"]] + to_excerpt = net.line.loc[net.line.to_bus == bus, ["EIC_Code", "name", "Tieline"]] + line_excerpt = pd.concat([from_excerpt, to_excerpt]) + n_ends = len(line_excerpt) + if n_ends == 0: + logger.error(f"Bus {bus} (name {net.bus.at[bus, 'name']}) is not found in " + "line_geo_data.") + return None + + is_dupl = pd.concat([ + _isin_frame(from_excerpt, dupl_eics, dupl_names, "from"), + _isin_frame(to_excerpt, dupl_eics, dupl_names, "to"), + ]) + is_missing = pd.DataFrame({ + "EIC": ~line_excerpt.EIC_Code.isin( + lgd_bus.loc["EIC_Code"].index.get_level_values("identifier")), + "name": ~line_excerpt.name.isin( + lgd_bus.loc["name"].index.get_level_values("identifier")), + }).set_axis(is_dupl.index, axis=0) + is_tieline = pd.Series(net.line.loc[is_dupl.index.get_level_values("line_index"), + "Tieline"].values, index=is_dupl.index) + + # default to the EIC code, switch to the line name where the EIC is duplicated/missing + access = pd.DataFrame({ + "col_name": "EIC_Code", + "identifier": line_excerpt.EIC_Code.values, + "bus": is_dupl.index.get_level_values("bus").values, + }) + take_from_name = ((is_dupl.EIC | is_missing.EIC) + & (~is_dupl.name & ~is_missing.name)).values + access.loc[take_from_name, "col_name"] = "name" + access.loc[take_from_name, "identifier"] = line_excerpt.name.loc[take_from_name].values + + keep = (~(is_dupl | is_missing)).any(axis=1).values + if np.all(is_missing): + msg = (f"For bus {bus} (name {net.bus.at[bus, 'name']}), {n_ends} line ends were found " + "but no EIC_Codes or names of the connected lines exist in the HTML geo data.") + logger.debug(msg) if is_tieline.all() else logger.warning(msg) + return None + if keep.sum() == 0: + logger.info(f"For {bus=}, all EIC_Codes and names of connected lines are ambiguous. " + "No geo data is dropped at this point.") + keep[(~is_missing).any(axis=1)] = True + access = access.loc[keep] + + this_bus_geo = lgd_bus.loc[slicer[access.col_name, access.identifier, access.bus], :] + if len(this_bus_geo) > 1: + this_bus_geo = this_bus_geo.loc[this_bus_geo.round(2).drop_duplicates().index] + + if len(this_bus_geo) == 1: + return geo_json(this_bus_geo.iloc[0]) + if len(this_bus_geo) == 2: + how_often = pd.Series( + [int((np.isclose(lgd_eic_bus["lat"], this_bus_geo["lat"].iat[i]) + & np.isclose(lgd_eic_bus["lng"], this_bus_geo["lng"].iat[i])).sum()) + for i in range(len(this_bus_geo))], + index=this_bus_geo.index) + if how_often.at[how_often.idxmax()] >= 1: + logger.warning(f"Bus {bus} (name {net.bus.at[bus, 'name']}) was found multiple " + "times in line_geo_data. The first of the most used geo positions " + "is used.") + return geo_json(this_bus_geo.loc[how_often.idxmax()]) + return None + + net.bus.geo = [bus_geo(bus) for bus in net.bus.index] + + +def _extend_index(pivot: pd.DataFrame, col_name: str) -> pd.MultiIndex: + """Prepend a constant ``col_name`` level to a ``(identifier, bus)`` pivot index.""" + frame = pivot.index.to_frame().assign(col_name=col_name).rename( + columns={pivot.index.names[0]: "identifier"}) + return pd.MultiIndex.from_frame(frame.loc[:, ["col_name", "identifier", "bus"]]) + + +def _isin_frame(excerpt: pd.DataFrame, dupl_eics: pd.Series, dupl_names: pd.Series, + side: str) -> pd.DataFrame: + return pd.DataFrame( + {"EIC": excerpt.EIC_Code.isin(dupl_eics).values, + "name": excerpt.name.isin(dupl_names).values}, + index=pd.MultiIndex.from_product([[side], excerpt.index], + names=["bus", "line_index"])) + + +def fill_geo_at_one_sided_branches_without_geo_extent(net: pandapowerNet) -> None: + """Propagate bus geodata across branches that have geodata on only one end.""" + + def availability(net: pandapowerNet) -> dict: + av = {} + with_geo = net.bus.index[~net.bus.geo.isnull()] + av["lines_fbw_tbwo"] = net.line.index[net.line.from_bus.isin(with_geo) + & ~net.line.to_bus.isin(with_geo)] + av["lines_fbwo_tbw"] = net.line.index[~net.line.from_bus.isin(with_geo) + & net.line.to_bus.isin(with_geo)] + av["trafos_hvbw_lvbwo"] = net.trafo.index[net.trafo.hv_bus.isin(with_geo) + & ~net.trafo.lv_bus.isin(with_geo)] + av["trafos_hvbwo_lvbw"] = net.trafo.index[~net.trafo.hv_bus.isin(with_geo) + & net.trafo.lv_bus.isin(with_geo)] + av["n_lines_one_side_geo"] = len(av["lines_fbw_tbwo"]) + len(av["lines_fbwo_tbw"]) + return av + + geo_avail = availability(net) + while geo_avail["n_lines_one_side_geo"]: + for et, bus_w_geo, bus_wo_geo, idx_key in zip( + ["line", "line", "trafo", "trafo"], + ["to_bus", "from_bus", "lv_bus", "hv_bus"], + ["from_bus", "to_bus", "hv_bus", "lv_bus"], + ["lines_fbwo_tbw", "lines_fbw_tbwo", "trafos_hvbwo_lvbw", "trafos_hvbw_lvbwo"]): + net.bus.loc[net[et].loc[geo_avail[idx_key], bus_wo_geo].values, "geo"] = \ + net.bus.loc[net[et].loc[geo_avail[idx_key], bus_w_geo].values, "geo"].values + geo_avail = availability(net) + set_line_geodata_from_bus_geodata(net) diff --git a/pandapower/converter/jao/_schema.py b/pandapower/converter/jao/_schema.py new file mode 100644 index 0000000000..4241ffed57 --- /dev/null +++ b/pandapower/converter/jao/_schema.py @@ -0,0 +1,233 @@ +# -*- coding: utf-8 -*- + +# Copyright (c) 2016-2026 by University of Kassel and Fraunhofer Institute for Energy Economics +# and Energy System Technology (IEE), Kassel. All rights reserved. + +""" +Column resolution for the JAO static grid model sheets. + +The JAO Excel files are maintained largely by hand, so the two-level column headers drift +between releases (a voltage column may live under ``Substation_2`` in one release and under an +``Unnamed:`` top level in another, ``TSO`` may be split into ``TSO 1``/``TSO 2``, unicode Ω/µ/μ +vary, etc.). Instead of hard-coding column tuples, the builders ask a :class:`SheetSchema` for +the column they need; the schema resolves it with a single fuzzy matcher based on a canonical, +accent- and punctuation-insensitive form of the label. +""" + +import re +import difflib +import unicodedata + +import numpy as np +import pandas as pd + + +# --- Level-1 (column) labels the converter looks for ----------------------------------------- +FULL_NAME = "Full Name" +VOLTAGE_LEVEL = "voltage level kv" +LENGTH = "Length_(km)" +RESISTANCE = "Resistance_R(Ω)" +REACTANCE = "Reactance_X(Ω)" + +# --- Level-0 (group) labels ------------------------------------------------------------------ +ELECTRICAL_PARAMETERS = "Electrical Parameters" +PHASE_SHIFT_PROPERTIES = "Phase Shifting Properties" + +# --- Imax column tuples ---------------------------------------------------------------------- +LINE_IMAX = ("Maximum Current Imax (A)", "Fixed") +TAPS = (PHASE_SHIFT_PROPERTIES, "Taps used for RAO") + + +def canon(label: object) -> str: + """Return a canonical form of a header label for fuzzy comparison. + + Lower-cases, strips accents, unifies the micro sign (µ/μ -> u) and drops every character + that is not a letter or digit, so that e.g. ``"Susceptance_B (µS)"`` and + ``"Susceptance_B(μS)"`` collapse to the same token. + """ + s = str(label or "").strip() + s = s.replace("µ", "u").replace("μ", "u") + s = unicodedata.normalize("NFKD", s) + s = "".join(ch for ch in s if not unicodedata.combining(ch)) + s = s.casefold() + return re.sub(r"[^0-9a-z]+", "", s) + + +def _similarity(a: str, b: str) -> float: + return difflib.SequenceMatcher(None, a, b).ratio() + + +def to_numeric(values) -> np.ndarray: + """Coerce string cells with decimal commas / stray whitespace to floats (NaN on failure).""" + return pd.to_numeric( + pd.Series(values).astype(str).str.replace(",", ".", regex=False), + errors="coerce", + ).to_numpy() + + +def numeric_column(df: pd.DataFrame, col: tuple) -> np.ndarray: + """Numeric values of a MultiIndex column, robust to duplicate labels (position based).""" + pos = list(df.columns).index(col) + return to_numeric(df.iloc[:, pos]) + + +class SheetSchema: + """Resolve the columns a single JAO sheet exposes via fuzzy label matching. + + A schema is built once per sheet (``Lines``, ``Tielines`` or ``Transformers``) and shared by + the correction and element-building steps. The generic :meth:`resolve` does the matching; + the named accessors express the converter's intent (voltage column, full-name column, R/X/B + columns, TSO series, ...). + """ + + def __init__(self, df: pd.DataFrame): + self.df = df + + # --- generic matcher --------------------------------------------------------------------- + + def resolve(self, target_label: str, min_ratio: float = 0.55, + required_tokens: list[str] | None = None) -> tuple | None: + """Return the MultiIndex column whose level-1 label best matches ``target_label``. + + ``required_tokens`` (already canonical substrings) are penalised when absent, which + avoids accidental matches between similar labels. Returns ``None`` if nothing clears + ``min_ratio``. + """ + target = canon(target_label) + required = required_tokens or [] + best, best_score = None, -1.0 + for col in self.df.columns: + lvl1 = canon(col[1]) + score = _similarity(lvl1, target) + if required and not all(tok in lvl1 for tok in required): + score -= 0.15 + if score >= min_ratio and score > best_score: + best, best_score = col, score + return best + + def _position(self, col: tuple) -> int: + return list(self.df.columns).index(col) + + def series(self, col: tuple) -> pd.Series: + """Stripped string series of a column, robust to duplicate labels.""" + return self.df.iloc[:, self._position(col)].astype(str).str.strip() + + # --- Lines / Tielines -------------------------------------------------------------------- + + def voltage_col(self) -> tuple | None: + return self.resolve(VOLTAGE_LEVEL, min_ratio=0.45, required_tokens=["volt", "kv"]) + + def fullname_col(self, side: str | None) -> tuple | None: + """``("Substation_1"|"Substation_2", "Full name")`` for lines, ``("Location", ...)`` for + transformers (``side=None``).""" + if side is not None: + top_target, top_element = canon(side), "substation" + else: + top_target, top_element = canon("location"), "location" + lvl1_target = canon("full name") + best, best_score = None, -1.0 + for col in self.df.columns: + top_c, lvl1_c = canon(col[0]), canon(col[1]) + s0 = _similarity(top_c, top_target) + s1 = _similarity(lvl1_c, lvl1_target) + if top_element not in top_c: + s0 -= 0.1 + if "fullname" not in lvl1_c and not ("full" in lvl1_c and "name" in lvl1_c): + s1 -= 0.1 + score = 0.5 * (s0 + s1) + if s0 >= 0.5 and s1 >= 0.6 and score > best_score: + best, best_score = col, score + return best + + def resistance_col(self) -> tuple | None: + return self.resolve("resistance r ohm", min_ratio=0.5, required_tokens=["resistance"]) + + def reactance_col(self) -> tuple | None: + return self.resolve("reactance x ohm", min_ratio=0.5, required_tokens=["reactance"]) + + def susceptance_col(self) -> tuple | None: + return self.resolve("susceptance b us", min_ratio=0.5, + required_tokens=["susceptance", "b", "us"]) + + def line_tso_array(self) -> np.ndarray: + """Line-wise TSO string: prefer a generic ``TSO`` column, else join ``TSO 1``/``TSO 2``.""" + col = self.resolve("TSO", min_ratio=0.6, required_tokens=["tso"]) + if col is not None: + return self.series(col).to_numpy() + c1 = self.resolve("TSO 1", min_ratio=0.6, required_tokens=["tso"]) + c2 = self.resolve("TSO 2", min_ratio=0.6, required_tokens=["tso"]) + if c1 is not None and c2 is not None: + return (self.series(c1) + "/" + self.series(c2)).to_numpy() + if c1 is not None: + return self.series(c1).to_numpy() + if c2 is not None: + return self.series(c2).to_numpy() + return np.array([""] * len(self.df)) + + def tso_series_for_side(self, side: str) -> pd.Series: + """TSO series for one substation side, falling back to a generic ``TSO`` column.""" + target = "TSO 1" if side == "Substation_1" else "TSO 2" + col = self.resolve(target, min_ratio=0.6, required_tokens=["tso"]) + if col is None: + col = self.resolve("TSO", min_ratio=0.6, required_tokens=["tso"]) + if col is not None: + return self.series(col) + return pd.Series([""] * len(self.df), index=self.df.index) + + def string_values(self, target_label: str, tokens: list[str] | None = None, + default: object = "") -> np.ndarray: + col = self.resolve(target_label, min_ratio=0.5, required_tokens=tokens) + if col is None: + return np.array([default] * len(self.df)) + return self.series(col).to_numpy() + + def numeric_values(self, target_label: str, tokens: list[str] | None = None, + default: float = 0.0) -> np.ndarray: + vals = self.string_values(target_label, tokens=tokens, default=str(default)) + return pd.to_numeric( + pd.Series(vals).str.replace(",", ".", regex=False), errors="coerce" + ).fillna(default).to_numpy() + + # --- Transformers ------------------------------------------------------------------------ + + def transformer_voltage_cols(self) -> tuple[tuple | None, tuple | None]: + """Primary and secondary voltage columns, preferring a pair under the same top level.""" + top_target = canon(VOLTAGE_LEVEL) + primaries, secondaries = [], [] + for col in self.df.columns: + top_c, lvl1_c = canon(col[0]), canon(col[1]) + top_ok = _similarity(top_c, top_target) >= 0.45 and "volt" in top_c and "kv" in top_c + if not top_ok: + continue + if _similarity(lvl1_c, canon("primary")) >= 0.7: + primaries.append(col) + if _similarity(lvl1_c, canon("secondary")) >= 0.7: + secondaries.append(col) + for p in primaries: + same_top = [s for s in secondaries if canon(s[0]) == canon(p[0])] + if same_top: + return p, same_top[0] + prim = max(primaries, key=lambda c: _similarity(canon(c[0]), top_target), default=None) + sec = max(secondaries, key=lambda c: _similarity(canon(c[0]), top_target), default=None) + return prim, sec + + def transformer_location_series(self) -> pd.Series: + col = self.fullname_col(None) + if col is None: + raise KeyError("Transformers: could not fuzzy-match the Location / Full Name column.") + return self.series(col) + + def transformer_tso_series(self) -> pd.Series: + """TSO column for transformers, preferring one under a ``Location`` top level.""" + best, best_score = None, -1.0 + for col in self.df.columns: + if "tso" not in canon(col[1]): + continue + score = _similarity(canon(col[1]), canon("tso")) + if "location" in canon(col[0]): + score += 0.2 + if score > best_score: + best, best_score = col, score + if best is None: + return pd.Series([""] * len(self.df), index=self.df.index) + return self.series(best) diff --git a/pandapower/converter/jao/from_jao.py b/pandapower/converter/jao/from_jao.py index b4d3a68182..429ff9a1d9 100644 --- a/pandapower/converter/jao/from_jao.py +++ b/pandapower/converter/jao/from_jao.py @@ -1,34 +1,50 @@ -# -*- coding: utf-8 -*-nt +# -*- coding: utf-8 -*- -# Copyright (c) 2016-2025 by University of Kassel and Fraunhofer Institute for Energy Economics +# Copyright (c) 2016-2026 by University of Kassel and Fraunhofer Institute for Energy Economics # and Energy System Technology (IEE), Kassel. All rights reserved. -from copy import deepcopy +""" +Convert the JAO (Joint Allocation Office) Core static grid model into a pandapower network. + +The heavy lifting is split across sibling modules: + + * :mod:`._schema` resolves the drifting two-level Excel column headers, + * :mod:`._correction` cleans the sheets and unifies the many spellings of each location, + * :mod:`._elements` builds buses, lines and transformers, + * :mod:`._geodata` extracts geodata from the HTML map. + +This module wires those steps together and provides the grid-group utilities used afterwards. +""" + import os import json +import logging from functools import reduce -from typing import Optional, Union + import numpy as np import pandas as pd -from pandas.api.types import is_integer_dtype, is_object_dtype + from pandapower.io_utils import pandapowerNet -from pandapower.create import create_empty_network, create_buses, create_lines_from_parameters, \ - create_transformers_from_parameters +from pandapower.create import create_empty_network from pandapower.topology import create_nxgraph, connected_components from pandapower.plotting import set_line_geodata_from_bus_geodata from pandapower.toolbox import drop_buses, fuse_buses -import logging +from pandapower.converter.jao._correction import data_correction, report_problematic_names +from pandapower.converter.jao._elements import ( + create_buses_from_line_data, create_lines, create_transformers_and_buses, get_bus_idx) +from pandapower.converter.jao._geodata import parse_html_str, add_bus_geo logger = logging.getLogger(__name__) def from_jao(excel_file_path: str, - html_file_path: Optional[str], + html_file_path: str | None, extend_data_for_grid_group_connections: bool, drop_grid_groups_islands: bool = False, apply_data_correction: bool = True, - max_i_ka_fillna: Union[float, int] = 999, + max_i_ka_fillna: float | int = 999, + strict: bool = True, **kwargs) -> pandapowerNet: """ Converts European (Core) EHV grid data provided by JAO (Joint Allocation Office), the @@ -37,109 +53,100 @@ def from_jao(excel_file_path: str, **Data Sources and Availability:** The data are available at the website - `JAO Static Grid Model `_ (November 2024). - There, a map is provided to get an fine overview of the geographical extent and the scope of - the data. These inlcude information about European (Core) lines, tielines, and transformers. + `JAO Static Grid Model `_. There, a map is provided to + get a fine overview of the geographical extent and the scope of the data. These include + information about European (Core) lines, tielines, and transformers. **Limitations:** - No information is available on load or generation. - The data quality with regard to the interconnection of the equipment, the information provided - and the (incomplete) geodata should be considered with caution. + No information is available on load or generation. The data quality with regard to the + interconnection of the equipment, the information provided and the (incomplete) geodata should + be considered with caution. The published data is maintained largely by hand, so the converter + applies robust fuzzy matching to cope with inconsistent column headers and location names. **Features of the converter:** - - **Data Correction:** corrects known data inconsistencies, such as inconsistent spellings and missing necessary information. - - **Geographical Data Parsing:** Parses geographical data from the HTML file to add geolocation information to buses and lines. - - **Grid Group Connections:** Optionally extends the network by connecting islanded grid groups to avoid disconnected components. - - **Data Customization:** Allows for customization through additional parameters to control transformer creation, grid group dropping, and voltage level deviations. - - :param str excel_file_path: - input data including electrical parameters of grids' utilities, stored in multiple sheets - of an excel file - - :param str html_file_path: - input data for geo information. If The converter should be run without geo information, None - can be passed., provided by an html file - - :param bool extend_data_for_grid_group_connections: - if True, connections (additional transformers and merging buses) are created to avoid - islanded grid groups, by default False - - :param Optional[bool] drop_grid_groups_islands: - if True, islanded grid groups will be dropped if their number of buses is below - `min_bus_number` default for this is 6 (default: False) - - :param Optional[bool] apply_data_correction: - _description_ (default: True) - - :param Optional[float|int] max_i_ka_fillna: - value to fill missing values or data of false type in max_i_ka of lines and transformers. - If no value should be set, you can also pass np.nan. (default: 999) - - :param '**'kwargs: following params are available - - :param Optional[bool] minimal_trafo_invention: - applies if extend_data_for_grid_group_connections is True. Then, if minimal_trafo_invention - is True, adding transformers stops when no grid groups is islanded anymore (does not apply - for release version 5 or 6, i.e. it does not care what value is passed to - minimal_trafo_invention). If False, all equally named buses that have different voltage - level and lay in different groups will be connected via additional transformers (default: False) - - :param Optional[int|str] min_bus_number: - Threshold value to decide which small grid groups should be dropped and which large grid - groups should be kept. If all islanded grid groups should be dropped except of the one - largest, set "max". If all grid groups that do not contain a slack element should be - dropped, set "unsupplied". (default: 6) - - :param Optional[float] rel_deviation_threshold_for_trafo_bus_creation: - If the voltage level of transformer locations is far different than the transformer data, - additional buses are created. rel_deviation_threshold_for_trafo_bus_creation defines the - tolerance in which no additional buses are created. (default: 0.2) - - :param Optional[float] log_rel_vn_deviation: - This parameter allows a range below rel_deviation_threshold_for_trafo_bus_creation in which - a warning is logged instead of a creating additional buses. (default: 0.12) - - :return: net created from the jao data - :rtype: pandapowerNet - - :example: + + - **Data Correction:** corrects known data inconsistencies, such as inconsistent spellings and + missing necessary information. + - **Geographical Data Parsing:** Parses geodata from the HTML file to add geolocation + information to buses and lines. + - **Grid Group Connections:** Optionally extends the network by connecting islanded grid + groups to avoid disconnected components. + - **Data Customization:** Allows for customization through additional parameters to control + transformer creation, grid group dropping, and voltage level deviations. + + Parameters: + excel_file_path: input data including electrical parameters of the grid's utilities, stored + in multiple sheets of an Excel file (typically "Lines", "Tielines", "Transformers"). + html_file_path: input data for geo information provided by an HTML file. Pass None to run + the converter without geo information. + extend_data_for_grid_group_connections: if True, connections (additional transformers and + merged buses) are created to avoid islanded grid groups. + drop_grid_groups_islands: if True, islanded grid groups are dropped if their number of + buses is below `min_bus_number` (default 6). (default: False) + apply_data_correction: if True, apply the data-correction routines. (default: True) + max_i_ka_fillna: value to fill missing or invalid max_i_ka of lines and transformers. + Pass np.nan to leave such values unset. (default: 999) + strict: how to handle transformers whose location cannot be matched to any bus (which + happens for hand-maintained data with missing buses or inconsistent naming). If True, + a ValueError is raised listing every affected transformer. If False, those + transformers are skipped (with a warning listing them) and the conversion still + completes; the resulting network is then incomplete and should be used with care. + (default: True) + + Keyword Arguments: + minimal_trafo_invention (bool): applies if extend_data_for_grid_group_connections is True. + If True, adding transformers stops once no grid group is islanded anymore. (default: + False) + min_bus_number (int|str): threshold to decide which small grid groups are dropped. Use + "max" to keep only the largest group, or "unsupplied" to drop groups without a slack + element. (default: 6) + rel_deviation_threshold_for_trafo_bus_creation (float): if a transformer location's voltage + deviates from the transformer data by more than this fraction, an additional bus is + created. (default: 0.2) + log_rel_vn_deviation (float): range below rel_deviation_threshold_for_trafo_bus_creation in + which a warning is logged instead of creating additional buses. (default: 0.12) + sn_mva (float): system base apparent power (MVA) for the pandapower net. + + Returns: + pandapowerNet: the network created from the JAO data. + + Example: >>> from pathlib import Path >>> import os - >>> from pandapower.converter.jao.from_jao import from_jao - >>> net = from_jao() + >>> from pandapower.converter.jao import from_jao >>> home = str(Path.home()) - >>> # assume that the files are located at your desktop: - >>> excel_file_path = os.path.join(home, "desktop", "202409_Core Static Grid Mode_6th release") + >>> excel_file_path = os.path.join(home, "desktop", "202409_Core Static Grid Model.xlsx") >>> html_file_path = os.path.join(home, "desktop", "2024-09-13_Core_SGM_publication.html") >>> net = from_jao(excel_file_path, html_file_path, True, drop_grid_groups_islands=True) """ - # --- read data data = pd.read_excel(excel_file_path, sheet_name=None, header=[0, 1]) + report_problematic_names(data) if html_file_path is not None: - with open(html_file_path, mode='r', encoding=kwargs.get("encoding", "utf-8")) as f: + with open(html_file_path, mode="r", encoding=kwargs.get("encoding", "utf-8")) as f: html_str = f.read() else: html_str = "" - # --- manipulate data / data corrections + # --- correct data if apply_data_correction: - html_str = _data_correction(data, html_str, max_i_ka_fillna) + html_str = data_correction(data, html_str, max_i_ka_fillna) - # --- parse html_str to line_geo_data + # --- parse geodata from html line_geo_data = None if html_str: try: - line_geo_data = _parse_html_str(html_str) + line_geo_data = parse_html_str(html_str) except (json.JSONDecodeError, KeyError, AssertionError) as e: logger.error(f"html data were ignored due to this error:\n{e}") # --- create the pandapower net - net = create_empty_network(name=os.path.splitext(os.path.basename(excel_file_path))[0], - **{key: val for key, val in kwargs.items() if key == "sn_mva"}) - _create_buses_from_line_data(net, data) - _create_lines(net, data, max_i_ka_fillna) - _create_transformers_and_buses(net, data, **kwargs) + net = create_empty_network( + name=os.path.splitext(os.path.basename(excel_file_path))[0], + **{key: val for key, val in kwargs.items() if key == "sn_mva"}) + create_buses_from_line_data(net, data) + create_lines(net, data, max_i_ka_fillna) + create_transformers_and_buses(net, data, strict=strict, **kwargs) # --- invent connections between grid groups if extend_data_for_grid_group_connections: @@ -149,299 +156,74 @@ def from_jao(excel_file_path: str, if drop_grid_groups_islands: drop_islanded_grid_groups(net, kwargs.get("min_bus_number", 6)) - # --- add geo data to buses and lines + # --- add geodata to buses and lines if line_geo_data is not None: - _add_bus_geo(net, line_geo_data) + add_bus_geo(net, line_geo_data) set_line_geodata_from_bus_geodata(net) return net -# --- secondary functions -------------------------------------------------------------------------- +# ================================================================================================== +# Grid groups +# ================================================================================================== -def _data_correction( - data: dict[str, pd.DataFrame], - html_str: Optional[str], - max_i_ka_fillna: Union[float, int]) -> Optional[str]: - """ - Corrects input data in particular with regard to obvious weaknesses in the data provided, - such as inconsistent spellings and missing necessary information - - :param dict[str, pd.DataFrame] data: data provided by the excel file which will be corrected - :param str|None html_str: data provided by the html file which will be corrected - :param float|int max_i_ka_fillna: value to fill missing values or data of false type in max_i_ka of lines and transformers. - If no value should be set, you can also pass np.nan. +def get_grid_groups(net: pandapowerNet, **kwargs) -> pd.DataFrame: + """Return the connected components (grid groups) of the network and their bus counts.""" + notravbuses = {"notravbuses": kwargs.pop("notravbuses")} if "notravbuses" in kwargs else {} + grid_group_buses = list(connected_components(create_nxgraph(net, **kwargs), **notravbuses)) + grid_groups = pd.DataFrame({"buses": grid_group_buses}) + grid_groups["n_buses"] = grid_groups["buses"].apply(len) + return grid_groups - :return: corrected html_str - :rtype: str - """ - # old name -> new name - rename_locnames = [("PSTMIKULOWA", "PST MIKULOWA"), - ("Chelm", "CHELM"), - ("OLSZTYN-MATK", "OLSZTYN-MATKI"), - ("OLSZTYN-MATKII", "OLSZTYN-MATKI"), - ("STANISLAWOW", "Stanislawow"), - ("VIERRADEN", "Vierraden")] - - # --- Line and Tieline data --------------------------- - for key in ["Lines", "Tielines"]: - - # --- correct column names - cols = data[key].columns.to_frame().reset_index(drop=True) - cols.loc[cols[1] == "Voltage_level(kV)", 0] = None - cols.loc[cols[1] == "Comment", 0] = None - cols.loc[cols[0].str.startswith("Unnamed:").astype(bool), 0] = None - cols.loc[cols[1] == "Length_(km)", 0] = "Electrical Parameters" # might be wrong in - # Tielines otherwise - data[key].columns = pd.MultiIndex.from_arrays(cols.values.T) - - # --- correct comma separation and cast to floats - data[key][("Maximum Current Imax (A)", "Fixed")] = \ - data[key][("Maximum Current Imax (A)", "Fixed")].replace( - "\xa0", max_i_ka_fillna*1e3).replace( - "-", max_i_ka_fillna*1e3).replace(" ", max_i_ka_fillna*1e3) - col_names = [("Electrical Parameters", col_level1) for col_level1 in [ - "Length_(km)", "Resistance_R(Ω)", "Reactance_X(Ω)", "Susceptance_B(μS)", - "Length_(km)"]] + [("Maximum Current Imax (A)", "Fixed")] - _float_col_comma_correction(data, key, col_names) - - # --- consolidate to one way of name capitalization - for loc_name in [(None, "NE_name"), ("Substation_1", "Full_name"), - ("Substation_2", "Full_name")]: - data[key].loc[:, loc_name] = data[key].loc[:, loc_name].str.strip().apply( - _multi_str_repl, repl=rename_locnames) - html_str = _multi_str_repl(html_str, rename_locnames) - - # --- Transformer data -------------------------------- - key = "Transformers" - - # --- fix Locations - loc_name = ("Location", "Full Name") - data[key].loc[:, loc_name] = data[key].loc[:, loc_name].str.strip().apply( - _multi_str_repl, repl=rename_locnames) - - # --- fix data in nonnull_taps - taps = data[key].loc[:, ("Phase Shifting Properties", "Taps used for RAO")].fillna("").astype( - str).str.replace(" ", "") - nonnull = taps.apply(len).astype(bool) - nonnull_taps = taps.loc[nonnull] - surrounded = nonnull_taps.str.startswith("<") & nonnull_taps.str.endswith(">") - nonnull_taps.loc[surrounded] = nonnull_taps.loc[surrounded].str[1:-1] - slash_sep = (~nonnull_taps.str.contains(";")) & nonnull_taps.str.contains("/") - nonnull_taps.loc[slash_sep] = nonnull_taps.loc[slash_sep].str.replace("/", ";") - nonnull_taps.loc[nonnull_taps == "0"] = "0;0" - data[key].loc[nonnull, ("Phase Shifting Properties", "Taps used for RAO")] = nonnull_taps - data[key].loc[~nonnull, ("Phase Shifting Properties", "Taps used for RAO")] = "0;0" - - # --- phase shifter with double info - cols = ["Phase Regulation δu (%)", "Angle Regulation δu (%)"] - for col in cols: - if is_object_dtype(data[key].loc[:, ("Phase Shifting Properties", col)]): - tr_double = data[key].index[data[key].loc[:, ( - "Phase Shifting Properties", col)].str.contains("/").fillna(0).astype(bool)] - data[key].loc[tr_double, ("Phase Shifting Properties", col)] = data[key].loc[ - tr_double, ("Phase Shifting Properties", col)].str.split("/", expand=True)[ - 1].str.replace(",", ".").astype(float).values # take second info and correct - # separation: , -> . - - return html_str - - -def _parse_html_str(html_str: str) -> pd.DataFrame: - """ - Converts ths geodata from the html file (information hidden in the string), from Lines in - particular, to a DataFrame that can be used later in _add_bus_geo() - :param str html_str: html file that includes geodata information +def drop_islanded_grid_groups(net: pandapowerNet, min_bus_number: int | str, **kwargs) -> None: + """Drop islanded grid groups by size or supply condition. - :return: extracted geodata for a later and easy use - :rtype: pd.DataFrame - """ - def _filter_name(st: str) -> str: - name_start = "NE name: " - name_end = "" - pos0 = st.find(name_start) + len(name_start) - pos1 = st.find(name_end, pos0) - assert pos0 >= 0 - assert pos1 >= len(name_start) - return st[pos0:pos1] - - json_start_str = '') - json_str = html_str[json_start_pos:(json_start_pos+json_end_pos)] - geo_data = json.loads(json_str) - geo_data = geo_data["x"]["calls"] - methods_pos = pd.Series({item["method"]: i for i, item in enumerate(geo_data)}) - polylines = geo_data[methods_pos.at["addPolylines"]]["args"] - EIC_start = "EIC Code: " - if len(polylines[6]) != len(polylines[0]): - raise AssertionError("The lists of EIC Code data and geo data are not of the same length.") - line_EIC = [polylines[6][i][polylines[6][i].find(EIC_start)+len(EIC_start):] for i in range( - len(polylines[6]))] - line_name = [_filter_name(polylines[6][i]) for i in range(len(polylines[6]))] - line_geo_data = pd.concat([_lng_lat_to_df(polylines[0][i][0][0], line_EIC[i], line_name[i]) for - i in range(len(polylines[0]))], ignore_index=True) - - # remove trailing whitespaces - for col in ["EIC_Code", "name"]: - line_geo_data[col] = line_geo_data[col].str.strip() - - return line_geo_data - - -def _create_buses_from_line_data(net: pandapowerNet, data: dict[str, pd.DataFrame]) -> None: - """Creates buses to the pandapower net using information from the lines and tielines sheets - (excel file). - - :param pandapowerNet net: net to be filled by buses - :param dict[str, pd.DataFrame data: data provided by the excel file which will be corrected - """ - bus_df_empty = pd.DataFrame({"name": str(), "vn_kv": float(), "TSO": str()}, index=[]) - bus_df = deepcopy(bus_df_empty) - for key in ["Lines", "Tielines"]: - for subst in ['Substation_1', 'Substation_2']: - data_col_tuples = [(subst, "Full_name"), (None, "Voltage_level(kV)"), (None, "TSO")] - to_add = data[key].loc[:, data_col_tuples].set_axis(bus_df.columns, axis="columns") - if len(bus_df): - bus_df = pd.concat([bus_df, to_add]) - else: - bus_df = to_add - bus_df = _drop_duplicates_and_join_TSO(bus_df) - new_bus_idx = create_buses( - net, len(bus_df), vn_kv=bus_df.vn_kv, name=bus_df.name, zone=bus_df.TSO) - assert np.allclose(new_bus_idx, bus_df.index) - - -def _create_lines( - net: pandapowerNet, - data: dict[str, pd.DataFrame], - max_i_ka_fillna: Union[float, int]) -> None: - """ - Creates lines to the pandapower net using information from the lines and tielines sheets (excel file). - - :param pandapowerNet net: net to be filled by buses - :param dict[str, pd.DataFrame] data: data provided by the excel file which will be corrected - :param float|int max_i_ka_fillna: value to fill missing values or data of false type in max_i_ka of lines and transformers. - If no value should be set, you can also pass np.nan. + ``min_bus_number`` may be an int (drop groups smaller than this), ``"max"`` (keep only the + largest group) or ``"unsupplied"`` (drop groups without any slack element). """ + def to_drop_by_size(): + return grid_groups.loc[grid_groups["n_buses"] < min_bus_number] - bus_idx = _get_bus_idx(net) - - for key in ["Lines", "Tielines"]: - length_km = data[key][("Electrical Parameters", "Length_(km)")].values - zero_length = np.isclose(length_km, 0) - no_length = np.isnan(length_km) - if sum(zero_length) or sum(no_length): - logger.warning(f"According to given data, {sum(zero_length)} {key.lower()} have zero " - f"length and {sum(zero_length)} {key.lower()} have no length data. " - "Both types of wrong data are replaced by 1 km.") - length_km[zero_length | no_length] = 1 - vn_kvs = data[key].loc[:, (None, "Voltage_level(kV)")].values - - _ = create_lines_from_parameters( - net, - bus_idx.loc[list(tuple(zip(data[key].loc[:, ("Substation_1", "Full_name")].values, - vn_kvs)))].values, - bus_idx.loc[list(tuple(zip(data[key].loc[:, ("Substation_2", "Full_name")].values, - vn_kvs)))].values, - length_km, - data[key][("Electrical Parameters", "Resistance_R(Ω)")].values / length_km, - data[key][("Electrical Parameters", "Reactance_X(Ω)")].values / length_km, - data[key][("Electrical Parameters", "Susceptance_B(μS)")].values / length_km, - data[key][("Maximum Current Imax (A)", "Fixed")].fillna( - max_i_ka_fillna*1e3).values / 1e3, - name=data[key].xs("NE_name", level=1, axis=1).values[:, 0], - EIC_Code=data[key].xs("EIC_Code", level=1, axis=1).values[:, 0], - TSO=data[key].xs("TSO", level=1, axis=1).values[:, 0], - Comment=data[key].xs("Comment", level=1, axis=1).values[:, 0], - Tieline=key == "Tielines", - ) - - -def _create_transformers_and_buses( - net: pandapowerNet, data: dict[str, pd.DataFrame], **kwargs) -> None: - """ - Creates transformers to the pandapower net using information from the transformers sheet (excel file). + grid_groups = get_grid_groups(net, **kwargs) - :param pandapowerNet net: net to be filled by buses - :param dict[str, pd.DataFrame] data: data provided by the excel file which will be corrected - """ + if min_bus_number == "unsupplied": + slack_buses = set(net.ext_grid.loc[net.ext_grid.in_service, "bus"]) | \ + set(net.gen.loc[net.gen.in_service & net.gen.slack, "bus"]) + grid_groups_to_drop = grid_groups.loc[ + ~grid_groups.buses.apply(lambda x: not x.isdisjoint(slack_buses))] + elif min_bus_number == "max": + min_bus_number = grid_groups["n_buses"].max() + grid_groups_to_drop = to_drop_by_size() + elif isinstance(min_bus_number, int): + grid_groups_to_drop = to_drop_by_size() + else: + raise NotImplementedError( + f"{min_bus_number=} is not implemented. Use an int, 'max', or 'unsupplied' instead.") - # --- data preparations - key = "Transformers" - bus_idx = _get_bus_idx(net) - vn_hv_kv, vn_lv_kv = _get_transformer_voltages(data, bus_idx) - trafo_connections = _allocate_trafos_to_buses_and_create_buses( - net, data, bus_idx, vn_hv_kv, vn_lv_kv, **kwargs) - max_i_a = data[key].loc[:, ("Maximum Current Imax (A) primary", "Fixed")] - empty_i_idx = max_i_a.index[max_i_a.isnull()] - max_i_a.loc[empty_i_idx] = data[key].loc[empty_i_idx, ( - "Maximum Current Imax (A) primary", "Max")].values - sn_mva = np.sqrt(3) * max_i_a * vn_hv_kv / 1e3 - z_pu = vn_lv_kv**2 / sn_mva - rk = data[key].xs("Resistance_R(Ω)", level=1, axis=1).values[:, 0] / z_pu - xk = data[key].xs("Reactance_X(Ω)", level=1, axis=1).values[:, 0] / z_pu - b0 = data[key].xs("Susceptance_B (µS)", level=1, axis=1).values[:, 0] * 1e-6 * z_pu - g0 = data[key].xs("Conductance_G (µS)", level=1, axis=1).values[:, 0] * 1e-6 * z_pu - zk = np.sqrt(rk**2 + xk**2) - vk_percent = np.sign(xk) * zk * 100 - vkr_percent = rk * 100 - pfe_kw = g0 * sn_mva * 1e3 - i0_percent = 100 * np.sqrt(b0**2 + g0**2) * net.sn_mva / sn_mva - taps = data[key].loc[:, ("Phase Shifting Properties", "Taps used for RAO")].str.split( - ";", expand=True).astype(int).set_axis(["tap_min", "tap_max"], axis=1) - - du = _get_float_column(data[key], ("Phase Shifting Properties", "Phase Regulation δu (%)")) - dphi = _get_float_column(data[key], ("Phase Shifting Properties", "Angle Regulation δu (%)")) - phase_shifter = np.isclose(du, 0) & (~np.isclose(dphi, 0)) # Symmetrical/Asymmetrical not - # considered - - _ = create_transformers_from_parameters( - net, - trafo_connections.hv_bus.values, - trafo_connections.lv_bus.values, - sn_mva, - vn_hv_kv, - vn_lv_kv, - vkr_percent, - vk_percent, - pfe_kw, - i0_percent, - shift_degree=data[key].xs("Theta θ (°)", level=1, axis=1).values[:, 0], - tap_pos=0, - tap_neutral=0, - tap_side="lv", - tap_min=taps["tap_min"].values, - tap_max=taps["tap_max"].values, - tap_phase_shifter=phase_shifter, - tap_step_percent=du, - tap_step_degree=dphi, - name=data[key].loc[:, ("Location", "Full Name")].str.strip().values, - EIC_Code=data[key].xs("EIC_Code", level=1, axis=1).values[:, 0], - TSO=data[key].xs("TSO", level=1, axis=1).values[:, 0], - Comment=data[key].xs("Comment", level=1, axis=1).replace("\xa0", "").values[:, 0], - ) + buses_to_drop = reduce(set.union, grid_groups_to_drop.buses) + drop_buses(net, buses_to_drop) + logger.info(f"drop_islanded_grid_groups() drops {len(grid_groups_to_drop)} grid groups with a " + f"total of {grid_groups_to_drop.n_buses.sum()} buses.") def _invent_connections_between_grid_groups( net: pandapowerNet, minimal_trafo_invention: bool = False, **kwargs) -> None: """ - Adds connections between islanded grid groups via: + Add connections between islanded grid groups by: - - adding transformers between equally named buses that have different voltage level and lay in different groups - - merge buses of same voltage level, different grid groups and equal name base - - fuse buses that are close to each other + - adding transformers between equally named buses of different voltage level in different + groups, + - merging buses of the same voltage level, equal name base and different grid groups, + - fusing selected known close-by bus pairs. :param pandapowerNet net: net to be manipulated - :param Optional[bool] minimal_trafo_invention: if True, adding transformers stops when no grid groups is islanded anymore (does not apply - for release version 5 or 6, i.e. it does not care what value is passed to - minimal_trafo_invention). If False, all equally named buses that have different voltage - level and lay in different groups will be connected via additional transformers, - (default: False) + :param bool minimal_trafo_invention: if True, adding transformers stops once no grid group is + islanded anymore. """ grid_groups = get_grid_groups(net) - bus_idx = _get_bus_idx(net) + bus_idx = get_bus_idx(net) bus_grid_groups = pd.concat([pd.Series(group, index=buses) for group, buses in zip( grid_groups.index, grid_groups.buses)]).sort_index() @@ -452,8 +234,8 @@ def _invent_connections_between_grid_groups( [location_names.values, bus_idx.index.get_level_values(1).to_numpy()], names=bus_idx.index.names) - # --- add Transformers between equally named buses that have different voltage level and lay in - # --- different groups + # --- 1) add transformers between equally named buses of different voltage level in different + # groups connected_vn_kvs_by_trafos = pd.DataFrame({ "hv": net.bus.vn_kv.loc[net.trafo.hv_bus.values].values, "lv": net.bus.vn_kv.loc[net.trafo.lv_bus.values].values, @@ -462,7 +244,7 @@ def _invent_connections_between_grid_groups( for location_name in dupl_location_names: if minimal_trafo_invention and len(bus_grid_groups.unique()) <= 1: - break # break with regard to minimal_trafo_invention + break grid_groups_at_location = bus_grid_groups.loc[bus_idx.loc[location_name].values] grid_groups_at_location = grid_groups_at_location.drop_duplicates() if len(grid_groups_at_location) < 2: @@ -474,36 +256,37 @@ def _invent_connections_between_grid_groups( TSO = net.bus.zone.at[grid_groups_at_location.index[0]] vn_kvs = net.bus.vn_kv.loc[grid_groups_at_location.index].sort_values(ascending=False) try: - trafos_connecting_same_voltage_levels = \ - connected_vn_kvs_by_trafos.loc[tuple(vn_kvs)] + trafos_same_vn = connected_vn_kvs_by_trafos.loc[tuple(vn_kvs)] except KeyError: logger.info(f"For location {location_name}, no transformer data can be reused since " - f"no transformer connects {vn_kvs.sort_values(ascending=False).iat[0]} kV " - f"and {vn_kvs.sort_values(ascending=False).iat[1]} kV.") + f"no transformer connects {vn_kvs.iat[0]} kV and {vn_kvs.iat[1]} kV.") continue - trafos_of_same_TSO = trafos_connecting_same_voltage_levels.loc[(net.bus.zone.loc[ - net.trafo.hv_bus.loc[trafos_connecting_same_voltage_levels.values.flatten( - )].values] == TSO).values].values.flatten() - - # from which trafo parameters are copied: - tr_to_be_copied = trafos_of_same_TSO[0] if len(trafos_of_same_TSO) else \ - trafos_connecting_same_voltage_levels.values.flatten()[0] - - # copy transformer data - duplicated_row = net.trafo.loc[[tr_to_be_copied]].copy() - duplicated_row.index = [net.trafo.index.max() + 1] # adjust index - duplicated_row.hv_bus = vn_kvs.index[0] # adjust hv_bus, lv_bus - duplicated_row.lv_bus = vn_kvs.index[1] # adjust hv_bus, lv_bus + trafos_same_tso = trafos_same_vn.loc[(net.bus.zone.loc[ + net.trafo.hv_bus.loc[trafos_same_vn.values.flatten()].values] + == TSO).values].values.flatten() + + # choose the transformer to copy parameters from + tr_to_copy = trafos_same_tso[0] if len(trafos_same_tso) else \ + trafos_same_vn.values.flatten()[0] + + duplicated_row = net.trafo.loc[[tr_to_copy]].copy() + duplicated_row.index = [net.trafo.index.max() + 1] + duplicated_row.hv_bus = vn_kvs.index[0] + duplicated_row.lv_bus = vn_kvs.index[1] duplicated_row.name = "additional transformer to connect the grid" net.trafo = pd.concat([net.trafo, duplicated_row]) bus_grid_groups.loc[bus_grid_groups == grid_groups_at_location.iat[1]] = \ grid_groups_at_location.iat[0] - # --- merge buses of same voltage level, different grid groups and equal name base + # --- 2) merge buses of same voltage level, different grid groups and equal name base bus_name_splits = net.bus.name.str.split(r"[ -/]+", expand=True) buses_with_single_base = net.bus.name.loc[(~bus_name_splits.isnull()).sum(axis=1) == 1] for idx, name_base in buses_with_single_base.items(): + # a previous fuse in this loop may already have removed this bus; buses invented after + # bus_grid_groups was built (e.g. duplicated same-bus trafo buses) are not tracked in it + if idx not in net.bus.index or idx not in bus_grid_groups.index: + continue same_name_base = net.bus.drop(idx).name.str.contains(name_base) if not any(same_name_base): continue @@ -519,7 +302,7 @@ def _invent_connections_between_grid_groups( is_fuse_candidate].unique())] = grid_groups_at_location.iat[0] bus_grid_groups = bus_grid_groups.drop(to_fuse.index) - # --- fuse buses that are close to each other + # --- 3) fuse buses that are close to each other (known cases) for name1, name2 in [("CROISIERE", "BOLLENE (POSTE RESEAU)"), ("CAEN", "DRONNIERE (LA)"), ("TRINITE-VICTOR", "MENTON/TRINITE VICTOR")]: @@ -531,476 +314,3 @@ def _invent_connections_between_grid_groups( else: logger.info("Buses of the following names were intended to be fused but were not found." f"\n'{name1}' and '{name2}'") - - -def drop_islanded_grid_groups( - net: pandapowerNet, - min_bus_number: Union[int, str], - **kwargs) -> None: - """ - Drops grid groups that are islanded and include a number of buses below min_bus_number. - - :param panadpowerNet net: net in which islanded grid groups will be dropped - :param Optional[int|str] min_bus_number: Threshold value to decide which small grid groups should be dropped and which large grid - groups should be kept. If all islanded grid groups should be dropped except of the one - largest, set "max". If all grid groups that do not contain a slack element should be - dropped, set "unsupplied". - """ - def _grid_groups_to_drop_by_min_bus_number(): - return grid_groups.loc[grid_groups["n_buses"] < min_bus_number] - - grid_groups = get_grid_groups(net, **kwargs) - - if min_bus_number == "unsupplied": - slack_buses = set(net.ext_grid.loc[net.ext_grid.in_service, "bus"]) | \ - set(net.gen.loc[net.gen.in_service & net.gen.slack, "bus"]) - grid_groups_to_drop = grid_groups.loc[~grid_groups.buses.apply( - lambda x: not x.isdisjoint(slack_buses))] - - elif min_bus_number == "max": - min_bus_number = grid_groups["n_buses"].max() - grid_groups_to_drop = _grid_groups_to_drop_by_min_bus_number() - - elif isinstance(min_bus_number, int): - grid_groups_to_drop = _grid_groups_to_drop_by_min_bus_number() - - else: - raise NotImplementedError( - f"{min_bus_number=} is not implemented. Use an int, 'max', or 'unsupplied' instead.") - - buses_to_drop = reduce(set.union, grid_groups_to_drop.buses) - drop_buses(net, buses_to_drop) - logger.info(f"drop_islanded_grid_groups() drops {len(grid_groups_to_drop)} grid groups with a " - f"total of {grid_groups_to_drop.n_buses.sum()} buses.") - - -def _add_bus_geo(net: pandapowerNet, line_geo_data: pd.DataFrame) -> None: - """Adds geodata to the buses. The function needs to handle cases where line_geo_data does not - include no or multiple geodata per bus. Primarly, the geodata are allocate via EIC Code names, - if ambigous, names are considered. - - :param pandapowerNet net: net in which geodata are added to the buses - :param pd.DataFrame: line_geo_data: Converted geodata from the html file - """ - iSl = pd.IndexSlice - lgd_EIC_bus = line_geo_data.pivot_table(values="value", index=["EIC_Code", "bus"], - columns="geo_dim") - lgd_name_bus = line_geo_data.pivot_table(values="value", index=["name", "bus"], - columns="geo_dim") - lgd_EIC_bus_idx_extended = pd.MultiIndex.from_frame(lgd_EIC_bus.index.to_frame().assign( - **dict(col_name="EIC_Code")).rename(columns=dict(EIC_Code="identifier")).loc[ - :, ["col_name", "identifier", "bus"]]) - lgd_name_bus_idx_extended = pd.MultiIndex.from_frame(lgd_name_bus.index.to_frame().assign( - **dict(col_name="name")).rename(columns=dict(name="identifier")).loc[ - :, ["col_name", "identifier", "bus"]]) - lgd_bus = pd.concat([lgd_EIC_bus.set_axis(lgd_EIC_bus_idx_extended), - lgd_name_bus.set_axis(lgd_name_bus_idx_extended)]) - dupl_EICs = net.line.EIC_Code.loc[net.line.EIC_Code.duplicated()] - dupl_names = net.line.name.loc[net.line.name.duplicated()] - - def _geo_json_str(this_bus_geo: pd.Series) -> str: - return f'{{"coordinates": [{this_bus_geo.at["lng"]}, {this_bus_geo.at["lat"]}], "type": "Point"}}' - - def _add_bus_geo_inner(bus: int) -> Optional[str]: - from_bus_line_excerpt = net.line.loc[net.line.from_bus == - bus, ["EIC_Code", "name", "Tieline"]] - to_bus_line_excerpt = net.line.loc[net.line.to_bus == bus, ["EIC_Code", "name", "Tieline"]] - line_excerpt = pd.concat([from_bus_line_excerpt, to_bus_line_excerpt]) - n_connected_line_ends = len(line_excerpt) - if n_connected_line_ends == 0: - logger.error( - f"Bus {bus} (name {net.bus.at[bus, 'name']}) is not found in line_geo_data.") - return None - is_dupl = pd.concat([ - pd.DataFrame({"EIC": from_bus_line_excerpt.EIC_Code.isin(dupl_EICs).values, - "name": from_bus_line_excerpt.name.isin(dupl_names).values}, - index=pd.MultiIndex.from_product([["from"], from_bus_line_excerpt.index], - names=["bus", "line_index"])), - pd.DataFrame({"EIC": to_bus_line_excerpt.EIC_Code.isin(dupl_EICs).values, - "name": to_bus_line_excerpt.name.isin(dupl_names).values}, - index=pd.MultiIndex.from_product([["to"], to_bus_line_excerpt.index], - names=["bus", "line_index"])) - ]) - is_missing = pd.DataFrame({ - "EIC": ~line_excerpt.EIC_Code.isin( - lgd_bus.loc["EIC_Code"].index.get_level_values("identifier")), - "name": ~line_excerpt.name.isin( - lgd_bus.loc["name"].index.get_level_values("identifier")) - }).set_axis(is_dupl.index, axis=0) - is_tieline = pd.Series(net.line.loc[is_dupl.index.get_level_values("line_index"), - "Tieline"].values, index=is_dupl.index) - - # --- construct access_vals, i.e. values to take line geo data from lgd_bus - # --- if not duplicated, take "EIC_Code". Otherwise and if not dupl, take "name". - # --- Otherwise ignore. Do it for both from and to bus - access_vals = pd.DataFrame({ - "col_name": "EIC_Code", - "identifier": line_excerpt.EIC_Code.values, - "bus": is_dupl.index.get_level_values("bus").values - }) # default is EIC_Code - take_from_name = ((is_dupl.EIC | is_missing.EIC) & ( - ~is_dupl.name & ~is_missing.name)).values - access_vals.loc[take_from_name, "col_name"] = "name" - access_vals.loc[take_from_name, "identifier"] = line_excerpt.name.loc[take_from_name].values - keep = (~(is_dupl | is_missing)).any(axis=1).values - if np.all(is_missing): - log_msg = (f"For bus {bus} (name {net.bus.at[bus, 'name']}), {n_connected_line_ends} " - "were found but no EIC_Codes or names of corresponding lines were found ." - "in the geo data from the html file.") - if is_tieline.all(): - logger.debug(log_msg) - else: - logger.warning(log_msg) - return None - elif sum(keep) == 0: - logger.info(f"For {bus=}, all EIC_Codes and names of connected lines are ambiguous. " - "No geo data is dropped at this point.") - keep[(~is_missing).any(axis=1)] = True - access_vals = access_vals.loc[keep] - - # --- get this_bus_geo from EIC_Code or name with regard to access_vals - this_bus_geo = lgd_bus.loc[iSl[ - access_vals.col_name, access_vals.identifier, access_vals.bus], :] - - if len(this_bus_geo) > 1: - # reduce similar/equal lines - this_bus_geo = this_bus_geo.loc[this_bus_geo.round(2).drop_duplicates().index] - - # --- return geo_json_str - len_this_bus_geo = len(this_bus_geo) - if len_this_bus_geo == 1: - return _geo_json_str(this_bus_geo.iloc[0]) - elif len_this_bus_geo == 2: - how_often = pd.Series( - [sum(np.isclose(lgd_EIC_bus["lat"], this_bus_geo["lat"].iat[i]) & - np.isclose(lgd_EIC_bus["lng"], this_bus_geo["lng"].iat[i])) for i in - range(len_this_bus_geo)], index=this_bus_geo.index) - if how_often.at[how_often.idxmax()] >= 1: - logger.warning(f"Bus {bus} (name {net.bus.at[bus, 'name']}) was found multiple times" - " in line_geo_data. No value exists more often than others. " - "The first of most used geo positions is used.") - return _geo_json_str(this_bus_geo.loc[how_often.idxmax()]) - - net.bus.geo = [_add_bus_geo_inner(bus) for bus in net.bus.index] - - -# --- tertiary functions --------------------------------------------------------------------------- - -def _float_col_comma_correction(data: dict[str, pd.DataFrame], key: str, col_names: list): - for col_name in col_names: - data[key][col_name] = pd.to_numeric(data[key][col_name].astype(str).str.replace( - ",", "."), errors="coerce") - - -def _get_transformer_voltages( - data: dict[str, pd.DataFrame], bus_idx: pd.Series) -> tuple[np.ndarray, np.ndarray]: - - key = "Transformers" - vn = data[key].loc[:, [("Voltage_level(kV)", "Primary"), - ("Voltage_level(kV)", "Secondary")]].values - vn_hv_kv = np.max(vn, axis=1) - vn_lv_kv = np.min(vn, axis=1) - if is_integer_dtype(list(bus_idx.index.dtypes)[1]): - vn_hv_kv = vn_hv_kv.astype(int) - vn_lv_kv = vn_lv_kv.astype(int) - - return vn_hv_kv, vn_lv_kv - - -def _allocate_trafos_to_buses_and_create_buses( - net: pandapowerNet, data: dict[str, pd.DataFrame], bus_idx: pd.Series, - vn_hv_kv: np.ndarray, vn_lv_kv: np.ndarray, - rel_deviation_threshold_for_trafo_bus_creation: float = 0.2, - log_rel_vn_deviation: float = 0.12, **kwargs) -> pd.DataFrame: - """Provides a DataFrame of data to allocate transformers to the buses according to their - location names. If locations of transformers do not exist due to the data of the lines and - tielines sheets, additional buses are created. If locations exist but have a far different - voltage level than the transformer, either a warning is logged or additional buses are created - according to rel_deviation_threshold_for_trafo_bus_creation and log_rel_vn_deviation. - - :param pandapowerNet net: pandapower net - :param dict[str, pd.DataFrame] data: _description_ - :param pd.Series bus_idx: Series of indices and corresponding location names and voltage levels in the MultiIndex of - the Series - :param np.ndarray vn_hv_kv: nominal voltages of the hv side of the transformers - :param np.ndarray vn_lv_kv: Nominal voltages of the lv side of the transformers - :param Optional[float] rel_deviation_threshold_for_trafo_bus_creation: If the voltage level of transformer locations - is far different than the transformer data, - additional buses are created. rel_deviation_threshold_for_trafo_bus_creation defines the - tolerance in which no additional buses are created. (default: 0.2) - :param Optional[float] log_rel_vn_deviation: This parameter allows a range below rel_deviation_threshold_for_trafo_bus_creation in which - a warning is logged instead of a creating additional buses. (default: 0.12) - - :rtype: pd.DataFrame - :return: information to which bus the trafos should be connected to. Columns are - ["name", "hv_bus", "lv_bus", "vn_hv_kv", "vn_lv_kv", ...] - """ - - if rel_deviation_threshold_for_trafo_bus_creation < log_rel_vn_deviation: - logger.warning( - f"Given parameters violates the ineqation " - f"{rel_deviation_threshold_for_trafo_bus_creation=} >= {log_rel_vn_deviation=}. " - f"Therefore, rel_deviation_threshold_for_trafo_bus_creation={log_rel_vn_deviation} " - "is assumed.") - rel_deviation_threshold_for_trafo_bus_creation = log_rel_vn_deviation - - key = "Transformers" - bus_location_names = set(net.bus.name) - trafo_bus_names = data[key].loc[:, ("Location", "Full Name")] - trafo_location_names = _find_trafo_locations(trafo_bus_names, bus_location_names) - - # --- construct DataFrame trafo_connections including all information on trafo allocation to - # --- buses - empties = -1*np.ones(len(vn_hv_kv), dtype=int) - trafo_connections = pd.DataFrame({ - "name": trafo_location_names, - "hv_bus": empties, - "lv_bus": empties, - "vn_hv_kv": vn_hv_kv, - "vn_lv_kv": vn_lv_kv, - "vn_hv_kv_next_bus": vn_hv_kv, - "vn_lv_kv_next_bus": vn_lv_kv, - "hv_rel_deviation": np.zeros(len(vn_hv_kv)), - "lv_rel_deviation": np.zeros(len(vn_hv_kv)), - }) - trafo_connections[["hv_bus", "lv_bus"]] = trafo_connections[[ - "hv_bus", "lv_bus"]].astype(np.int64) - - for side in ["hv", "lv"]: - bus_col, trafo_vn_col, next_col, rel_dev_col, has_dev_col = \ - f"{side}_bus", f"vn_{side}_kv", f"vn_{side}_kv_next_bus", f"{side}_rel_deviation", \ - f"trafo_{side}_to_bus_deviation" - name_vn_series = pd.Series( - tuple(zip(trafo_location_names, trafo_connections[trafo_vn_col]))) - isin = name_vn_series.isin(bus_idx.index) - trafo_connections[has_dev_col] = ~isin - trafo_connections.loc[isin, bus_col] = bus_idx.loc[name_vn_series.loc[isin]].values - - # --- code to find bus locations with vn deviation - next_vn = np.array([bus_idx.loc[tln.name].index.values[ - (pd.Series(bus_idx.loc[tln.name].index) - getattr(tln, trafo_vn_col)).abs().idxmin( - )] for tln in trafo_connections.loc[~isin, ["name", trafo_vn_col]].itertuples()]) - trafo_connections.loc[~isin, next_col] = next_vn - rel_dev = np.abs(next_vn - trafo_connections.loc[~isin, trafo_vn_col].values) / next_vn - trafo_connections.loc[~isin, rel_dev_col] = rel_dev - trafo_connections.loc[~isin, bus_col] = \ - bus_idx.loc[list(tuple(zip(trafo_connections.loc[~isin, "name"], - trafo_connections.loc[~isin, next_col])))].values - - # --- create buses to avoid too large vn deviations between nodes and transformers - need_bus_creation = trafo_connections[rel_dev_col] > \ - rel_deviation_threshold_for_trafo_bus_creation - new_bus_data = pd.DataFrame({ - "vn_kv": trafo_connections.loc[need_bus_creation, trafo_vn_col].values, - "name": trafo_connections.loc[need_bus_creation, "name"].values, - "TSO": data[key].loc[need_bus_creation, ("Location", "TSO")].values - }) - new_bus_data_dd = _drop_duplicates_and_join_TSO(new_bus_data) - new_bus_idx = create_buses(net, len(new_bus_data_dd), vn_kv=new_bus_data_dd.vn_kv, - name=new_bus_data_dd.name, zone=new_bus_data_dd.TSO) - trafo_connections.loc[need_bus_creation, bus_col] = net.bus.loc[new_bus_idx, [ - "name", "vn_kv"]].reset_index().set_index(["name", "vn_kv"]).loc[list(new_bus_data[[ - "name", "vn_kv"]].itertuples(index=False, name=None))].values - trafo_connections.loc[need_bus_creation, next_col] = \ - trafo_connections.loc[need_bus_creation, trafo_vn_col].values - trafo_connections.loc[need_bus_creation, rel_dev_col] = 0 - trafo_connections.loc[need_bus_creation, has_dev_col] = False - - # --- create buses for trafos that are connected to the same bus at both sides (possible if - # --- vn_hv_kv < vn_lv_kv *(1+rel_deviation_threshold_for_trafo_bus_creation) which usually - # --- occurs for PSTs only) - same_bus_connection = trafo_connections.hv_bus == trafo_connections.lv_bus - duplicated_buses = net.bus.loc[trafo_connections.loc[same_bus_connection, "lv_bus"]].copy() - duplicated_buses["name"] += " (2)" - duplicated_buses.index = list(range(net.bus.index.max()+1, - net.bus.index.max()+1+len(duplicated_buses))) - trafo_connections.loc[same_bus_connection, "lv_bus"] = duplicated_buses.index - net.bus = pd.concat([net.bus, duplicated_buses]) - if n_add_buses := len(duplicated_buses): - tr_names = data[key].loc[trafo_connections.index[same_bus_connection], - ("Location", "Full Name")] - are_PSTs = tr_names.str.contains("PST") - logger.info(f"{n_add_buses} additional buses were created to avoid that transformers are " - f"connected to the same bus at both side, hv and lv. Of the causing " - f"{len(tr_names)} transformers, {sum(are_PSTs)} contain 'PST' in their name. " - f"According to this converter, the power flows over all these transformers will" - f" end at the additional buses. Please consider to connect lines with the " - f"additional buses, so that the power flow is over the (PST) transformers into " - f"the lines.") - - # --- log according to log_rel_vn_deviation - for side in ["hv", "lv"]: - need_logging = trafo_connections.loc[trafo_connections[has_dev_col], - rel_dev_col] > log_rel_vn_deviation - if n_need_logging := sum(need_logging): - max_dev = trafo_connections[rel_dev_col].max() - idx_max_dev = trafo_connections[rel_dev_col].idxmax() - logger.warning( - f"For {n_need_logging} Transformers ({side} side), only locations were found (orig" - f"in are the line and tieline data) that have a higher relative deviation than " - f"{log_rel_vn_deviation}. The maximum relative deviation is {max_dev} which " - f"results from a Transformer rated voltage of " - f"{trafo_connections.at[idx_max_dev, trafo_vn_col]} and a bus " - f"rated voltage (taken from Lines/Tielines data sheet) of " - f"{trafo_connections.at[idx_max_dev, next_col]}. The best locations were " - f"nevertheless applied, due to {rel_deviation_threshold_for_trafo_bus_creation=}") - - assert (trafo_connections.hv_bus > -1).all() - assert (trafo_connections.lv_bus > -1).all() - assert (trafo_connections.hv_bus != trafo_connections.lv_bus).all() - - return trafo_connections - - -def _find_trafo_locations(trafo_bus_names, bus_location_names): - # --- split (original and lower case) strings at " " separators to remove impeding parts for - # identifying the location names - trafo_bus_names_expended = trafo_bus_names.str.split(r"[ ]+|-A[0-9]+|-TD[0-9]+|-PF[0-9]+", - expand=True).fillna("").replace(" ", "") - trafo_bus_names_expended_lower = trafo_bus_names.str.lower().str.split( - r"[ ]+|-A[0-9]+|-TD[0-9]+|-PF[0-9]+", expand=True).fillna("").replace(" ", "") - - # --- identify impeding parts - contains_number = trafo_bus_names_expended.map(lambda x: any(char.isdigit() for char in x)) - to_drop = (trafo_bus_names_expended_lower == "tr") | (trafo_bus_names_expended_lower == "pst") \ - | (trafo_bus_names_expended == "") | (trafo_bus_names_expended == "/") | ( - trafo_bus_names_expended == "LIPST") | (trafo_bus_names_expended == "EHPST") | ( - trafo_bus_names_expended == "TFO") | (trafo_bus_names_expended_lower == "trafo") | ( - trafo_bus_names_expended_lower == "kv") | contains_number - trafo_bus_names_expended[to_drop] = "" - - # --- reconstruct name strings for identification - trafo_bus_names_joined = trafo_bus_names_expended.where(~to_drop).fillna('').agg( - ' '.join, axis=1).str.strip() - trafo_bus_names_longest_part = trafo_bus_names_expended.apply( - lambda row: max(row, key=len), axis=1) - joined_in_buses = trafo_bus_names_joined.isin(bus_location_names) - longest_part_in_buses = trafo_bus_names_longest_part.isin(bus_location_names) - - # --- check whether all name strings point at location names of the buses - if False: # for easy testing - fail = ~(joined_in_buses | longest_part_in_buses) - a = pd.concat([trafo_bus_names_joined.loc[fail], - trafo_bus_names_longest_part.loc[fail]], axis=1) - - if n_bus_names_not_found := len(joined_in_buses) - sum(joined_in_buses | longest_part_in_buses): - raise ValueError( - f"For {n_bus_names_not_found} Tranformers, no suitable bus location names were found, " - f"i.e. the algorithm did not find a (part) of Transformers-Location-Full Name that fits" - " to Substation_1 or Substation_2 data in Lines or Tielines sheet.") - - # --- set the trafo location names and trafo bus indices respectively - trafo_location_names = trafo_bus_names_longest_part - trafo_location_names.loc[joined_in_buses] = trafo_bus_names_joined - - return trafo_location_names - - -def _drop_duplicates_and_join_TSO(bus_df: pd.DataFrame) -> pd.DataFrame: - bus_df = bus_df.drop_duplicates(ignore_index=True) - # just keep one bus per name and vn_kv. If there are multiple buses of different TSOs, join the - # TSO strings: - bus_df = bus_df.groupby(["name", "vn_kv"], as_index=False).agg({"TSO": lambda x: '/'.join(x)}) - assert not bus_df.duplicated(["name", "vn_kv"]).any() - return bus_df - - -def _get_float_column(df, col_tuple, fill=0): - series = df.loc[:, col_tuple] - series.loc[series == "\xa0"] = fill - return series.astype(float).fillna(fill) - - -def _get_bus_idx(net: pandapowerNet) -> pd.Series: - return net.bus[["name", "vn_kv"]].rename_axis("index").reset_index().set_index([ - "name", "vn_kv"])["index"] - - -def get_grid_groups(net: pandapowerNet, **kwargs) -> pd.DataFrame: - notravbuses_dict = dict() if "notravbuses" not in kwargs.keys() else { - "notravbuses": kwargs.pop("notravbuses")} - grid_group_buses = [set_ for set_ in connected_components(create_nxgraph(net, **kwargs), - **notravbuses_dict)] - grid_groups = pd.DataFrame({"buses": grid_group_buses}) - grid_groups["n_buses"] = grid_groups["buses"].apply(len) - return grid_groups - - -def _lng_lat_to_df(dict_: dict, line_EIC: str, line_name: str) -> pd.DataFrame: - return pd.DataFrame([ - [line_EIC, line_name, "from", "lng", dict_["lng"][0]], - [line_EIC, line_name, "to", "lng", dict_["lng"][1]], - [line_EIC, line_name, "from", "lat", dict_["lat"][0]], - [line_EIC, line_name, "to", "lat", dict_["lat"][1]], - ], columns=["EIC_Code", "name", "bus", "geo_dim", "value"]) - - -def _fill_geo_at_one_sided_branches_without_geo_extent(net: pandapowerNet): - - def _check_geo_availablitiy(net: pandapowerNet) -> dict[str, Union[pd.Index, int]]: - av = dict() # availablitiy of geodata - av["bus_with_geo"] = net.bus.index[~net.bus.geo.isnull()] - av["lines_fbw_tbwo"] = net.line.index[net.line.from_bus.isin(av["bus_with_geo"]) & - (~net.line.to_bus.isin(av["bus_with_geo"]))] - av["lines_fbwo_tbw"] = net.line.index[(~net.line.from_bus.isin(av["bus_with_geo"])) & - net.line.to_bus.isin(av["bus_with_geo"])] - av["trafos_hvbw_lvbwo"] = net.trafo.index[net.trafo.hv_bus.isin(av["bus_with_geo"]) & - (~net.trafo.lv_bus.isin(av["bus_with_geo"]))] - av["trafos_hvbwo_lvbw"] = net.trafo.index[(~net.trafo.hv_bus.isin(av["bus_with_geo"])) & - net.trafo.lv_bus.isin(av["bus_with_geo"])] - av["n_lines_one_side_geo"] = len(av["lines_fbw_tbwo"])+len(av["lines_fbwo_tbw"]) - return av - - geo_avail = _check_geo_availablitiy(net) - while geo_avail["n_lines_one_side_geo"]: - - # copy available geodata to the other end of branches where geodata are missing - for et, bus_w_geo, bus_wo_geo, idx_key in zip( - ["line", "line", "trafo", "trafo"], - ["to_bus", "from_bus", "lv_bus", "hv_bus"], - ["from_bus", "to_bus", "hv_bus", "lv_bus"], - ["lines_fbwo_tbw", "lines_fbw_tbwo", "trafos_hvbwo_lvbw", "trafos_hvbw_lvbwo"]): - net.bus.loc[net[et].loc[geo_avail[idx_key], bus_wo_geo].values, "geo"] = \ - net.bus.loc[net[et].loc[geo_avail[idx_key], bus_w_geo].values, "geo"].values - geo_avail = _check_geo_availablitiy(net) - - set_line_geodata_from_bus_geodata(net) - - -def _multi_str_repl(st: str, repl: list[tuple]) -> str: - for (old, new) in repl: - st = st.replace(old, new) - return st - - -if __name__ == "__main__": - from pathlib import Path - import os - from pandapower.file_io import from_json, to_json - - home = str(Path.home()) - jao_data_folder = os.path.join(home, "Documents", "JAO Static Grid Model") - - release5 = os.path.join(jao_data_folder, "20240329_Core Static Grid Model – 5th release") - excel_file_path = os.path.join(release5, "20240329_Core Static Grid Model_public.xlsx") - html_file_path = os.path.join(release5, "20240329_Core Static Grid Model Map_public", - "2024-03-18_Core_SGM_publication.html") - - release6 = os.path.join(jao_data_folder, "202409_Core Static Grid Mode_6th release") - excel_file_path = os.path.join(release6, "20240916_Core Static Grid Model_for publication.xlsx") - html_file_path = os.path.join(release6, "2024-09-13_Core_SGM_publication_files", - "2024-09-13_Core_SGM_publication.html") - - pp_net_json_file = os.path.join(home, "desktop", "jao_grid.json") - - if 1: # read from original data - net = from_jao(excel_file_path, html_file_path, True, drop_grid_groups_islands=True) - to_json(net, pp_net_json_file) - else: # load net from already converted and stored net - net = from_json(pp_net_json_file) - - print(net) - grid_groups = get_grid_groups(net) - print(grid_groups) - - _fill_geo_at_one_sided_branches_without_geo_extent(net)