Features/#856 cleanup results folder

etrago/appl.py

@@ -89,7 +89,7 @@
     "scn_extension": None,  # None or array of extension scenarios
     # Export options:
     "lpfile": False,  # save pyomo's lp file: False or /path/to/lpfile.lp
-    "csv_export": "results",  # save results as csv: False or /path/tofolder
+    "export_results_path": "results",  # save results as csv: False or /path/tofolder
     # Settings:
     "extendable": {
         "extendable_components": [
@@ -290,9 +290,9 @@ def run_etrago(args, json_path):
     lpfile : bool or str
         State if and where you want to save pyomo's lp file. Options:
         False or '/path/tofile.lp'. Default: False.
-    csv_export : bool or str
-        State if and where you want to save results as csv files. Options:
-        False or '/path/tofolder'. Default: False.
+    export_results_path : bool or str
+        State if and where you want to save results as csv and .nc files.
+        Options: False or '/path/tofolder'. Default: False.
 
     extendable : dict
         Choose components you want to optimize and set upper bounds for grid

etrago/cluster/electrical.py

@@ -42,6 +42,7 @@
     from etrago.cluster.spatial import (
         busmap_ehv_clustering,
         drop_nan_values,
+        export_clustering_results,
         focus_weighting,
         group_links,
         kmean_clustering,
@@ -1230,3 +1231,6 @@ def run_spatial_clustering(self):
             )
             + self.args["network_clustering"]["method"]["algorithm"]
         )
+
+        if self.args["export_results_path"]:
+            export_clustering_results(self)

etrago/cluster/gas.py

@@ -38,6 +38,7 @@
 if "READTHEDOCS" not in os.environ:
     from etrago.cluster.spatial import (
         drop_nan_values,
+        export_clustering_results,
         focus_weighting,
         group_links,
         kmedoids_dijkstra_clustering,
@@ -1266,3 +1267,6 @@ def run_spatial_clustering_gas(self):
                         method,
                     )
                 )
+
+        if self.args["export_results_path"]:
+            export_clustering_results(self)

etrago/cluster/spatial.py

@@ -20,6 +20,7 @@
 # File description for read-the-docs
 """spatial.py defines the methods to run spatial clustering on networks."""
 
+import json
 import logging
 import os
 
@@ -1092,3 +1093,15 @@ def drop_nan_values(network):
             (c.attrs.status == "Output") & (c.attrs.varying)
         ].index:
             c.pnl[pnl] = pd.DataFrame(index=network.snapshots)
+
+
+def export_clustering_results(etrago):
+
+    path = etrago.args["export_results_path"]
+
+    with open(os.path.join(path, "busmap.json"), "w") as d:
+        json.dump(etrago.busmap["busmap"], d, indent=4)
+
+    etrago.busmap["orig_network"].export_to_csv_folder(
+        path + "/original_network_topology"
+    )

etrago/disaggregate/spatial.py

@@ -926,6 +926,21 @@ def run_disaggregation(self):
                 )
             )
 
-            if self.args["csv_export"]:
-                path = self.args["csv_export"] + "/disaggregated_network"
-                self.disaggregated_network.export_to_csv_folder(path)
+            if self.args["export_results_path"]:
+                path = (
+                    self.args["export_results_path"]
+                    + "/disaggregated_network.nc"
+                )
+
+                for comp_df in [
+                    self.disaggregated_network.transformers,
+                    self.disaggregated_network.lines,
+                    self.disaggregated_network.links,
+                    self.disaggregated_network.buses,
+                ]:
+                    if "geom" in comp_df.columns:
+                        comp_df.drop(columns=["geom"], inplace=True)
+                    if "topo" in comp_df.columns:
+                        comp_df.drop(columns=["topo"], inplace=True)
+
+                self.disaggregated_network.export_to_netcdf(path)

etrago/disaggregate/temporal.py

@@ -208,7 +208,6 @@ def dispatch_disaggregation(self):
         )
         self.network.stores.e_cyclic = self.network_tsa.stores.e_cyclic
 
-        if self.args["csv_export"]:
-            path = self.args["csv_export"]
-            self.export_to_csv(path)
-            self.export_to_csv(path + "/temporal_disaggregaton")
+        if self.args["export_results_path"]:
+            path = self.args["export_results_path"]
+            self.network.export_to_csv_folder(path + "/temporal_disaggregaton")

etrago/execute/__init__.py

@@ -26,6 +26,7 @@
 
 if "READTHEDOCS" not in os.environ:
     import logging
+    import shutil
     import time
 
     from pypsa.linopf import network_lopf
@@ -202,13 +203,14 @@ def iterate_lopf(
     """
 
     args = etrago.args
-    path = args["csv_export"]
     lp_path = args["lpfile"]
 
-    if args["temporal_disaggregation"]["active"]:
-        if args["csv_export"]:
-            path = path + "/temporally_reduced"
+    if args["export_results_path"]:
+        path = args["export_results_path"] + "/grid_optimization"
+        if not os.path.exists(path):
+            os.makedirs(path, exist_ok=True)
 
+    if args["temporal_disaggregation"]["active"]:
         if args["lpfile"]:
             lp_path = lp_path[0:-3] + "_temporally_reduced.lp"
 
@@ -229,9 +231,20 @@ def iterate_lopf(
             for i in range(1, (1 + n_iter)):
                 run_lopf(etrago, extra_functionality, method)
 
-                if args["csv_export"]:
-                    path_it = path + "/lopf_iteration_" + str(i)
-                    etrago.export_to_csv(path_it)
+                if args["export_results_path"]:
+                    # Store intermediate iterations
+                    if i < n_iter:
+                        path_it = path + "/lopf_iteration_" + str(i)
+                        etrago.network.export_to_csv_folder(path_it)
+                    # Store final result
+                    else:
+                        etrago.network.export_to_csv_folder(path)
+
+                    # Delete previous iteration's results
+                    if i > 1:
+                        path_prev = path + "/lopf_iteration_" + str(i - 1)
+                        if os.path.exists(path_prev):
+                            shutil.rmtree(path_prev)
 
                 if i < n_iter:
                     l_snom_pre, t_snom_pre = update_electrical_parameters(
@@ -263,18 +276,18 @@ def iterate_lopf(
 
                 i += 1
 
-                if args["csv_export"]:
+                if args["export_results_path"]:
                     path_it = path + "/lopf_iteration_" + str(i)
-                    etrago.export_to_csv(path_it)
+                    etrago.network.export_to_csv_folder(path_it)
 
                 if abs(pre - network.objective) <= diff_obj:
                     print("Threshold reached after " + str(i) + " iterations.")
                     break
 
     else:
         run_lopf(etrago, extra_functionality, method)
-        if etrago.args["csv_export"]:
-            etrago.export_to_csv(path)
+        if etrago.args["export_results_path"]:
+            etrago.network.export_to_csv_folder(path)
 
     if args["lpfile"]:
         network.model.write(lp_path)
@@ -322,12 +335,6 @@ def lopf(self):
     z = (y - x) / 60
     logger.info("Time for LOPF [min]: {}".format(round(z, 2)))
 
-    if self.args["csv_export"]:
-        path = self.args["csv_export"]
-        if self.args["temporal_disaggregation"]["active"] is True:
-            path = path + "/temporally_reduced"
-        self.export_to_csv(path)
-
 
 def optimize(self):
     """Run optimization of dispatch and grid and storage expansion based on
@@ -1054,15 +1061,25 @@ def drop_foreign_components(network):
                     foreign_series[comp][attr], comp, attr
                 )
 
-    if args["csv_export"]:
-        path = args["csv_export"] + "/pf_post_lopf"
-        etrago.export_to_csv(path)
+    if args["export_results_path"]:
+        path = args["export_results_path"] + "/pf_post_lopf"
+        etrago.network.export_to_csv_folder(path)
         pf_solve.to_csv(os.path.join(path, "pf_solution.csv"), index=True)
 
         # Save un-solved disaggregated network
         if args["spatial_disaggregation"]:
-            etrago.disaggregated_network.export_to_csv_folder(
-                args["csv_export"] + "/disaggregated_network"
+            for comp_df in [
+                etrago.disaggregated_network.transformers,
+                etrago.disaggregated_network.lines,
+                etrago.disaggregated_network.links,
+                etrago.disaggregated_network.buses,
+            ]:
+                if "geom" in comp_df.columns:
+                    comp_df.drop(columns=["geom"], inplace=True)
+                if "topo" in comp_df.columns:
+                    comp_df.drop(columns=["topo"], inplace=True)
+            etrago.disaggregated_network.export_to_netcdf(
+                args["export_results_path"] + "/disaggregated_network.nc"
             )
 
     return network

etrago/execute/market_optimization.py

@@ -91,11 +91,13 @@ def market_optimization(self):
         logger.warning("Method type must be either 'pyomo' or 'linopy'")
 
     # Export results of pre-market model
-    if self.args["csv_export"]:
-        path = self.args["csv_export"]
+    if self.args["export_results_path"]:
+        path = self.args["export_results_path"]
         if not os.path.exists(path):
             os.makedirs(path, exist_ok=True)
-        self.pre_market_model.export_to_csv_folder(path + "/pre_market")
+        self.pre_market_model.export_to_csv_folder(
+            path + "/pre_market_optimization"
+        )
     logger.info("Preparing short-term UC market model")
 
     build_shortterm_market_model(self, unit_commitment)
@@ -128,11 +130,11 @@ def market_optimization(self):
     self.args["method"]["formulation"] = method_args
 
     # Export results of market model
-    if self.args["csv_export"]:
-        path = self.args["csv_export"]
+    if self.args["export_results_path"]:
+        path = self.args["export_results_path"]
         if not os.path.exists(path):
             os.makedirs(path, exist_ok=True)
-        self.market_model.export_to_csv_folder(path + "/market")
+        self.market_model.export_to_csv_folder(path + "/market_optimization")
 
 
 def build_market_model(self, unit_commitment=False):