Merge branch 'dev' into wp_flex_changed_reinforcement_2

.github/workflows/tests-coverage.yml

@@ -38,7 +38,7 @@ jobs:
           python-version: ${{ matrix.python-version }}
 
       - name: Set up julia
-        if: runner.os == 'Linux' && matrix.python-version == 3.8 && matrix.name-suffix == 'coverage'
+        if: runner.os == 'Linux'
         uses: julia-actions/setup-julia@v1
         with:
           version: "1.6"
@@ -58,8 +58,14 @@ jobs:
           environment-file: eDisGo_env_dev.yml
           python-version: ${{ matrix.python-version }}
 
-      - name: Run tests
-        if: ${{ !(runner.os == 'Linux' && matrix.python-version == 3.8 && matrix.name-suffix == 'coverage') }}
+      - name: Run tests Linux
+        if: runner.os == 'Linux' && matrix.name-suffix != 'coverage'
+        run: |
+          python -m pip install pytest pytest-notebook
+          python -m pytest --runslow --runonlinux --disable-warnings --color=yes -v
+
+      - name: Run tests Windows
+        if: runner.os == 'Windows'
         run: |
           python -m pip install pytest pytest-notebook
           python -m pytest --runslow --disable-warnings --color=yes -v

edisgo/flex_opt/costs.py

@@ -1,3 +1,4 @@
+import logging
 import os
 
 import pandas as pd
@@ -7,6 +8,8 @@
 
 from edisgo.tools.geo import proj2equidistant
 
+logger = logging.getLogger(__name__)
+
 
 def grid_expansion_costs(edisgo_obj, without_generator_import=False):
     """
@@ -67,7 +70,8 @@ def _get_transformer_costs(trafos):
         costs_trafos = pd.DataFrame(
             {
                 "costs_transformers": len(hvmv_trafos)
-                * [float(edisgo_obj.config["costs_transformers"]["mv"])]
+                * [float(edisgo_obj.config["costs_transformers"]["mv"])],
+                "voltage_level": len(hvmv_trafos) * ["hv/mv"],
             },
             index=hvmv_trafos,
         )
@@ -77,13 +81,14 @@ def _get_transformer_costs(trafos):
                 pd.DataFrame(
                     {
                         "costs_transformers": len(mvlv_trafos)
-                        * [float(edisgo_obj.config["costs_transformers"]["lv"])]
+                        * [float(edisgo_obj.config["costs_transformers"]["lv"])],
+                        "voltage_level": len(mvlv_trafos) * ["mv/lv"],
                     },
                     index=mvlv_trafos,
                 ),
             ]
         )
-        return costs_trafos.loc[trafos.index, "costs_transformers"].values
+        return costs_trafos.loc[trafos.index, :]
 
     def _get_line_costs(lines_added):
         costs_lines = line_expansion_costs(edisgo_obj, lines_added.index)
@@ -128,16 +133,16 @@ def _get_line_costs(lines_added):
         )
         trafos = all_trafos.loc[added_transformers["equipment"]]
         # calculate costs for each transformer
-        # ToDo voltage level should be hv/mv for HV/MV transformers
+        transformer_costs = _get_transformer_costs(trafos)
         costs = pd.concat(
             [
                 costs,
                 pd.DataFrame(
                     {
                         "type": trafos.type_info.values,
-                        "total_costs": _get_transformer_costs(trafos),
+                        "total_costs": transformer_costs.costs_transformers,
                         "quantity": len(trafos) * [1],
-                        "voltage_level": len(trafos) * ["mv/lv"],
+                        "voltage_level": transformer_costs.voltage_level,
                     },
                     index=trafos.index,
                 ),

edisgo/network/results.py

@@ -628,7 +628,7 @@ def _add_line_to_equipment_changes(self, line):
                         "iteration_step": [0],
                         "change": ["added"],
                         "equipment": [line.type_info],
-                        "quantity": [1],
+                        "quantity": [line.num_parallel],
                     },
                     index=[line.name],
                 ),

edisgo/tools/logger.py

@@ -140,9 +140,9 @@ def create_home_dir():
         log_dir = os.path.join(
             get_default_root_dir(), cfg_edisgo.get("user_dirs", "log_dir")
         )
-    create_dir(log_dir)
 
     if log_dir is not None:
+        create_dir(log_dir)
         file_name = os.path.join(log_dir, file_name)
 
     if reset_loggers:

examples/electromobility_example.ipynb

@@ -430,8 +430,7 @@
     "    url = (f\"https://github.com/openego/eDisGo/tree/dev/\" +\n",
     "           f\"tests/data/simbev_example_scenario/{ags}/\")\n",
     "    page = requests.get(url).text\n",
-    "    items = json.loads(page)[\"payload\"][\"tree\"][\"items\"]\n",
-    "    filenames = [f[\"name\"] for f in items if \"csv\" in f[\"name\"]]\n",
+    "    filenames = [_ for _ in page.split('\"') if \".csv\" in _ and \"/\" not in _]\n",
     "\n",
     "    for file in filenames:\n",
     "        req = requests.get(f\"{raw_url}/{ags}/{file}\")\n",
@@ -470,8 +469,7 @@
     "    url = (\"https://github.com/openego/eDisGo/tree/dev/\" +\n",
     "           \"tests/data/tracbev_example_scenario/\")\n",
     "    page = requests.get(url).text\n",
-    "    items = json.loads(page)[\"payload\"][\"tree\"][\"items\"]\n",
-    "    filenames = [f[\"name\"] for f in items if \"gpkg\" in f[\"name\"]]\n",
+    "    filenames = [_ for _ in page.split('\"') if \".gpkg\" in _ and \"/\" not in _]\n",
     "\n",
     "    for file in filenames:\n",
     "        req = requests.get(\n",

tests/flex_opt/test_costs.py

@@ -3,7 +3,7 @@
 import pytest
 
 from edisgo import EDisGo
-from edisgo.flex_opt.costs import grid_expansion_costs, line_expansion_costs
+from edisgo.flex_opt import costs as costs_mod
 
 
 class TestCosts:
@@ -76,12 +76,12 @@ def test_costs(self):
             ],
         )
 
-        costs = grid_expansion_costs(self.edisgo)
+        costs = costs_mod.grid_expansion_costs(self.edisgo)
 
         assert len(costs) == 4
         assert (
             costs.loc["MVStation_1_transformer_reinforced_2", "voltage_level"]
-            == "mv/lv"
+            == "hv/mv"
         )
         assert costs.loc["MVStation_1_transformer_reinforced_2", "quantity"] == 1
         assert costs.loc["MVStation_1_transformer_reinforced_2", "total_costs"] == 1000
@@ -97,13 +97,13 @@ def test_costs(self):
         assert costs.loc["Line_10019", "type"] == "48-AL1/8-ST1A"
         assert costs.loc["Line_10019", "voltage_level"] == "mv"
         assert np.isclose(costs.loc["Line_50000002", "total_costs"], 2.34)
-        assert np.isclose(costs.loc["Line_50000002", "length"], 0.09)
-        assert costs.loc["Line_50000002", "quantity"] == 3
+        assert np.isclose(costs.loc["Line_50000002", "length"], 0.03)
+        assert costs.loc["Line_50000002", "quantity"] == 1
         assert costs.loc["Line_50000002", "type"] == "NAYY 4x1x35"
         assert costs.loc["Line_50000002", "voltage_level"] == "lv"
 
     def test_line_expansion_costs(self):
-        costs = line_expansion_costs(self.edisgo)
+        costs = costs_mod.line_expansion_costs(self.edisgo)
         assert len(costs) == len(self.edisgo.topology.lines_df)
         assert (costs.index == self.edisgo.topology.lines_df.index).all()
         assert len(costs[costs.voltage_level == "mv"]) == len(
@@ -116,7 +116,9 @@ def test_line_expansion_costs(self):
         assert np.isclose(costs.at["Line_10000015", "costs_cable"], 0.27)
         assert costs.at["Line_10000015", "voltage_level"] == "lv"
 
-        costs = line_expansion_costs(self.edisgo, ["Line_10003", "Line_10000015"])
+        costs = costs_mod.line_expansion_costs(
+            self.edisgo, ["Line_10003", "Line_10000015"]
+        )
         assert len(costs) == 2
         assert (costs.index.values == ["Line_10003", "Line_10000015"]).all()
         assert np.isclose(costs.at["Line_10003", "costs_earthworks"], 0.083904 * 60)
@@ -125,3 +127,28 @@ def test_line_expansion_costs(self):
         assert np.isclose(costs.at["Line_10000015", "costs_earthworks"], 1.53)
         assert np.isclose(costs.at["Line_10000015", "costs_cable"], 0.27)
         assert costs.at["Line_10000015", "voltage_level"] == "lv"
+
+    def test_transformer_expansion_costs(self):
+        costs = costs_mod.transformer_expansion_costs(self.edisgo)
+        transformers_df = pd.concat(
+            [
+                self.edisgo.topology.transformers_df,
+                self.edisgo.topology.transformers_hvmv_df,
+            ]
+        )
+        assert len(costs) == len(transformers_df)
+        assert sorted(costs.index) == sorted(transformers_df.index)
+        assert len(costs[costs.voltage_level == "hv/mv"]) == len(
+            self.edisgo.topology.transformers_hvmv_df
+        )
+        assert np.isclose(costs.at["MVStation_1_transformer_1", "costs"], 1000)
+        assert costs.at["MVStation_1_transformer_1", "voltage_level"] == "hv/mv"
+        assert np.isclose(costs.at["LVStation_4_transformer_2", "costs"], 10)
+        assert costs.at["LVStation_4_transformer_2", "voltage_level"] == "mv/lv"
+
+        costs = costs_mod.transformer_expansion_costs(
+            self.edisgo, ["LVStation_5_transformer_1"]
+        )
+        assert len(costs) == 1
+        assert np.isclose(costs.at["LVStation_5_transformer_1", "costs"], 10)
+        assert costs.at["LVStation_5_transformer_1", "voltage_level"] == "mv/lv"

tests/flex_opt/test_reinforce_grid.py

@@ -81,7 +81,7 @@ def test_run_separate_lv_grids(self):
                 assert len(g.buses_df) > 1
 
         assert len(lv_grids_new) == 26
-        assert np.isclose(edisgo.results.grid_expansion_costs.total_costs.sum(), 280.06)
+        assert np.isclose(edisgo.results.grid_expansion_costs.total_costs.sum(), 440.06)
 
         # check if all generators are still present
         assert np.isclose(

tests/test_edisgo.py

@@ -521,7 +521,7 @@ def test_reinforce_catch_convergence(self):
         )
         results = self.edisgo.reinforce(catch_convergence_problems=True)
         assert results.unresolved_issues.empty
-        assert len(results.grid_expansion_costs) == 132
+        assert len(results.grid_expansion_costs) == 134
         assert len(results.equipment_changes) == 218
         assert results.v_res.shape == (4, 142)
 
@@ -543,7 +543,7 @@ def test_enhanced_reinforce_grid(self):
 
         results = edisgo_obj.results
 
-        assert len(results.grid_expansion_costs) == 445
+        assert len(results.grid_expansion_costs) == 454
         assert len(results.equipment_changes) == 892
         assert results.v_res.shape == (4, 148)
 

tests/tools/test_logger.py

@@ -1,65 +1,85 @@
 import logging
 import os
 
+import pytest
+
 from edisgo.tools.logger import setup_logger
 
 
+def check_file_output(filename, output):
+    with open(filename) as file:
+        last_line = file.readlines()[-1].split(" ")[3:]
+        last_line = " ".join(last_line)
+    assert last_line == output
+
+
+def reset_loggers(filename):
+    logger = logging.getLogger("edisgo")
+    logger.handlers.clear()
+    logger.propagate = True
+    # try removing file - when run on github for Windows removing the file leads
+    # to a PermissionError
+    try:
+        os.remove(filename)
+    except PermissionError:
+        pass
+
+
 class TestClass:
     def test_setup_logger(self):
-        def check_file_output(output):
-            with open("edisgo.log") as file:
-                last_line = file.readlines()[-1].split(" ")[3:]
-                last_line = " ".join(last_line)
-            assert last_line == output
-
-        def reset_loggers():
-            logger = logging.getLogger("edisgo")
-            logger.propagate = True
-            logger.handlers.clear()
-            logger = logging.getLogger()
-            logger.handlers.clear()
-
-        if os.path.exists("edisgo.log"):
-            os.remove("edisgo.log")
+        filename = os.path.join(
+            os.path.expanduser("~"), ".edisgo", "log", "test_log.log"
+        )
+        if os.path.exists(filename):
+            os.remove(filename)
 
         setup_logger(
             loggers=[
                 {"name": "root", "file_level": "debug", "stream_level": "debug"},
                 {"name": "edisgo", "file_level": "debug", "stream_level": "debug"},
             ],
-            file_name="edisgo.log",
+            file_name="test_log.log",
+            log_dir="default",
         )
 
         logger = logging.getLogger("edisgo")
         # Test that edisgo logger writes to file.
         logger.debug("root")
-        check_file_output("edisgo - DEBUG: root\n")
+        check_file_output(filename, "edisgo - DEBUG: root\n")
         # Test that root logger writes to file.
         logging.debug("root")
-        check_file_output("root - DEBUG: root\n")
+        check_file_output(filename, "root - DEBUG: root\n")
+
+        reset_loggers(filename)
 
-        # reset_loggers()
+    @pytest.mark.runonlinux
+    def test_setup_logger_2(self):
+        """
+        This test is only run on linux, as the log file is written to the user
+        home directory, which is not allowed when tests are run on github.
+
+        """
+
+        # delete any existing log files
+        log_files = [_ for _ in os.listdir(os.getcwd()) if ".log" in _]
+        for log_file in log_files:
+            os.remove(log_file)
 
         setup_logger(
             loggers=[
                 {"name": "edisgo", "file_level": "debug", "stream_level": "debug"},
             ],
-            file_name="edisgo.log",
             reset_loggers=True,
             debug_message=True,
         )
         logger = logging.getLogger("edisgo")
+
+        filename = [_ for _ in os.listdir(os.getcwd()) if ".log" in _][0]
         # Test that edisgo logger writes to file.
         logger.debug("edisgo")
-        check_file_output("edisgo - DEBUG: edisgo\n")
-        # Test that root logger doesn't writes to file.
-        logging.debug("edisgo")
-        check_file_output("edisgo - DEBUG: edisgo\n")
-
-    @classmethod
-    def teardown_class(cls):
-        logger = logging.getLogger("edisgo")
-        logger.handlers.clear()
-        logger.propagate = True
+        check_file_output(filename, "edisgo - DEBUG: edisgo\n")
+        # Test that root logger doesn't write to file.
+        logging.debug("root")
+        check_file_output(filename, "edisgo - DEBUG: edisgo\n")
 
-        os.remove("edisgo.log")
+        reset_loggers(filename)