Merge pull request #578 from BDonnot/bd_dev

Bd dev
Grid2op · Jan 26, 2024 · 60c8301 · 60c8301
2 parents 95290d9 + ca7b66f
commit 60c8301
Show file tree

Hide file tree

Showing 20 changed files with 393 additions and 191 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -47,13 +47,17 @@ jobs:
                   export _GRID2OP_FORCE_TEST=1
                   cd grid2op/tests/
                   python3 helper_list_test.py | circleci tests split > /tmp/tests_run
+            - run:
+                command: |
+                  source venv_test/bin/activate
+                  pip freeze
             - run: cat /tmp/tests_run
             - run:
                 command: |
                   source venv_test/bin/activate
                   cd grid2op/tests/
                   export _GRID2OP_FORCE_TEST=1
-                  python3 -m unittest $(cat /tmp/tests_run)
+                  python3 -m unittest -v $(cat /tmp/tests_run)
                   
     install36:
         executor: python36
@@ -136,37 +140,28 @@ jobs:
             - run:
                command: |
                    source venv_test/bin/activate
-                   python -m pip install -U "numpy>=1.20,<1.21"
-                   python -m pip install -U .[test]
+                   python -m pip install -U "numpy>=1.20,<1.21" "pandas<2.2" "scipy<1.12" numba .[test]
+                   pip freeze
+            - run:
+                command: |
+                   source venv_test/bin/activate
                    export _GRID2OP_FORCE_TEST=1
+                   cd /tmp
                    grid2op.testinstall
-            # - run:
-            #    command: |
-            #        source venv_test/bin/activate
-            #        python -m pip install -U "numpy>=1.21,<1.22"
-            #        python -m pip install -U .[test]
-            #        export _GRID2OP_FORCE_TEST=1
-            #        grid2op.testinstall
-            # - run:
-            #    command: |
-            #        source venv_test/bin/activate
-            #        python -m pip install -U "numpy>=1.22,<1.23"
-            #        python -m pip install -U .[test]
-            #        export _GRID2OP_FORCE_TEST=1
-            #        grid2op.testinstall
-            # - run:
-            #    command: |
-            #        source venv_test/bin/activate
-            #        python -m pip install -U "numpy>=1.23,<1.24"
-            #        python -m pip install -U .[test]
-            #        export _GRID2OP_FORCE_TEST=1
-            #        grid2op.testinstall
             - run:
                command: |
                    source venv_test/bin/activate
-                   python -m pip install -U "numpy>=1.24,<1.25"
+                   python -m pip install -U "numpy>=1.24,<1.25" "pandas<2.2" "scipy<1.12" numba
                    python -m pip install -U .[test]
+            - run:
+                command: |
+                   source venv_test/bin/activate
+                   pip freeze
+            - run:
+                command: |
+                   source venv_test/bin/activate
                    export _GRID2OP_FORCE_TEST=1
+                   cd /tmp
                    grid2op.testinstall
 
     install39:
@@ -187,59 +182,27 @@ jobs:
                    python -m pip install -U pip setuptools wheel
                    python -m pip install chronix2grid>="1.1.0.post1"
                    python -m pip uninstall -y grid2op
-            # - run:
-            #    command: |
-            #        source venv_test/bin/activate
-            #        python -m pip install -U numba
-            #        python -m pip install -U .[test]
-            # - run:
-            #    command: |
-            #        source venv_test/bin/activate
-            #        python -m pip install -U "numpy>=1.20,<1.21"
-            #        python -m pip install -U .[test]
-            #        export _GRID2OP_FORCE_TEST=1
-            #        grid2op.testinstall
-            # - run:
-            #    command: |
-            #        source venv_test/bin/activate
-            #        python -m pip install -U "numpy>=1.21,<1.22"
-            #        python -m pip install -U .[test]
-            #        export _GRID2OP_FORCE_TEST=1
-            #        grid2op.testinstall
-            # - run:
-            #    command: |
-            #        source venv_test/bin/activate
-            #        python -m pip install -U "numpy>=1.22,<1.23"
-            #        python -m pip install -U .[test]
-            #        export _GRID2OP_FORCE_TEST=1
-            #        grid2op.testinstall
-            # - run:
-            #    command: |
-            #        source venv_test/bin/activate
-            #        python -m pip install -U "numpy>=1.23,<1.24"
-            #        python -m pip install -U .[test]
-            #        export _GRID2OP_FORCE_TEST=1
-            #        grid2op.testinstall
-            # - run:
-            #    command: |
-            #        source venv_test/bin/activate
-            #        python -m pip install -U "numpy>=1.24,<1.25"
-            #        python -m pip install -U .[test]
-            #        export _GRID2OP_FORCE_TEST=1
-            #        grid2op.testinstall
-            # - run:
-            #    command: |
-            #        source venv_test/bin/activate
-            #        python -m pip install -U "numpy>=1.25,<1.26"
-            #        python -m pip install -U .[test]
-            #        export _GRID2OP_FORCE_TEST=1
-            #        grid2op.testinstall
             - run:
                command: |
                    source venv_test/bin/activate
-                   python -m pip install -U "numpy>=1.26,<1.27"
-                   python -m pip install -U .[test]
+                   python -m pip install -U "numpy>=1.20,<1.21" "pandas<2.2" "scipy<1.12" numba .[test]
+                   pip freeze
+            - run:
+                command: |
+                   source venv_test/bin/activate
+                   export _GRID2OP_FORCE_TEST=1
+                   cd /tmp
+                   grid2op.testinstall
+            - run:
+               command: |
+                   source venv_test/bin/activate
+                   python -m pip install -U "numpy>=1.26,<1.27" "pandas<2.2" "scipy<1.12" numba .[test]
+                   pip freeze
+            - run:
+                command: |
+                   source venv_test/bin/activate
                    export _GRID2OP_FORCE_TEST=1
+                   cd /tmp
                    grid2op.testinstall
 
     install310:
@@ -261,44 +224,24 @@ jobs:
             - run:
                command: |
                    source venv_test/bin/activate
-                   python -m pip install -U "numpy>=1.21,<1.22"
-                   python -m pip install -U .[test]
+                   python -m pip install -U "numpy>=1.21,<1.22" "pandas<2.2" "scipy<1.12" numba .[test]
+                   pip freeze
+            - run:
+                command: |
+                   source venv_test/bin/activate
                    export _GRID2OP_FORCE_TEST=1
+                   cd /tmp
                    grid2op.testinstall
-            # - run:
-            #    command: |
-            #        source venv_test/bin/activate
-            #        python -m pip install -U "numpy>=1.22,<1.23"
-            #        python -m pip install -U .[test]
-            #        export _GRID2OP_FORCE_TEST=1
-            #        grid2op.testinstall
-            # - run:
-            #    command: |
-            #        source venv_test/bin/activate
-            #        python -m pip install -U "numpy>=1.23,<1.24"
-            #        python -m pip install -U .[test]
-            #        export _GRID2OP_FORCE_TEST=1
-            #        grid2op.testinstall
-            # - run:
-            #    command: |
-            #        source venv_test/bin/activate
-            #        python -m pip install -U "numpy>=1.24,<1.25"
-            #        python -m pip install -U .[test]
-            #        export _GRID2OP_FORCE_TEST=1
-            #        grid2op.testinstall
-            # - run:
-            #    command: |
-            #        source venv_test/bin/activate
-            #        python -m pip install -U "numpy>=1.25,<1.26"
-            #        python -m pip install -U .[test]
-            #        export _GRID2OP_FORCE_TEST=1
-            #        grid2op.testinstall
             - run:
                command: |
                    source venv_test/bin/activate
-                   python -m pip install -U "numpy>=1.26,<1.27"
-                   python -m pip install -U .[test]
+                   python -m pip install -U "numpy>=1.26,<1.27" "pandas<2.2" "scipy<1.12" numba .[test]
+                   pip freeze
+            - run:
+                command: |
+                   source venv_test/bin/activate
                    export _GRID2OP_FORCE_TEST=1
+                   cd /tmp
                    grid2op.testinstall
 
     install311:
@@ -316,34 +259,27 @@ jobs:
                command: |
                    source venv_test/bin/activate
                    python -m pip install -U pip setuptools wheel
-                   python -m pip install -U numba
             - run:
                command: |
                    source venv_test/bin/activate
-                   python -m pip install -U "numpy>=1.23,<1.24"
-                   python -m pip install -U .[test]
+                   python -m pip install -U "numpy>=1.23,<1.24" "pandas<2.2" "scipy<1.12" numba .[test]
+                   pip freeze
+            - run:
+                command: |
+                   source venv_test/bin/activate
                    export _GRID2OP_FORCE_TEST=1
+                   cd /tmp
                    grid2op.testinstall
-            # - run:
-            #    command: |
-            #        source venv_test/bin/activate
-            #        python -m pip install -U "numpy>=1.24,<1.25"
-            #        python -m pip install -U .[test]
-            #        export _GRID2OP_FORCE_TEST=1
-            #        grid2op.testinstall
-            # - run:
-            #    command: |
-            #        source venv_test/bin/activate
-            #        python -m pip install -U "numpy>=1.25,<1.26"
-            #        python -m pip install -U .[test]
-            #        export _GRID2OP_FORCE_TEST=1
-            #        grid2op.testinstall
             - run:
                command: |
                    source venv_test/bin/activate
-                   python -m pip install -U "numpy>=1.26,<1.27"
-                   python -m pip install -U .[test]
+                   python -m pip install -U "numpy>=1.26,<1.27" "pandas<2.2" "scipy<1.12" numba .[test]
+                   pip freeze
+            - run:
+                command: |
+                   source venv_test/bin/activate
                    export _GRID2OP_FORCE_TEST=1
+                   cd /tmp
                    grid2op.testinstall
     install312:
         executor: python312
@@ -364,9 +300,13 @@ jobs:
             - run:
                command: |
                    source venv_test/bin/activate
-                   python -m pip install -U "numpy>=1.26,<1.27"
-                   python -m pip install -U .[test]
+                   python -m pip install -U "numpy>=1.26,<1.27" "pandas<2.2" "scipy<1.12" .[test]
+                   pip freeze
+            - run:
+                command: |
+                   source venv_test/bin/activate
                    export _GRID2OP_FORCE_TEST=1
+                   cd /tmp
                    grid2op.testinstall
 
 workflows:
@@ -380,4 +320,4 @@ workflows:
           - install39
           - install310
           - install311
-          - install312  # failing because of dependencies of numba, torch etc. Tired of it so ignoring it !
+          - install312
diff --git a/CHANGELOG.rst b/CHANGELOG.rst
@@ -40,8 +40,14 @@ Change Log
 - [FIXED] the "BaseBackendTest" class did not correctly detect divergence in most cases (which lead 
   to weird bugs in failing tests)
 - [FIXED] an issue with imageio having deprecated the `fps` kwargs (see https://github.com/rte-france/Grid2Op/issues/569)
+- [FIXED] adding the "`loads_charac.csv`" in the package data
+- [FIXED] a bug when using grid2op, not "utils.py" script could be used (see 
+  https://github.com/rte-france/Grid2Op/issues/577). This was caused by the modification of
+  `sys.path` when importing the grid2op test suite.
 - [ADDED] A type of environment that does not perform the "emulation of the protections"
   for some part of the grid (`MaskedEnvironment`) see https://github.com/rte-france/Grid2Op/issues/571
+- [ADDED] a "gym like" API for reset allowing to set the seed and the time serie id directly when calling
+  `env.reset(seed=.., options={"time serie id": ...})`
 - [IMPROVED] the CI speed: by not testing every possible numpy version but only most ancient and most recent
 - [IMPROVED] Runner now test grid2op version 1.9.6 and 1.9.7
 - [IMPROVED] refacto `gridobj_cls._clear_class_attribute` and `gridobj_cls._clear_grid_dependant_class_attributes`

diff --git a/MANIFEST.in b/MANIFEST.in
@@ -1,4 +1,4 @@
-recursive-include grid2op/data *.bz2 *.json *.zip prods_charac.csv *.py .multimix storage_units_charac.csv start_datetime.info time_interval.info
+recursive-include grid2op/data *.bz2 *.json *.zip loads_charac.csv prods_charac.csv *.py .multimix storage_units_charac.csv start_datetime.info time_interval.info
 global-exclude */__pycache__/*
 global-exclude *.pyc
 global-exclude  grid2op/data_test/*

diff --git a/README.md b/README.md
@@ -312,6 +312,10 @@ but it is currently not on our priorities.
 A quick fix that is known to work include to set the `experimental_read_from_local_dir` when creating the
 environment with `grid2op.make(..., experimental_read_from_local_dir=True)` (see doc for more information)
 
+Sometimes, on some configuration (python version) we do not recommend to use grid2op with pandas>=2.2
+If you encounter any trouble, please downgrade to pandas<2.2. This behaviour occured in our continuous 
+integration environment for python >=3.9 but could not be reproduced locally.
+
 ### Perform tests locally
 
 Provided that Grid2Op is installed *from source*:

diff --git a/docs/conf.py b/docs/conf.py
@@ -22,7 +22,7 @@
 author = 'Benjamin Donnot'
 
 # The full version, including alpha/beta/rc tags
-release = '1.9.8.dev0'
+release = '1.9.8.dev1'
 version = '1.9'
 
 

diff --git a/grid2op/Backend/pandaPowerBackend.py b/grid2op/Backend/pandaPowerBackend.py
@@ -21,7 +21,7 @@
 from grid2op.dtypes import dt_int, dt_float, dt_bool
 from grid2op.Backend.backend import Backend
 from grid2op.Action import BaseAction
-from grid2op.Exceptions import *
+from grid2op.Exceptions import BackendError
 
 try:
     import numba
@@ -118,6 +118,12 @@ def __init__(
         can_be_copied: bool=True,
         with_numba: bool=NUMBA_,
     ):
+        from grid2op.MakeEnv.Make import _force_test_dataset
+        if _force_test_dataset():
+            if with_numba:
+                warnings.warn(f"Forcing `test=True` will disable numba for {type(self)}")
+            with_numba = False
+
         Backend.__init__(
             self,
             detailed_infos_for_cascading_failures=detailed_infos_for_cascading_failures,
@@ -214,6 +220,8 @@ def __init__(
         self._lightsim2grid : bool = lightsim2grid
         self._dist_slack : bool = dist_slack
         self._max_iter : bool = max_iter
+        self._in_service_line_col_id = None
+        self._in_service_trafo_col_id = None
 
     def _check_for_non_modeled_elements(self):
         """This function check for elements in the pandapower grid that will have no impact on grid2op.
@@ -337,6 +345,9 @@ def load_grid(self,
             warnings.filterwarnings("ignore", category=FutureWarning)
             self._grid = pp.from_json(full_path)
         self._check_for_non_modeled_elements()
+
+        self._in_service_line_col_id = int(np.where(self._grid.line.columns == "in_service")[0][0])
+        self._in_service_trafo_col_id = int(np.where(self._grid.trafo.columns == "in_service")[0][0])
 
         # add the slack bus that is often not modeled as a generator, but i need it for this backend to work
         bus_gen_added = None
@@ -879,7 +890,7 @@ def apply_action(self, backendAction: Union["grid2op.Action._backendAction._Back
                 self._grid.shunt["in_service"].iloc[shunt_bus.changed] = sh_service           
                 chg_and_in_service = sh_service & shunt_bus.changed
                 self._grid.shunt["bus"].loc[chg_and_in_service] = cls.local_bus_to_global(shunt_bus.values[chg_and_in_service],
-                                                                                         cls.shunt_to_subid[chg_and_in_service])
+                                                                                          cls.shunt_to_subid[chg_and_in_service])
 
         # i made at least a real change, so i implement it in the backend
         for id_el, new_bus in topo__:
@@ -1292,6 +1303,9 @@ def copy(self) -> "PandaPowerBackend":
         res.load_theta = copy.deepcopy(self.load_theta)
         res.gen_theta = copy.deepcopy(self.gen_theta)
         res.storage_theta = copy.deepcopy(self.storage_theta)
+
+        res._in_service_line_col_id = self._in_service_line_col_id
+        res._in_service_trafo_col_id = self._in_service_trafo_col_id
 
         return res
 
@@ -1347,18 +1361,18 @@ def get_line_flow(self) -> np.ndarray:
 
     def _disconnect_line(self, id_):
         if id_ < self._number_true_line:
-            self._grid.line["in_service"].iloc[id_] = False
+            self._grid.line.iloc[id_, self._in_service_line_col_id] = False
         else:
-            self._grid.trafo["in_service"].iloc[id_ - self._number_true_line] = False
+            self._grid.trafo.iloc[id_ - self._number_true_line, self._in_service_trafo_col_id]  = False
         self._topo_vect[self.line_or_pos_topo_vect[id_]] = -1
         self._topo_vect[self.line_ex_pos_topo_vect[id_]] = -1
         self.line_status[id_] = False
 
     def _reconnect_line(self, id_):
         if id_ < self._number_true_line:
-            self._grid.line["in_service"].iloc[id_] = True
+            self._grid.line.iloc[id_, self._in_service_line_col_id] = True
         else:
-            self._grid.trafo["in_service"].iloc[id_ - self._number_true_line] = True
+            self._grid.trafo.iloc[id_ - self._number_true_line, self._in_service_trafo_col_id] = True
         self.line_status[id_] = True
 
     def get_topo_vect(self) -> np.ndarray: