Backport genkw fix

.github/workflows/test_ert_with_slurm.yml

@@ -13,7 +13,7 @@ jobs:
   test-ert-on-slurm:
     name: Run ert tests
     timeout-minutes: 20
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
 
     steps:
     - uses: actions/checkout@v4

src/ert/__main__.py

@@ -17,7 +17,7 @@
 
 import ert.shared
 from _ert.threading import set_signal_handler
-from ert.cli.main import ErtCliError, ErtTimeoutError, run_cli
+from ert.cli.main import ErtCliError, run_cli
 from ert.config import ConfigValidationError, ErtConfig, lint_file
 from ert.logging import LOGGING_CONFIG
 from ert.mode_definitions import (
@@ -681,12 +681,12 @@ def main() -> None:
         with ErtPluginContext(logger=logging.getLogger()) as context:
             logger.info(f"Running ert with {args}")
             args.func(args, context.plugin_manager)
-    except (ErtCliError, ErtTimeoutError) as err:
-        logger.exception(str(err))
+    except ErtCliError as err:
+        logger.debug(str(err))
         sys.exit(str(err))
     except ConfigValidationError as err:
         err_msg = err.cli_message()
-        logger.exception(err_msg)
+        logger.debug(err_msg)
         sys.exit(err_msg)
     except BaseException as err:
         logger.exception(f'ERT crashed unexpectedly with "{err}"')

src/ert/cli/main.py

@@ -34,10 +34,6 @@ class ErtCliError(Exception):
     pass
 
 
-class ErtTimeoutError(Exception):
-    pass
-
-
 def run_cli(args: Namespace, plugin_manager: Optional[ErtPluginManager] = None) -> None:
     ert_dir = os.path.abspath(os.path.dirname(args.config))
     os.chdir(ert_dir)

src/ert/config/ert_script.py

@@ -121,7 +121,7 @@ def initializeAndRun(
             arg_type = argument_types[index] if index < len(argument_types) else str
 
             if arg_value is not None:
-                arguments.append(arg_type(arg_value))  # type: ignore
+                arguments.append(arg_type(arg_value))
             else:
                 arguments.append(None)
         fixtures["workflow_args"] = arguments

src/ert/dark_storage/common.py

@@ -129,7 +129,7 @@ def data_for_key(
         dataframe.index.name = "Realization"
 
         data = dataframe.sort_index(axis=1)
-        if data.empty:
+        if data.empty or key not in data:
             return pd.DataFrame()
         data = data[key].to_frame().dropna()
         data.columns = pd.Index([0])

src/ert/gui/tools/plot/plot_window.py

@@ -106,7 +106,7 @@ def __init__(self, config_file: str, parent: Optional[QWidget]):
             self._api = PlotApi()
             self._key_definitions = self._api.all_data_type_keys()
         except (RequestError, TimeoutError) as e:
-            logger.exception(e)
+            logger.exception(f"plot api request failed: {e}")
             open_error_dialog("Request failed", str(e))
             self._key_definitions = []
         QApplication.restoreOverrideCursor()
@@ -191,7 +191,7 @@ def updatePlot(self, layer: Optional[int] = None) -> None:
                         ensemble.id, key
                     )
                 except (RequestError, TimeoutError) as e:
-                    logger.exception(e)
+                    logger.exception(f"plot api request failed: {e}")
                     open_error_dialog("Request failed", f"{e}")
 
             observations = None
@@ -201,7 +201,7 @@ def updatePlot(self, layer: Optional[int] = None) -> None:
                         [ensembles.id for ensembles in selected_ensembles], key
                     )
                 except (RequestError, TimeoutError) as e:
-                    logger.exception(e)
+                    logger.exception(f"plot api request failed: {e}")
                     open_error_dialog("Request failed", f"{e}")
 
             std_dev_images: Dict[str, npt.NDArray[np.float32]] = {}
@@ -221,7 +221,7 @@ def updatePlot(self, layer: Optional[int] = None) -> None:
                             key, ensemble.id, layer
                         )
                     except (RequestError, TimeoutError) as e:
-                        logger.exception(e)
+                        logger.exception(f"plot api request failed: {e}")
                         open_error_dialog("Request failed", f"{e}")
             else:
                 plot_widget.showLayerWidget.emit(False)
@@ -241,7 +241,7 @@ def updatePlot(self, layer: Optional[int] = None) -> None:
                     )
 
                 except (RequestError, TimeoutError) as e:
-                    logger.exception(e)
+                    logger.exception(f"plot api request failed: {e}")
                     open_error_dialog("Request failed", f"{e}")
                     plot_context.history_data = None
 

tests/unit_tests/gui/tools/plot/test_plot_api.py

@@ -1,12 +1,28 @@
+import gc
+from textwrap import dedent
+
 import httpx
 import pandas as pd
 import pytest
+import xarray as xr
 from pandas.testing import assert_frame_equal
-
-from ert.gui.tools.plot.plot_api import PlotApiKeyDefinition
+from starlette.testclient import TestClient
+
+from ert.config import GenKwConfig
+from ert.dark_storage import enkf
+from ert.dark_storage.app import app
+from ert.gui.tools.plot.plot_api import PlotApi, PlotApiKeyDefinition
+from ert.services import StorageService
+from ert.storage import open_storage
 from tests.unit_tests.gui.tools.plot.conftest import MockResponse
 
 
+@pytest.fixture(autouse=True)
+def use_testclient(monkeypatch):
+    client = TestClient(app)
+    monkeypatch.setattr(StorageService, "session", lambda: client)
+
+
 def test_key_def_structure(api):
     key_defs = api.all_data_type_keys()
     fopr = next(x for x in key_defs if x.key == "FOPR")
@@ -146,3 +162,87 @@ def test_plot_api_request_errors(api):
 
     with pytest.raises(httpx.RequestError):
         api.data_for_key(ensemble.id, "should_not_be_there")
+
+
+@pytest.fixture
+def api_and_storage(monkeypatch, tmp_path):
+    with open_storage(tmp_path / "storage", mode="w") as storage:
+        monkeypatch.setenv("ERT_STORAGE_NO_TOKEN", "yup")
+        monkeypatch.setenv("ERT_STORAGE_ENS_PATH", storage.path)
+        api = PlotApi()
+        yield api, storage
+    if enkf._storage is not None:
+        enkf._storage.close()
+    enkf._storage = None
+    gc.collect()
+
+
+def test_plot_api_handles_empty_gen_kw(api_and_storage):
+    api, storage = api_and_storage
+    key = "gen_kw"
+    name = "<poro>"
+    experiment = storage.create_experiment(
+        parameters=[
+            GenKwConfig(
+                name=key,
+                forward_init=False,
+                update=False,
+                template_file=None,
+                output_file=None,
+                transform_function_definitions=[],
+            ),
+        ],
+        responses=[],
+        observations={},
+    )
+    ensemble = storage.create_ensemble(experiment.id, ensemble_size=10)
+    assert api.data_for_key(str(ensemble.id), key).empty
+    ensemble.save_parameters(
+        key,
+        1,
+        xr.Dataset(
+            {
+                "values": ("names", [1.0]),
+                "transformed_values": ("names", [1.0]),
+                "names": [name],
+            }
+        ),
+    )
+    assert api.data_for_key(str(ensemble.id), key + ":" + name).to_csv() == dedent(
+        """\
+        Realization,0
+        1,1.0
+        """
+    )
+
+
+def test_plot_api_handles_non_existant_gen_kw(api_and_storage):
+    api, storage = api_and_storage
+    experiment = storage.create_experiment(
+        parameters=[
+            GenKwConfig(
+                name="gen_kw",
+                forward_init=False,
+                update=False,
+                template_file=None,
+                output_file=None,
+                transform_function_definitions=[],
+            ),
+        ],
+        responses=[],
+        observations={},
+    )
+    ensemble = storage.create_ensemble(experiment.id, ensemble_size=10)
+    ensemble.save_parameters(
+        "gen_kw",
+        1,
+        xr.Dataset(
+            {
+                "values": ("names", [1.0]),
+                "transformed_values": ("names", [1.0]),
+                "names": ["key"],
+            }
+        ),
+    )
+    assert api.data_for_key(str(ensemble.id), "gen_kw").empty
+    assert api.data_for_key(str(ensemble.id), "gen_kw:does_not_exist").empty