Sync response config keys w/ actual response keys

src/ert/config/gen_data_config.py

@@ -22,6 +22,7 @@ class GenDataConfig(ResponseConfig):
     report_steps_list: List[Optional[List[int]]] = dataclasses.field(
         default_factory=list
     )
+    has_finalized_keys: bool = True
 
     def __post_init__(self) -> None:
         if len(self.report_steps_list) == 0:

src/ert/config/response_config.py

@@ -21,6 +21,7 @@ class ResponseConfig(ABC):
     name: str
     input_files: List[str] = dataclasses.field(default_factory=list)
     keys: List[str] = dataclasses.field(default_factory=list)
+    has_finalized_keys: bool = False
 
     @abstractmethod
     def read_from_file(self, run_path: str, iens: int) -> polars.DataFrame:

src/ert/config/summary_config.py

@@ -23,6 +23,7 @@
 class SummaryConfig(ResponseConfig):
     name: str = "summary"
     refcase: Union[Set[datetime], List[str], None] = None
+    has_finalized_keys = False
 
     def __post_init__(self) -> None:
         if isinstance(self.refcase, list):

src/ert/dark_storage/common.py

@@ -73,7 +73,13 @@ def ensemble_parameters(storage: Storage, ensemble_id: UUID) -> List[Dict[str, A
     return param_list
 
 
-def gen_data_keys(ensemble: Ensemble) -> Iterator[str]:
+def get_response_names(ensemble: Ensemble) -> List[str]:
+    result = ensemble.experiment.response_type_to_response_keys["summary"]
+    result.extend(sorted(gen_data_display_keys(ensemble), key=lambda k: k.lower()))
+    return result
+
+
+def gen_data_display_keys(ensemble: Ensemble) -> Iterator[str]:
     gen_data_config = ensemble.experiment.response_configuration.get("gen_data")
 
     if gen_data_config:
@@ -245,7 +251,7 @@ def get_observation_keys_for_response(
     Get all observation keys for given response key
     """
 
-    if displayed_response_key in gen_data_keys(ensemble):
+    if displayed_response_key in gen_data_display_keys(ensemble):
         response_key, report_step = displayed_key_to_response_key["gen_data"](
             displayed_response_key
         )
@@ -262,7 +268,10 @@ def get_observation_keys_for_response(
 
             return filtered["observation_key"].unique().to_list()
 
-    elif displayed_response_key in ensemble.get_summary_keyset():
+    elif (
+        displayed_response_key
+        in ensemble.experiment.response_type_to_response_keys["summary"]
+    ):
         response_key = displayed_key_to_response_key["summary"](displayed_response_key)[
             0
         ]

src/ert/dark_storage/endpoints/records.py

@@ -12,13 +12,14 @@
 from ert.dark_storage.common import (
     data_for_key,
     ensemble_parameters,
-    gen_data_keys,
+    gen_data_display_keys,
     get_observation_keys_for_response,
     get_observations_for_obs_keys,
     response_key_to_displayed_key,
 )
 from ert.dark_storage.enkf import get_storage
 from ert.storage import Storage
+from ert.storage.realization_storage_state import RealizationStorageState
 
 router = APIRouter(tags=["record"])
 
@@ -133,15 +134,23 @@ def get_ensemble_responses(
             )
             response_names_with_observations.update(set(obs_with_responses))
 
-    for name in ensemble.get_summary_keyset():
+    has_responses = any(
+        s == RealizationStorageState.HAS_DATA for s in ensemble.get_ensemble_state()
+    )
+
+    for name in (
+        ensemble.experiment.response_type_to_response_keys.get("summary", [])
+        if has_responses
+        else []
+    ):
         response_map[str(name)] = js.RecordOut(
             id=UUID(int=0),
             name=name,
             userdata={"data_origin": "Summary"},
             has_observations=name in response_names_with_observations,
         )
 
-    for name in gen_data_keys(ensemble):
+    for name in gen_data_display_keys(ensemble) if has_responses else []:
         response_map[str(name)] = js.RecordOut(
             id=UUID(int=0),
             name=name,

src/ert/storage/local_ensemble.py

@@ -29,7 +29,6 @@
 logger = logging.getLogger(__name__)
 
 import polars
-from polars.exceptions import ColumnNotFoundError
 
 
 class _Index(BaseModel):
@@ -311,11 +310,22 @@ def _has_response(_key: str) -> bool:
         if key:
             return _has_response(key)
 
-        return all(
-            _has_response(response)
-            for response in self.experiment.response_configuration
+        is_expecting_any_responses = any(
+            bool(config.keys)
+            for config in self.experiment.response_configuration.values()
         )
 
+        if not is_expecting_any_responses:
+            return True
+
+        non_empty_response_configs = [
+            response
+            for response, config in self.experiment.response_configuration.items()
+            if bool(config.keys)
+        ]
+
+        return all(_has_response(response) for response in non_empty_response_configs)
+
     def is_initalized(self) -> List[int]:
         """
         Return the realization numbers where all parameters are internalized. In
@@ -502,27 +512,6 @@ def _find_state(realization: int) -> RealizationStorageState:
 
         return [_find_state(i) for i in range(self.ensemble_size)]
 
-    def get_summary_keyset(self) -> List[str]:
-        """
-        Find the first folder with summary data then load the
-        summary keys from this.
-
-        Returns
-        -------
-        keys : list of str
-            List of summary keys.
-        """
-
-        try:
-            summary_data = self.load_responses(
-                "summary",
-                tuple(self.get_realization_list_with_responses("summary")),
-            )
-
-            return sorted(summary_data["response_key"].unique().to_list())
-        except (ValueError, KeyError, ColumnNotFoundError):
-            return []
-
     def _load_single_dataset(
         self,
         group: str,
@@ -696,8 +685,6 @@ def load_all_summary_data(
                 raise IndexError(f"No such realization {realization_index}")
             realizations = [realization_index]
 
-        summary_keys = self.get_summary_keyset()
-
         try:
             df_pl = self.load_responses("summary", tuple(realizations))
 
@@ -715,6 +702,7 @@ def load_all_summary_data(
         )
 
         if keys:
+            summary_keys = self.experiment.response_type_to_response_keys["summary"]
             summary_keys = sorted(
                 [key for key in keys if key in summary_keys]
             )  # ignore keys that doesn't exist
@@ -877,6 +865,10 @@ def save_response(
             output_path / f"{response_type}.parquet", data
         )
 
+        if not self.experiment._has_finalized_response_keys(response_type):
+            response_keys = data["response_key"].unique().to_list()
+            self.experiment._update_response_keys(response_type, response_keys)
+
     def calculate_std_dev_for_parameter(self, parameter_group: str) -> xr.Dataset:
         if parameter_group not in self.experiment.parameter_configuration:
             raise ValueError(f"{parameter_group} is not registered to the experiment.")

src/ert/storage/local_experiment.py

@@ -334,7 +334,67 @@ def observation_keys(self) -> List[str]:
     def response_key_to_response_type(self) -> Dict[str, str]:
         mapping = {}
         for config in self.response_configuration.values():
-            for key in config.keys:
+            for key in config.keys if config.has_finalized_keys else []:
                 mapping[key] = config.response_type
 
         return mapping
+
+    @cached_property
+    def response_type_to_response_keys(self) -> Dict[str, List[str]]:
+        result: Dict[str, List[str]] = {}
+
+        for response_key, response_type in self.response_key_to_response_type.items():
+            if response_type not in result:
+                result[response_type] = []
+
+            result[response_type].append(response_key)
+
+        for keys in result.values():
+            keys.sort()
+
+        return result
+
+    def _has_finalized_response_keys(self, response_type: str) -> bool:
+        responses_configuration = self.response_configuration
+        if response_type not in responses_configuration:
+            raise KeyError(
+                f"Response type {response_type} does not exist in current responses.json"
+            )
+
+        return responses_configuration[response_type].has_finalized_keys
+
+    def _update_response_keys(
+        self, response_type: str, response_keys: List[str]
+    ) -> None:
+        """
+        When a response is saved to storage, it may contain keys
+        that are not explicitly declared in the config. Calling this ensures
+        that the response config saved in this storage has keys corresponding
+        to the actual received responses.
+        """
+        responses_configuration = self.response_configuration
+        if response_type not in responses_configuration:
+            raise KeyError(
+                f"Response type {response_type} does not exist in current responses.json"
+            )
+
+        config = responses_configuration[response_type]
+        config.keys = sorted(response_keys)
+        config.has_finalized_keys = True
+        self._storage._write_transaction(
+            self._path / self._responses_file,
+            json.dumps(
+                {
+                    c.response_type: c.to_dict()
+                    for c in responses_configuration.values()
+                },
+                default=str,
+                indent=2,
+            ).encode("utf-8"),
+        )
+
+        if self.response_key_to_response_type is not None:
+            del self.response_key_to_response_type
+
+        if self.response_type_to_response_keys is not None:
+            del self.response_type_to_response_keys

src/everest/export.py

@@ -171,10 +171,12 @@ def get_internalized_keys(
 
             ensemble = experiment.get_ensemble_by_name(case_name)
             if not internal_keys:
-                internal_keys = set(ensemble.get_summary_keyset())
+                internal_keys = set(
+                    ensemble.experiment.response_type_to_response_keys["summary"]
+                )
             else:
                 internal_keys = internal_keys.intersection(
-                    set(ensemble.get_summary_keyset())
+                    set(ensemble.experiment.response_type_to_response_keys["summary"])
                 )
 
     return internal_keys

tests/ert/unit_tests/storage/snapshots/test_storage_migration/test_that_storage_matches/responses

@@ -1 +1 @@
-{'gen_data': GenDataConfig(name='gen_data', input_files=['gen%d.txt'], keys=['GEN'], report_steps_list=[[1]]), 'summary': SummaryConfig(name='summary', input_files=['CASE'], keys=['FOPR', 'RWPR'], refcase={})}
+{'gen_data': GenDataConfig(name='gen_data', input_files=['gen%d.txt'], keys=['GEN'], has_finalized_keys=True, report_steps_list=[[1]]), 'summary': SummaryConfig(name='summary', input_files=['CASE'], keys=['FOPR', 'RWPR'], has_finalized_keys=False, refcase={})}