Add run experiment with design matrix to ensemble experiment panel

- Prefil active realization box with realizations from design matrix
- Use design_matrix parameters in ensemble experiment
- add test run cli with design matrix and poly example
- add test that save parameters internalize DataFrame parameters in the storage
- add merge function to merge design parameters with existing parameters

src/ert/config/design_matrix.py

@@ -11,15 +11,10 @@
 from ert.config.gen_kw_config import GenKwConfig, TransformFunctionDefinition
 
 from ._option_dict import option_dict
-from .parsing import (
-    ConfigValidationError,
-    ErrorInfo,
-)
+from .parsing import ConfigValidationError, ErrorInfo
 
 if TYPE_CHECKING:
-    from ert.config import (
-        ParameterConfig,
-    )
+    from ert.config import ParameterConfig
 
 DESIGN_MATRIX_GROUP = "DESIGN_MATRIX"
 
@@ -74,6 +69,58 @@ def from_config_list(cls, config_list: List[str]) -> "DesignMatrix":
             default_sheet=default_sheet,
         )
 
+    def merge_with_existing_parameters(
+        self, existing_parameters: List[ParameterConfig]
+    ) -> tuple[List[ParameterConfig], ParameterConfig | None]:
+        """
+        This method merges the design matrix parameters with the existing parameters and
+        returns the new list of existing parameters, wherein we drop GEN_KW group having a full overlap with the design matrix group.
+        GEN_KW group that was dropped will acquire a new name from the design matrix group.
+        Additionally, the ParameterConfig which is the design matrix group is returned separately.
+
+        Args:
+            existing_parameters (List[ParameterConfig]): List of existing parameters
+
+        Raises:
+            ConfigValidationError: If there is a partial overlap between the design matrix group and any existing GEN_KW group
+
+        Returns:
+            tuple[List[ParameterConfig], ParameterConfig]: List of existing parameters and the dedicated design matrix group
+        """
+        if self.parameter_configuration is None:
+            self.read_design_matrix()
+
+        if self.parameter_configuration is None or not isinstance(
+            self.parameter_configuration[DESIGN_MATRIX_GROUP], GenKwConfig
+        ):
+            return existing_parameters, None
+
+        new_param_config: List[ParameterConfig] = []
+
+        design_parameter_group = self.parameter_configuration[DESIGN_MATRIX_GROUP]
+        design_keys = []
+        if isinstance(design_parameter_group, GenKwConfig):
+            design_keys = design_parameter_group.getKeyWords()
+
+        design_group_added = False
+        for genkw_group in existing_parameters:
+            if not isinstance(genkw_group, GenKwConfig):
+                new_param_config += [genkw_group]
+                continue
+            existing_keys = genkw_group.getKeyWords()
+            if set(existing_keys) == set(design_keys):
+                design_parameter_group.name = genkw_group.name
+                design_group_added = True
+            elif set(design_keys) & set(existing_keys):
+                raise ConfigValidationError(
+                    "Overlapping parameter names found in design matrix!"
+                )
+            else:
+                new_param_config += [genkw_group]
+        if not design_group_added:
+            new_param_config += [design_parameter_group]
+        return new_param_config, design_parameter_group
+
     def read_design_matrix(
         self,
     ) -> None:

src/ert/config/gen_kw_config.py

@@ -34,6 +34,7 @@
 
     from ert.storage import Ensemble
 
+
 _logger = logging.getLogger(__name__)
 
 

src/ert/enkf_main.py

@@ -19,20 +19,17 @@
 )
 
 import orjson
+import pandas as pd
+import xarray as xr
 from numpy.random import SeedSequence
 
 from ert.config.ert_config import forward_model_data_to_json
 from ert.config.forward_model_step import ForwardModelStep
 from ert.config.model_config import ModelConfig
 from ert.substitutions import Substitutions
 
-from .config import (
-    ExtParamConfig,
-    Field,
-    GenKwConfig,
-    ParameterConfig,
-    SurfaceConfig,
-)
+from .config import ExtParamConfig, Field, GenKwConfig, ParameterConfig, SurfaceConfig
+from .config.design_matrix import DESIGN_MATRIX_GROUP
 from .run_arg import RunArg
 from .runpaths import Runpaths
 
@@ -162,6 +159,29 @@ def _seed_sequence(seed: Optional[int]) -> int:
     return int_seed
 
 
+def save_design_matrix_to_ensemble(
+    design_matrix_df: pd.DataFrame,
+    ensemble: Ensemble,
+    active_realizations: Iterable[int],
+    design_group_name: str = DESIGN_MATRIX_GROUP,
+) -> None:
+    assert not design_matrix_df.empty
+    for realization_nr in active_realizations:
+        row = design_matrix_df.loc[realization_nr][DESIGN_MATRIX_GROUP]
+        ds = xr.Dataset(
+            {
+                "values": ("names", list(row.values)),
+                "transformed_values": ("names", list(row.values)),
+                "names": list(row.keys()),
+            }
+        )
+        ensemble.save_parameters(
+            design_group_name,
+            realization_nr,
+            ds,
+        )
+
+
 def sample_prior(
     ensemble: Ensemble,
     active_realizations: Iterable[int],

src/ert/gui/simulation/ensemble_experiment_panel.py

@@ -15,7 +15,7 @@
 from ert.gui.tools.design_matrix.design_matrix_panel import DesignMatrixPanel
 from ert.mode_definitions import ENSEMBLE_EXPERIMENT_MODE
 from ert.run_models import EnsembleExperiment
-from ert.validation import RangeStringArgument
+from ert.validation import ActiveRange, RangeStringArgument
 from ert.validation.proper_name_argument import ExperimentValidation, ProperNameArgument
 
 from .experiment_config_panel import ExperimentConfigPanel
@@ -85,6 +85,13 @@ def __init__(
 
         design_matrix = analysis_config.design_matrix
         if design_matrix is not None:
+            if design_matrix.design_matrix_df is None:
+                design_matrix.read_design_matrix()
+
+            if design_matrix.active_realizations:
+                self._active_realizations_field.setText(
+                    ActiveRange(design_matrix.active_realizations).rangestring
+                )
             show_dm_param_button = QPushButton("Show parameters")
             show_dm_param_button.setObjectName("show-dm-parameters")
             show_dm_param_button.setMinimumWidth(50)

src/ert/run_models/ensemble_experiment.py

@@ -6,7 +6,7 @@
 
 import numpy as np
 
-from ert.enkf_main import sample_prior
+from ert.enkf_main import sample_prior, save_design_matrix_to_ensemble
 from ert.ensemble_evaluator import EvaluatorServerConfig
 from ert.storage import Ensemble, Experiment, Storage
 from ert.trace import tracer
@@ -63,10 +63,26 @@ def run_experiment(
         restart: bool = False,
     ) -> None:
         self.log_at_startup()
+        # If design matrix is present, we try to merge design matrix parameters
+        # to the experiment parameters and set new active realizations
+        parameters_config = self.ert_config.ensemble_config.parameter_configuration
+        design_matrix = self.ert_config.analysis_config.design_matrix
+        design_matrix_group = None
+        if design_matrix is not None:
+            parameters_config, design_matrix_group = (
+                design_matrix.merge_with_existing_parameters(parameters_config)
+            )
+
+            assert design_matrix.active_realizations is not None
+            self.active_realizations = design_matrix.active_realizations
         if not restart:
             self.experiment = self._storage.create_experiment(
                 name=self.experiment_name,
-                parameters=self.ert_config.ensemble_config.parameter_configuration,
+                parameters=(
+                    [*parameters_config, design_matrix_group]
+                    if design_matrix_group is not None
+                    else parameters_config
+                ),
                 observations=self.ert_config.observations,
                 responses=self.ert_config.ensemble_config.response_configuration,
             )
@@ -89,12 +105,25 @@ def run_experiment(
             np.array(self.active_realizations, dtype=bool),
             ensemble=self.ensemble,
         )
+
         sample_prior(
             self.ensemble,
             np.where(self.active_realizations)[0],
             random_seed=self.random_seed,
         )
 
+        if (
+            design_matrix_group is not None
+            and design_matrix is not None
+            and design_matrix.design_matrix_df is not None
+        ):
+            save_design_matrix_to_ensemble(
+                design_matrix.design_matrix_df,
+                self.ensemble,
+                np.where(self.active_realizations)[0],
+                design_matrix_group.name,
+            )
+
         self._evaluate_and_postprocess(
             run_args,
             self.ensemble,

src/ert/storage/local_experiment.py

@@ -12,12 +12,7 @@
 import xtgeo
 from pydantic import BaseModel
 
-from ert.config import (
-    ExtParamConfig,
-    Field,
-    GenKwConfig,
-    SurfaceConfig,
-)
+from ert.config import ExtParamConfig, Field, GenKwConfig, SurfaceConfig
 from ert.config.parsing.context_values import ContextBoolEncoder
 from ert.config.response_config import ResponseConfig
 from ert.storage.mode import BaseMode, Mode, require_write

test-data/ert/poly_design/.ert_runpath_list

@@ -0,0 +1,10 @@
+000  /data/workspace/ert/test-data/ert/poly_design/poly_out/realization-0/iter-0  poly.ert-0  000
+001  /data/workspace/ert/test-data/ert/poly_design/poly_out/realization-1/iter-0  poly.ert-1  000
+002  /data/workspace/ert/test-data/ert/poly_design/poly_out/realization-2/iter-0  poly.ert-2  000
+003  /data/workspace/ert/test-data/ert/poly_design/poly_out/realization-3/iter-0  poly.ert-3  000
+004  /data/workspace/ert/test-data/ert/poly_design/poly_out/realization-4/iter-0  poly.ert-4  000
+005  /data/workspace/ert/test-data/ert/poly_design/poly_out/realization-5/iter-0  poly.ert-5  000
+006  /data/workspace/ert/test-data/ert/poly_design/poly_out/realization-6/iter-0  poly.ert-6  000
+007  /data/workspace/ert/test-data/ert/poly_design/poly_out/realization-7/iter-0  poly.ert-7  000
+008  /data/workspace/ert/test-data/ert/poly_design/poly_out/realization-8/iter-0  poly.ert-8  000
+009  /data/workspace/ert/test-data/ert/poly_design/poly_out/realization-9/iter-0  poly.ert-9  000

test-data/ert/poly_design/POLY_EVAL

@@ -0,0 +1 @@
+EXECUTABLE poly_eval.py

test-data/ert/poly_design/coeff_priors

@@ -0,0 +1,3 @@
+a UNIFORM 0 1
+b UNIFORM 0 2
+c UNIFORM 0 5

test-data/ert/poly_design/observations

@@ -0,0 +1,5 @@
+GENERAL_OBSERVATION POLY_OBS {
+   DATA       = POLY_RES;
+   INDEX_LIST = 0,2,4,6,8;
+   OBS_FILE   = poly_obs_data.txt;
+};

test-data/ert/poly_design/poly.ert

@@ -0,0 +1,10 @@
+QUEUE_OPTION LOCAL MAX_RUNNING 10
+RUNPATH poly_out/realization-<IENS>/iter-<ITER>
+OBS_CONFIG observations
+NUM_REALIZATIONS 10
+MIN_REALIZATIONS 1
+GEN_DATA POLY_RES RESULT_FILE:poly.out
+DESIGN_MATRIX poly_design.xlsx DESIGN_SHEET:DesignSheet01 DEFAULT_SHEET:DefaultSheet
+GEN_KW COEFFS coeff_priors
+INSTALL_JOB poly_eval POLY_EVAL
+FORWARD_MODEL poly_eval

test-data/ert/poly_design/poly_eval.py

@@ -0,0 +1,18 @@
+#!/usr/bin/env python
+import json
+
+
+def _load_coeffs(filename):
+    with open(filename, encoding="utf-8") as f:
+        return json.load(f)["COEFFS"]
+
+
+def _evaluate(coeffs, x):
+    return coeffs["a"] * x**2 + coeffs["b"] * x + coeffs["c"]
+
+
+if __name__ == "__main__":
+    coeffs = _load_coeffs("parameters.json")
+    output = [_evaluate(coeffs, x) for x in range(10)]
+    with open("poly.out", "w", encoding="utf-8") as f:
+        f.write("\n".join(map(str, output)))

test-data/ert/poly_design/poly_obs_data.txt

@@ -0,0 +1,5 @@
+2.1457049781272213 0.6
+8.769219841380755 1.4
+12.388014786122742 3.0
+25.600464531354252 5.4
+42.35204755970952 8.6

tests/ert/ui_tests/cli/analysis/test_design_matrix.py

@@ -0,0 +1,93 @@
+import os
+import stat
+from textwrap import dedent
+
+import numpy as np
+import pandas as pd
+import pytest
+
+from ert.config import ErtConfig
+from ert.mode_definitions import ENSEMBLE_EXPERIMENT_MODE
+from ert.storage import open_storage
+from tests.ert.ui_tests.cli.run_cli import run_cli
+
+
+@pytest.mark.usefixtures("copy_poly_case")
+def test_run_poly_example_with_design_matrix():
+    design_matrix = "poly_design.xlsx"
+    num_realizations = 10
+    a_values = list(range(num_realizations))
+    design_matrix_df = pd.DataFrame(
+        {
+            "REAL": list(range(num_realizations)),
+            "a": a_values,
+        }
+    )
+    default_sheet_df = pd.DataFrame([["b", 1], ["c", 2]])
+    with pd.ExcelWriter(design_matrix) as xl_write:
+        design_matrix_df.to_excel(xl_write, index=False, sheet_name="DesignSheet01")
+        default_sheet_df.to_excel(
+            xl_write, index=False, sheet_name="DefaultSheet", header=False
+        )
+
+    with open("poly.ert", "w", encoding="utf-8") as fout:
+        fout.write(
+            dedent(
+                """\
+                QUEUE_OPTION LOCAL MAX_RUNNING 10
+                RUNPATH poly_out/realization-<IENS>/iter-<ITER>
+                NUM_REALIZATIONS 10
+                MIN_REALIZATIONS 1
+                GEN_DATA POLY_RES RESULT_FILE:poly.out
+                DESIGN_MATRIX poly_design.xlsx DESIGN_SHEET:DesignSheet01 DEFAULT_SHEET:DefaultSheet
+                INSTALL_JOB poly_eval POLY_EVAL
+                FORWARD_MODEL poly_eval
+                """
+            )
+        )
+
+    with open("poly_eval.py", "w", encoding="utf-8") as f:
+        f.write(
+            dedent(
+                """\
+                #!/usr/bin/env python
+                import numpy as np
+                import sys
+                import json
+
+                def _load_coeffs(filename):
+                    with open(filename, encoding="utf-8") as f:
+                        return json.load(f)["DESIGN_MATRIX"]
+
+                def _evaluate(coeffs, x):
+                    return coeffs["a"] * x**2 + coeffs["b"] * x + coeffs["c"]
+
+                if __name__ == "__main__":
+                    coeffs = _load_coeffs("parameters.json")
+                    output = [_evaluate(coeffs, x) for x in range(10)]
+                    with open("poly.out", "w", encoding="utf-8") as f:
+                        f.write("\\n".join(map(str, output)))
+                """
+            )
+        )
+    os.chmod(
+        "poly_eval.py",
+        os.stat("poly_eval.py").st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH,
+    )
+
+    run_cli(
+        ENSEMBLE_EXPERIMENT_MODE,
+        "--disable-monitor",
+        "poly.ert",
+        "--experiment-name",
+        "test-experiment",
+    )
+    storage_path = ErtConfig.from_file("poly.ert").ens_path
+    with open_storage(storage_path) as storage:
+        experiment = storage.get_experiment_by_name("test-experiment")
+        params = experiment.get_ensemble_by_name("default").load_parameters(
+            "DESIGN_MATRIX"
+        )["values"]
+        np.testing.assert_array_equal(params[:, 0], a_values)
+        np.testing.assert_array_equal(params[:, 1], 10 * [1])
+        np.testing.assert_array_equal(params[:, 2], 10 * [2])