From 57ec2932c3f8c7652682299a5090f543108a388a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?H=C3=A5vard=20Berland?= Date: Wed, 29 Jan 2020 13:11:25 +0100 Subject: [PATCH] Manifestbug (#86) * Move batch testing to designated test file and yaml-application * Fix manifest bug --- src/fmu/ensemble/virtualensemble.py | 8 +-- tests/test_batch.py | 89 +++++++++++++++++++++++++++++ tests/test_ensemble.py | 37 ------------ tests/test_virtualensemble.py | 5 ++ 4 files changed, 95 insertions(+), 44 deletions(-) create mode 100644 tests/test_batch.py diff --git a/src/fmu/ensemble/virtualensemble.py b/src/fmu/ensemble/virtualensemble.py index b7b364a9..2f154e84 100644 --- a/src/fmu/ensemble/virtualensemble.py +++ b/src/fmu/ensemble/virtualensemble.py @@ -72,15 +72,9 @@ def __init__( self.realindices = [] if manifest and not fromdisk: + # The _manifest variable is set using a property decorator self.manifest = manifest - if isinstance(manifest, dict): - self._manifest = manifest - else: - logger.warning( - "The manifest supplied to VirtualEnsemble " "must be of type dict" - ) - # At ensemble level, this dictionary has dataframes only. # All dataframes have the column REAL. if data: diff --git a/tests/test_batch.py b/tests/test_batch.py new file mode 100644 index 00000000..16e0ce0a --- /dev/null +++ b/tests/test_batch.py @@ -0,0 +1,89 @@ +# -*- coding: utf-8 -*- +"""Testing fmu-ensemble.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import os + +import yaml + +from fmu.ensemble import etc +from fmu.ensemble import ScratchEnsemble, EnsembleSet + +fmux = etc.Interaction() +logger = fmux.basiclogger(__name__, level="INFO") + +if not fmux.testsetup(): + raise SystemExit() + + +def test_batch(): + """Test batch processing at time of object initialization""" + if "__file__" in globals(): + # Easen up copying test code into interactive sessions + testdir = os.path.dirname(os.path.abspath(__file__)) + else: + testdir = os.path.abspath(".") + + ens = ScratchEnsemble( + "reektest", + testdir + "/data/testensemble-reek001/" + "realization-*/iter-0", + batch=[ + {"load_scalar": {"localpath": "npv.txt"}}, + {"load_smry": {"column_keys": "FOPT", "time_index": "yearly"}}, + {"load_smry": {"column_keys": "*", "time_index": "daily"}}, + ], + ) + assert len(ens.get_df("npv.txt")) == 5 + assert len(ens.get_df("unsmry--daily")["FOPR"]) == 5490 + assert len(ens.get_df("unsmry--yearly")["FOPT"]) == 25 + + # Also possible to batch process afterwards: + ens = ScratchEnsemble( + "reektest", testdir + "/data/testensemble-reek001/" + "realization-*/iter-0" + ) + ens.process_batch( + batch=[ + {"load_scalar": {"localpath": "npv.txt"}}, + {"load_smry": {"column_keys": "FOPT", "time_index": "yearly"}}, + {"load_smry": {"column_keys": "*", "time_index": "daily"}}, + ], + ) + assert len(ens.get_df("npv.txt")) == 5 + assert len(ens.get_df("unsmry--daily")["FOPR"]) == 5490 + assert len(ens.get_df("unsmry--yearly")["FOPT"]) == 25 + + +def test_yaml(): + """Test loading batch commands from yaml files""" + + # This is subject to change + + yamlstr = """ +scratch_ensembles: + iter1: data/testensemble-reek001/realization-*/iter-0 +batch: + - load_scalar: + localpath: npv.txt + - load_smry: + column_keys: FOPT + time_index: yearly + - load_smry: + column_keys: "*" + time_index: daily""" + ymlconfig = yaml.safe_load(yamlstr) + + testdir = os.path.dirname(os.path.abspath(__file__)) + os.chdir(testdir) + ensset = EnsembleSet() + + for ensname, enspath in ymlconfig["scratch_ensembles"].items(): + ensset.add_ensemble(ScratchEnsemble(ensname, paths=enspath)) + ensset.process_batch(ymlconfig["batch"]) + + assert "parameters.txt" in ensset.keys() + assert "OK" in ensset.keys() + assert "npv.txt" in ensset.keys() + assert not ensset.get_df("unsmry--yearly").empty diff --git a/tests/test_ensemble.py b/tests/test_ensemble.py index ee82c8c9..724ec18e 100644 --- a/tests/test_ensemble.py +++ b/tests/test_ensemble.py @@ -179,43 +179,6 @@ def test_reek001(tmp="TMP"): assert len(reekensemble.keys()) == keycount - 1 -def test_batch(): - """Test batch processing at time of object initialization""" - if "__file__" in globals(): - # Easen up copying test code into interactive sessions - testdir = os.path.dirname(os.path.abspath(__file__)) - else: - testdir = os.path.abspath(".") - - ens = ScratchEnsemble( - "reektest", - testdir + "/data/testensemble-reek001/" + "realization-*/iter-0", - batch=[ - {"load_scalar": {"localpath": "npv.txt"}}, - {"load_smry": {"column_keys": "FOPT", "time_index": "yearly"}}, - {"load_smry": {"column_keys": "*", "time_index": "daily"}}, - ], - ) - assert len(ens.get_df("npv.txt")) == 5 - assert len(ens.get_df("unsmry--daily")["FOPR"]) == 5490 - assert len(ens.get_df("unsmry--yearly")["FOPT"]) == 25 - - # Also possible to batch process afterwards: - ens = ScratchEnsemble( - "reektest", testdir + "/data/testensemble-reek001/" + "realization-*/iter-0" - ) - ens.process_batch( - batch=[ - {"load_scalar": {"localpath": "npv.txt"}}, - {"load_smry": {"column_keys": "FOPT", "time_index": "yearly"}}, - {"load_smry": {"column_keys": "*", "time_index": "daily"}}, - ], - ) - assert len(ens.get_df("npv.txt")) == 5 - assert len(ens.get_df("unsmry--daily")["FOPR"]) == 5490 - assert len(ens.get_df("unsmry--yearly")["FOPT"]) == 25 - - def test_emptyens(): """Check that we can initialize an empty ensemble""" ens = ScratchEnsemble("emptyens") diff --git a/tests/test_virtualensemble.py b/tests/test_virtualensemble.py index 581116a6..f7262ac5 100644 --- a/tests/test_virtualensemble.py +++ b/tests/test_virtualensemble.py @@ -45,6 +45,11 @@ def test_virtualensemble(): assert "coordinate_system" in vens.manifest + # Overwrite the manifest: + vens.manifest = {"foo": "bar"} + assert "foo" in vens.manifest + assert "coordinate_system" not in vens.manifest + # Check that we have data for 5 realizations assert len(vens["unsmry--yearly"]["REAL"].unique()) == 5 assert len(vens["parameters.txt"]) == 5