From f5dc1075cdc5e89ceaa1b4791b8a5c7b6bb16a1f Mon Sep 17 00:00:00 2001 From: mferrera Date: Mon, 6 Jan 2025 08:13:23 +0100 Subject: [PATCH] STY: Apply more linting rules This implements some additional linting rules and prepares a few others. It also adds the `--cov --cov-report term-missing` coverage report to be emitted at the end of test runs, which may be noisier but still useful to see. The test runs install the package with `-e` now so that the code lines missing test coverage are visible. --- .github/workflows/ci-fmudataio.yml | 4 +- docs/ext/pydantic_autosummary/generate.py | 5 +- .../iter-0/any/bin/export_grid3d.py | 2 +- .../iter-0/rms/bin/export_a_surface.py | 2 +- .../iter-0/rms/bin/export_faultpolygons.py | 2 +- .../rms/bin/export_faultroom_surfaces.py | 2 +- .../iter-0/rms/bin/export_propmaps.py | 2 +- .../iter-0/rms/bin/export_a_surface.py | 2 +- .../iter-0/rms/bin/export_a_surface.py | 2 +- pyproject.toml | 48 +++++++++++++------ src/fmu/dataio/aggregation.py | 11 +++-- .../export/rms/_conditional_rms_imports.py | 8 ++-- src/fmu/dataio/providers/objectdata/_base.py | 4 +- .../test_export_rms_volumetrics.py | 4 +- .../test_wf_copy_preprocessed_data.py | 2 +- tests/test_units/test_aggregated_surfaces.py | 2 +- tests/test_units/test_checksum_md5.py | 2 +- tests/test_units/test_dataio.py | 2 +- tests/test_units/test_dunder_methods.py | 12 ++--- tests/test_units/test_ert_context.py | 2 +- tests/test_units/test_fmuprovider_class.py | 2 +- .../test_objectdataprovider_class.py | 2 +- tests/test_units/test_preprocessed.py | 2 +- .../test_prerealization_surfaces.py | 2 +- tests/test_units/test_rms_context.py | 3 +- tests/test_units/test_utils.py | 19 ++++---- 26 files changed, 84 insertions(+), 66 deletions(-) diff --git a/.github/workflows/ci-fmudataio.yml b/.github/workflows/ci-fmudataio.yml index ef6757b74..b5fff5b3a 100644 --- a/.github/workflows/ci-fmudataio.yml +++ b/.github/workflows/ci-fmudataio.yml @@ -26,7 +26,7 @@ jobs: - name: Install fmu-dataio run: | pip install pip -U - pip install .[dev] + pip install -e .[dev] - name: Install fmu-sumo-uploader for Python 3.8 if: matrix.python-version == '3.8' @@ -42,7 +42,7 @@ jobs: "fmu-sumo-uploader @ git+https://github.com/equinor/fmu-sumo-uploader.git" - name: Full test - run: pytest -v + run: pytest -v --cov --cov-report term-missing docker_build: name: Build Docker image diff --git a/docs/ext/pydantic_autosummary/generate.py b/docs/ext/pydantic_autosummary/generate.py index 4c8e956e8..478117414 100644 --- a/docs/ext/pydantic_autosummary/generate.py +++ b/docs/ext/pydantic_autosummary/generate.py @@ -516,9 +516,8 @@ def _get_modules( if public_members is not None: if modname in public_members: public.append(fullname) - else: - if not modname.startswith("_"): - public.append(fullname) + elif not modname.startswith("_"): + public.append(fullname) return public, items diff --git a/examples/s/d/nn/xcase/realization-0/iter-0/any/bin/export_grid3d.py b/examples/s/d/nn/xcase/realization-0/iter-0/any/bin/export_grid3d.py index 54e65295d..ac748bbac 100644 --- a/examples/s/d/nn/xcase/realization-0/iter-0/any/bin/export_grid3d.py +++ b/examples/s/d/nn/xcase/realization-0/iter-0/any/bin/export_grid3d.py @@ -5,7 +5,7 @@ import xtgeo -import fmu.dataio as dataio +from fmu import dataio from fmu.config import utilities as ut logging.basicConfig(level=logging.DEBUG) diff --git a/examples/s/d/nn/xcase/realization-0/iter-0/rms/bin/export_a_surface.py b/examples/s/d/nn/xcase/realization-0/iter-0/rms/bin/export_a_surface.py index b9a517001..78c2a8c2a 100644 --- a/examples/s/d/nn/xcase/realization-0/iter-0/rms/bin/export_a_surface.py +++ b/examples/s/d/nn/xcase/realization-0/iter-0/rms/bin/export_a_surface.py @@ -10,7 +10,7 @@ import xtgeo -import fmu.dataio as dataio +from fmu import dataio from fmu.config import utilities as ut logger = logging.getLogger(__name__) diff --git a/examples/s/d/nn/xcase/realization-0/iter-0/rms/bin/export_faultpolygons.py b/examples/s/d/nn/xcase/realization-0/iter-0/rms/bin/export_faultpolygons.py index 798a6f833..21f6ffdd4 100644 --- a/examples/s/d/nn/xcase/realization-0/iter-0/rms/bin/export_faultpolygons.py +++ b/examples/s/d/nn/xcase/realization-0/iter-0/rms/bin/export_faultpolygons.py @@ -5,7 +5,7 @@ import xtgeo -import fmu.dataio as dataio +from fmu import dataio from fmu.config import utilities as utils logger = logging.getLogger(__name__) diff --git a/examples/s/d/nn/xcase/realization-0/iter-0/rms/bin/export_faultroom_surfaces.py b/examples/s/d/nn/xcase/realization-0/iter-0/rms/bin/export_faultroom_surfaces.py index f4f5bcebb..d8fa56410 100644 --- a/examples/s/d/nn/xcase/realization-0/iter-0/rms/bin/export_faultroom_surfaces.py +++ b/examples/s/d/nn/xcase/realization-0/iter-0/rms/bin/export_faultroom_surfaces.py @@ -3,7 +3,7 @@ import logging from pathlib import Path -import fmu.dataio as dataio +from fmu import dataio from fmu.config import utilities as utils logger = logging.getLogger(__name__) diff --git a/examples/s/d/nn/xcase/realization-0/iter-0/rms/bin/export_propmaps.py b/examples/s/d/nn/xcase/realization-0/iter-0/rms/bin/export_propmaps.py index e40c40088..df4aca3a2 100644 --- a/examples/s/d/nn/xcase/realization-0/iter-0/rms/bin/export_propmaps.py +++ b/examples/s/d/nn/xcase/realization-0/iter-0/rms/bin/export_propmaps.py @@ -16,7 +16,7 @@ import xtgeo -import fmu.dataio as dataio +from fmu import dataio from fmu.config import utilities as ut logger = logging.getLogger(__name__) diff --git a/examples/s/d/nn/xcase/realization-1/iter-0/rms/bin/export_a_surface.py b/examples/s/d/nn/xcase/realization-1/iter-0/rms/bin/export_a_surface.py index b9a517001..78c2a8c2a 100644 --- a/examples/s/d/nn/xcase/realization-1/iter-0/rms/bin/export_a_surface.py +++ b/examples/s/d/nn/xcase/realization-1/iter-0/rms/bin/export_a_surface.py @@ -10,7 +10,7 @@ import xtgeo -import fmu.dataio as dataio +from fmu import dataio from fmu.config import utilities as ut logger = logging.getLogger(__name__) diff --git a/examples/s/d/nn/xcase/realization-9/iter-0/rms/bin/export_a_surface.py b/examples/s/d/nn/xcase/realization-9/iter-0/rms/bin/export_a_surface.py index b9a517001..78c2a8c2a 100644 --- a/examples/s/d/nn/xcase/realization-9/iter-0/rms/bin/export_a_surface.py +++ b/examples/s/d/nn/xcase/realization-9/iter-0/rms/bin/export_a_surface.py @@ -10,7 +10,7 @@ import xtgeo -import fmu.dataio as dataio +from fmu import dataio from fmu.config import utilities as ut logger = logging.getLogger(__name__) diff --git a/pyproject.toml b/pyproject.toml index bb39a439b..ffb7def81 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -89,6 +89,12 @@ write_to = "src/fmu/dataio/version.py" minversion = "6.0" testpaths = "tests" +[tool.coverage.run] +omit = [ + "version.py", + "tests/**", +] + [tool.pydocstyle] convention = "google" match = '(?!(test_|_)).*\.py' @@ -96,24 +102,36 @@ match = '(?!(test_|_)).*\.py' [tool.ruff] line-length = 88 exclude = ["version.py"] + [tool.ruff.lint] -ignore = ["C901"] +ignore = [ + "B028", # No explicit `stacklevel` keyword argument found + "C901", # mccabe complex-structure +] select = [ - "C", - "E", - "F", - "I", - "PIE", - "Q", - "RET", - "RSE", - "SIM", - "W", - # "C90", - # "NPY", - # "PD", - # "PL", + "B", # flake-8-bugbear + "C", # pylint-convention + "C90", # mccabe + "E", # pycodestyle-error + "F", # pyflakes + "I", # isort + "NPY", # numpy + # "PD", # pandas-vet + "PIE", # flake8-pie + # "PL", # pylint + "Q", # flake8-quotes + "RET", # flake8-return + "RSE", # flake8-raise + "SIM", # flake8-simplify + # "TCH", # flake8-type-checking + # "TID", # flake8-tidy-imports + # "UP", # pyupgrade + "W", # pylint-warnings ] + +[tool.ruff.lint.pydocstyle] +convention = "google" + [tool.ruff.lint.isort] combine-as-imports = true diff --git a/src/fmu/dataio/aggregation.py b/src/fmu/dataio/aggregation.py index ad64b8d94..41d0841e3 100644 --- a/src/fmu/dataio/aggregation.py +++ b/src/fmu/dataio/aggregation.py @@ -226,9 +226,8 @@ def _set_metadata( if self.aggregation_id is None: self.aggregation_id = self._generate_aggr_uuid(uuids) - else: - if not isinstance(self.aggregation_id, str): - raise ValueError("aggregation_id must be a string") + elif not isinstance(self.aggregation_id, str): + raise ValueError("aggregation_id must be a string") if not self.operation: raise ValueError("The 'operation' key has no value") @@ -346,8 +345,10 @@ def generate_metadata( try: rid = conf["fmu"]["realization"]["id"] xuuid = conf["fmu"]["realization"]["uuid"] - except Exception as error: - raise ValidationError(f"Seems that input config are not valid: {error}") + except Exception as e: + raise ValidationError( + f"Seems that input config are not valid: {e}" + ) from e real_ids.append(rid) uuids.append(xuuid) diff --git a/src/fmu/dataio/export/rms/_conditional_rms_imports.py b/src/fmu/dataio/export/rms/_conditional_rms_imports.py index 28fa69a98..ab8be689b 100644 --- a/src/fmu/dataio/export/rms/_conditional_rms_imports.py +++ b/src/fmu/dataio/export/rms/_conditional_rms_imports.py @@ -19,7 +19,7 @@ def import_rms_package() -> tuple[Any, Any]: """ try: import rmsapi - import rmsapi.jobs as jobs + from rmsapi import jobs except ImportError: try: @@ -28,13 +28,13 @@ def import_rms_package() -> tuple[Any, Any]: "ignore", category=DeprecationWarning, module="roxar" ) import roxar as rmsapi - import roxar.jobs as jobs + from roxar import jobs - except ImportError: + except ImportError as e: raise ImportError( "Neither 'roxar' nor 'rmsapi' are available. You have to be inside " "RMS to use this function." - ) + ) from e return (rmsapi, jobs) diff --git a/src/fmu/dataio/providers/objectdata/_base.py b/src/fmu/dataio/providers/objectdata/_base.py index 418287d4e..cb965f39c 100644 --- a/src/fmu/dataio/providers/objectdata/_base.py +++ b/src/fmu/dataio/providers/objectdata/_base.py @@ -265,8 +265,8 @@ def _validate_get_ext(fmt: str, validator: ValidFormats) -> str: """Validate that fmt (file format) matches data and return legal extension.""" try: return validator.value[fmt] - except KeyError: + except KeyError as e: raise ConfigurationError( f"The file format {fmt} is not supported. ", f"Valid formats are: {list(validator.value.keys())}", - ) + ) from e diff --git a/tests/test_export_rms/test_export_rms_volumetrics.py b/tests/test_export_rms/test_export_rms_volumetrics.py index 16b02169d..01e71c19b 100644 --- a/tests/test_export_rms/test_export_rms_volumetrics.py +++ b/tests/test_export_rms/test_export_rms_volumetrics.py @@ -1,7 +1,7 @@ """Test the dataio running RMS spesici utility function for volumetrics""" -import unittest.mock as mock from pathlib import Path +from unittest import mock import jsonschema import numpy as np @@ -10,7 +10,7 @@ import pytest from pydantic import ValidationError -import fmu.dataio as dataio +from fmu import dataio from fmu.dataio._logging import null_logger from fmu.dataio._products.inplace_volumes import ( InplaceVolumesResult, diff --git a/tests/test_integration/test_wf_copy_preprocessed_data.py b/tests/test_integration/test_wf_copy_preprocessed_data.py index e1ecbf85e..59a0ec7f8 100644 --- a/tests/test_integration/test_wf_copy_preprocessed_data.py +++ b/tests/test_integration/test_wf_copy_preprocessed_data.py @@ -2,7 +2,7 @@ import pytest import yaml -import fmu.dataio as dataio +from fmu import dataio from .ert_config_utils import ( add_copy_preprocessed_workflow, diff --git a/tests/test_units/test_aggregated_surfaces.py b/tests/test_units/test_aggregated_surfaces.py index 81c6ac203..ad1581f36 100644 --- a/tests/test_units/test_aggregated_surfaces.py +++ b/tests/test_units/test_aggregated_surfaces.py @@ -7,7 +7,7 @@ import xtgeo import fmu.dataio._utils as utils -import fmu.dataio.dataio as dataio +from fmu.dataio import dataio logger = logging.getLogger(__name__) diff --git a/tests/test_units/test_checksum_md5.py b/tests/test_units/test_checksum_md5.py index ee0831705..c6449e975 100644 --- a/tests/test_units/test_checksum_md5.py +++ b/tests/test_units/test_checksum_md5.py @@ -1,6 +1,6 @@ from pathlib import Path -import fmu.dataio.readers as readers +from fmu.dataio import readers from fmu.dataio._utils import md5sum from fmu.dataio.dataio import ExportData, read_metadata diff --git a/tests/test_units/test_dataio.py b/tests/test_units/test_dataio.py index 809bee858..e89a4e362 100644 --- a/tests/test_units/test_dataio.py +++ b/tests/test_units/test_dataio.py @@ -714,7 +714,7 @@ def test_global_config_from_env(monkeypatch, global_config2_path, globalconfig1) monkeypatch.setenv("FMU_GLOBAL_CONFIG", str(global_config2_path)) edata = ExportData(content="depth") # the env variable will override this - assert getattr(edata.config.masterdata, "smda") + assert edata.config.masterdata.smda assert edata.config.model.name == "ff" # do not use global config from environment when explicitly given diff --git a/tests/test_units/test_dunder_methods.py b/tests/test_units/test_dunder_methods.py index e3e1cb151..3ed3c9847 100644 --- a/tests/test_units/test_dunder_methods.py +++ b/tests/test_units/test_dunder_methods.py @@ -39,18 +39,18 @@ def _fixture_testdata_stratigraphy() -> Stratigraphy: def test_stratigraphy_dunder_iter(testdata_stratigraphy): try: count = 0 - for item in testdata_stratigraphy: + for _ in testdata_stratigraphy: count += 1 assert count == 3 except Exception: - assert False, "Stratigraphy class does not have __iter__()" + pytest.fail("Stratigraphy class does not have __iter__()") def test_stratigraphy_dunder_getitem(testdata_stratigraphy): try: testdata_stratigraphy["TopStratUnit2"] except Exception: - assert False, "Stratigraphy class does not have __getitem__()" + pytest.fail("Stratigraphy class does not have __getitem__()") # -------------------------------------------------------------------------------------- @@ -80,15 +80,15 @@ def _fixture_testdata_parameters() -> Parameters: def test_parameters_dunder_iter(testdata_parameters): try: count = 0 - for item in testdata_parameters: + for _ in testdata_parameters: count += 1 assert count == 3 except Exception: - assert False, "Parameters class does not have __iter__()" + pytest.fail("Parameters class does not have __iter__()") def test_parameters_dunder_getitem(testdata_parameters): try: testdata_parameters["p2"] except Exception: - assert False, "Parameters class does not have __getitem__()" + pytest.fail("Parameters class does not have __getitem__()") diff --git a/tests/test_units/test_ert_context.py b/tests/test_units/test_ert_context.py index a5abe3dae..99186a3fe 100644 --- a/tests/test_units/test_ert_context.py +++ b/tests/test_units/test_ert_context.py @@ -14,7 +14,7 @@ import pytest import yaml -import fmu.dataio.dataio as dataio +from fmu.dataio import dataio from fmu.dataio._utils import prettyprint_dict logger = logging.getLogger(__name__) diff --git a/tests/test_units/test_fmuprovider_class.py b/tests/test_units/test_fmuprovider_class.py index 27a2d81cc..e13ff71fd 100644 --- a/tests/test_units/test_fmuprovider_class.py +++ b/tests/test_units/test_fmuprovider_class.py @@ -7,7 +7,7 @@ import pydantic import pytest -import fmu.dataio as dataio +from fmu import dataio # from conftest import pretend_ert_env_run1 from fmu.dataio._model.enums import ErtSimulationMode, FMUContext diff --git a/tests/test_units/test_objectdataprovider_class.py b/tests/test_units/test_objectdataprovider_class.py index a02a21994..37527046f 100644 --- a/tests/test_units/test_objectdataprovider_class.py +++ b/tests/test_units/test_objectdataprovider_class.py @@ -6,7 +6,7 @@ import pytest import yaml -import fmu.dataio as dataio +from fmu import dataio from fmu.dataio._definitions import ConfigurationError, ValidFormats from fmu.dataio._model.specification import FaultRoomSurfaceSpecification from fmu.dataio.providers.objectdata._faultroom import FaultRoomSurfaceProvider diff --git a/tests/test_units/test_preprocessed.py b/tests/test_units/test_preprocessed.py index 555f5867c..e04a65ae0 100644 --- a/tests/test_units/test_preprocessed.py +++ b/tests/test_units/test_preprocessed.py @@ -6,7 +6,7 @@ import pytest import yaml -import fmu.dataio as dataio +from fmu import dataio from fmu.dataio import _utils as utils from fmu.dataio.exceptions import InvalidMetadataError from fmu.dataio.providers._fmu import ERT_RELATIVE_CASE_METADATA_FILE diff --git a/tests/test_units/test_prerealization_surfaces.py b/tests/test_units/test_prerealization_surfaces.py index 34eef6a39..b351141ce 100644 --- a/tests/test_units/test_prerealization_surfaces.py +++ b/tests/test_units/test_prerealization_surfaces.py @@ -15,7 +15,7 @@ import pytest -import fmu.dataio as dataio +from fmu import dataio from fmu.dataio import _utils as utils from ..conftest import remove_ert_env, set_ert_env_prehook diff --git a/tests/test_units/test_rms_context.py b/tests/test_units/test_rms_context.py index e0a76b5c8..a365369d8 100644 --- a/tests/test_units/test_rms_context.py +++ b/tests/test_units/test_rms_context.py @@ -14,8 +14,7 @@ import xtgeo import yaml -import fmu.dataio.dataio as dataio -import fmu.dataio.readers as readers +from fmu.dataio import dataio, readers from fmu.dataio._utils import prettyprint_dict from fmu.dataio.dataio import ValidationError diff --git a/tests/test_units/test_utils.py b/tests/test_units/test_utils.py index 3fd3a2852..19781862f 100644 --- a/tests/test_units/test_utils.py +++ b/tests/test_units/test_utils.py @@ -50,12 +50,13 @@ def test_get_object_name(): assert utils.get_object_name(Polygons(name="poly")) is None assert utils.get_object_name(Polygons(name="Not poly")) == "Not poly" + rng = np.random.default_rng() assert ( utils.get_object_name( Grid( - np.random.randn(2, 2, 6).astype(np.float64), - np.random.randn(2, 2, 2, 4).astype(np.float32), - np.random.randn(1, 1, 1).astype(np.int32), + rng.standard_normal((2, 2, 6)).astype(np.float64), + rng.standard_normal((2, 2, 2, 4)).astype(np.float32), + rng.standard_normal((1, 1, 1)).astype(np.int32), ) ) is None @@ -63,9 +64,9 @@ def test_get_object_name(): assert ( utils.get_object_name( Grid( - np.random.randn(2, 2, 6).astype(np.float64), - np.random.randn(2, 2, 2, 4).astype(np.float32), - np.random.randn(1, 1, 1).astype(np.int32), + rng.standard_normal((2, 2, 6)).astype(np.float64), + rng.standard_normal((2, 2, 2, 4)).astype(np.float32), + rng.standard_normal((1, 1, 1)).astype(np.int32), name="noname", ) ) @@ -74,9 +75,9 @@ def test_get_object_name(): assert ( utils.get_object_name( Grid( - np.random.randn(2, 2, 6).astype(np.float64), - np.random.randn(2, 2, 2, 4).astype(np.float32), - np.random.randn(1, 1, 1).astype(np.int32), + rng.standard_normal((2, 2, 6)).astype(np.float64), + rng.standard_normal((2, 2, 2, 4)).astype(np.float32), + rng.standard_normal((1, 1, 1)).astype(np.int32), name="Not noname", ) )