diff --git a/.github/workflows/ci-fmudataio.yml b/.github/workflows/ci-fmudataio.yml index b5fff5b3a..5745edb23 100644 --- a/.github/workflows/ci-fmudataio.yml +++ b/.github/workflows/ci-fmudataio.yml @@ -2,7 +2,7 @@ name: Test on: pull_request: - branches: [main] + branches: [main, staging] schedule: - cron: "0 0 * * *" @@ -42,7 +42,12 @@ jobs: "fmu-sumo-uploader @ git+https://github.com/equinor/fmu-sumo-uploader.git" - name: Full test - run: pytest -v --cov --cov-report term-missing + run: | + if [[ "${{ github.event.pull_request.base.ref }}" == "staging" ]]; then + pytest -n auto -v --cov --cov-report term-missing --prod + else + pytest -n auto -v --cov --cov-report term-missing + fi docker_build: name: Build Docker image diff --git a/.github/workflows/schemas-up-to-date.yml b/.github/workflows/schemas-up-to-date.yml index b3c702eef..68b68529d 100644 --- a/.github/workflows/schemas-up-to-date.yml +++ b/.github/workflows/schemas-up-to-date.yml @@ -28,7 +28,11 @@ jobs: - name: Check schema run: | - ./tools/update-schema --diff + if [[ "${{ github.event.pull_request.base.ref }}" == "staging" ]]; then + ./tools/update-schema --diff --prod + else + ./tools/update-schema --diff + fi - name: Ensure schema validates with AJV run: | diff --git a/pyproject.toml b/pyproject.toml index 19d11af34..1ec894638 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -57,10 +57,11 @@ dev = [ "pyarrow-stubs", "pydocstyle", "PyQt5-sip<12.16.0; python_version == '3.8'", + "pytest", "pytest-cov", "pytest-mock", "pytest-runner", - "pytest", + "pytest-xdist", "rstcheck", "ruff", "types-PyYAML", diff --git a/src/fmu/dataio/_models/_schema_base.py b/src/fmu/dataio/_models/_schema_base.py index 790cdd20e..47d10337b 100644 --- a/src/fmu/dataio/_models/_schema_base.py +++ b/src/fmu/dataio/_models/_schema_base.py @@ -138,9 +138,9 @@ def url(cls) -> str: DEV_URL = f"{FmuSchemas.DEV_URL}/{cls.PATH}" PROD_URL = f"{FmuSchemas.PROD_URL}/{cls.PATH}" - if os.environ.get("SCHEMA_RELEASE", False): - return PROD_URL - return DEV_URL + if os.environ.get("DEV_SCHEMA", False): + return DEV_URL + return PROD_URL @classmethod def default_generator(cls) -> type[GenerateJsonSchema]: diff --git a/tests/conftest.py b/tests/conftest.py index 42fb47288..23cdf5804 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -5,6 +5,7 @@ import logging import os import shutil +import sys from copy import deepcopy from pathlib import Path @@ -14,6 +15,12 @@ import xtgeo import yaml +# This must be set before anything in dataio is imported if not using pytest-xdist. +if "--prod" in sys.argv: + os.environ["DEV_SCHEMA"] = "" +else: + os.environ["DEV_SCHEMA"] = "1" + import fmu.dataio as dio from fmu.config import utilities as ut from fmu.dataio._models.fmu_results import FmuResults, fields, global_configuration @@ -45,6 +52,21 @@ ERT_RUNPATH = f"_ERT_{FmuEnv.RUNPATH.name}" +def pytest_addoption(parser): + parser.addoption( + "--prod", action="store_true", help="Use schemas/metadata with production URLs." + ) + + +def pytest_configure(config) -> None: + """If '--prod' is given to pytest, use schemas/metadata with prod urls (i.e. not dev + urls).""" + if config.getoption("--prod"): + os.environ["DEV_SCHEMA"] = "" + else: + os.environ["DEV_SCHEMA"] = "1" + + def _current_function_name(): """Helper to retrieve current function name, e.g. for logging""" return inspect.currentframe().f_back.f_code.co_name diff --git a/tests/test_schema/test_schemas_up_to_date.py b/tests/test_schema/test_schemas_up_to_date.py index 1094ddebc..e88877e18 100644 --- a/tests/test_schema/test_schemas_up_to_date.py +++ b/tests/test_schema/test_schemas_up_to_date.py @@ -35,7 +35,12 @@ def test_schemas_uptodate(schema: SchemaBase) -> None: the local schema with the output of `dump()`. To get more feedback or generate new schemas run: - `./tools/update_schema` + + ./tools/update_schema --diff + + If you are generating a production release try running: + + ./tools/update_schema --diff --prod """ with open(schema.PATH) as f: assert json.load(f) == schema.dump() @@ -45,11 +50,17 @@ def test_schemas_uptodate(schema: SchemaBase) -> None: def test_schema_url_changes_with_env_var( schema: SchemaBase, monkeypatch: MonkeyPatch ) -> None: - assert schema.url().startswith(FmuSchemas.DEV_URL) - assert schema.dump()["$id"].startswith(FmuSchemas.DEV_URL) - monkeypatch.setenv("SCHEMA_RELEASE", "1") + monkeypatch.setenv("DEV_SCHEMA", "") + json = schema.dump() assert schema.url().startswith(FmuSchemas.PROD_URL) - assert schema.dump()["$id"].startswith(FmuSchemas.PROD_URL) + assert json["$id"].startswith(FmuSchemas.PROD_URL) + assert json["$schema"] == "https://json-schema.org/draft/2020-12/schema" + + monkeypatch.setenv("DEV_SCHEMA", "1") + json = schema.dump() + assert schema.url().startswith(FmuSchemas.DEV_URL) + assert json["$id"].startswith(FmuSchemas.DEV_URL) + assert json["$schema"] == "https://json-schema.org/draft/2020-12/schema" @pytest.mark.parametrize("schema", schemas) diff --git a/tools/update-schema b/tools/update-schema index 7c5857f54..41a6b5653 100755 --- a/tools/update-schema +++ b/tools/update-schema @@ -21,10 +21,10 @@ import subprocess import sys from copy import deepcopy from pathlib import Path -from typing import Any, Dict, List, TypeVar +from typing import TYPE_CHECKING, Any, Dict, List, TypeVar -from fmu.dataio._models import schemas -from fmu.dataio._models._schema_base import SchemaBase +if TYPE_CHECKING: + from fmu.dataio._models._schema_base import SchemaBase GREEN = "\033[32m" RED = "\033[31m" @@ -47,6 +47,12 @@ def _get_parser() -> argparse.ArgumentParser: action="store_true", help="Show a diff between the current schema and the new one in output.", ) + parser.add_argument( + "-p", + "--prod", + action="store_true", + help="Produce schemas with production URLs", + ) parser.add_argument( "-t", "--test", @@ -183,13 +189,21 @@ def write_schema( _show_py_diff(existing_schema, new_schema) return False - if is_release: + if ( + is_release + and not force_overwrite + and existing_schema["$id"] != new_schema["$id"] + ): print( - INFO, - f"{BOLD}{schema.FILENAME}{NC} version {BOLD}{schema.VERSION}{NC}: " - f"modifying '$id' url to 'prod':\n {schema.url()}", + FAIL, + f"🚨 {BOLD}{schema.FILENAME}{NC} version " + f"{BOLD}{schema.VERSION}{NC}: mismatch in '$id'. you probably need " + "to run `./tools/update-schema --prod --force`", ) - elif new_schema == existing_schema: + if show_diff: + _show_py_diff(existing_schema, new_schema) + return False + if new_schema == existing_schema: print( PASS, f"{BOLD}{schema.FILENAME}{NC} version " @@ -219,11 +233,13 @@ def main() -> None: if args.force: print(INFO, "forcing overwrite of all schemas") - is_release = bool(os.environ.get("SCHEMA_RELEASE", False)) + os.environ["DEV_SCHEMA"] = "" if args.prod else "1" + # Ensures URLs will differ based on above + import fmu.dataio._models as models failed_a_write = False - for schema in schemas: - did_write = write_schema(schema, args.force, is_release, args.test, args.diff) + for schema in models.schemas: + did_write = write_schema(schema, args.force, args.prod, args.test, args.diff) if not did_write: failed_a_write = True