From fced371bf74cff17c12791d26e21e15d09ef613a Mon Sep 17 00:00:00 2001 From: wpbonelli Date: Tue, 17 Dec 2024 08:58:29 -0500 Subject: [PATCH] use devtools dfn utils, convert to toml before codegen, remove dfn attr from generated classes --- autotest/test_codegen.py | 37 +- docs/mf6_dev_guide.md | 2 +- etc/environment.yml | 1 - flopy/mf6/utils/codegen/__init__.py | 11 +- flopy/mf6/utils/codegen/context.py | 14 +- flopy/mf6/utils/codegen/dfn.py | 587 ---------------------------- flopy/mf6/utils/codegen/dfn2toml.py | 54 --- flopy/mf6/utils/codegen/filters.py | 79 ++-- flopy/mf6/utils/generate_classes.py | 10 +- pyproject.toml | 2 - 10 files changed, 65 insertions(+), 732 deletions(-) delete mode 100644 flopy/mf6/utils/codegen/dfn.py delete mode 100644 flopy/mf6/utils/codegen/dfn2toml.py diff --git a/autotest/test_codegen.py b/autotest/test_codegen.py index 5248a9045..d931cbf84 100644 --- a/autotest/test_codegen.py +++ b/autotest/test_codegen.py @@ -1,28 +1,33 @@ import pytest +from modflow_devtools.dfn import get_dfns +from modflow_devtools.dfn2toml import convert from autotest.conftest import get_project_root_path from flopy.mf6.utils.codegen import make_all -from flopy.mf6.utils.codegen.dfn import Dfn PROJ_ROOT = get_project_root_path() MF6_PATH = PROJ_ROOT / "flopy" / "mf6" -DFN_PATH = MF6_PATH / "data" / "dfn" -DFN_NAMES = [ - dfn.stem for dfn in DFN_PATH.glob("*.dfn") if dfn.stem not in ["common", "flopy"] -] +DFN_PATH = PROJ_ROOT / "autotest" / "temp" / "dfn" +TOML_PATH = DFN_PATH / "toml" +MF6_OWNER = "MODFLOW-USGS" +MF6_REPO = "modflow6" +MF6_REF = "develop" -@pytest.mark.parametrize("dfn_name", DFN_NAMES) -def test_dfn_load(dfn_name): - with ( - open(DFN_PATH / "common.dfn", "r") as common_file, - open(DFN_PATH / f"{dfn_name}.dfn", "r") as dfn_file, - ): - name = Dfn.Name.parse(dfn_name) - common, _ = Dfn._load_v1_flat(common_file) - Dfn.load(dfn_file, name=name, common=common) +def pytest_generate_tests(metafunc): + if not any(DFN_PATH.glob("*.dfn")): + get_dfns(MF6_OWNER, MF6_REPO, MF6_REF, DFN_PATH, verbose=True) + convert(DFN_PATH, TOML_PATH) + dfns = list(DFN_PATH.glob("*.dfn")) + assert all( + (TOML_PATH / f"{dfn.stem}.toml").is_file() + for dfn in dfns + if "common" not in dfn.stem + ) -def test_make_all(function_tmpdir): - make_all(DFN_PATH, function_tmpdir, verbose=True) + +@pytest.mark.parametrize("version,dfn_path", [(1, DFN_PATH), (2, TOML_PATH)]) +def test_make_all(function_tmpdir, version, dfn_path): + make_all(dfn_path, function_tmpdir, verbose=True, version=version) assert any(function_tmpdir.glob("*.py")) diff --git a/docs/mf6_dev_guide.md b/docs/mf6_dev_guide.md index 456266ac4..e4e7bcf4d 100644 --- a/docs/mf6_dev_guide.md +++ b/docs/mf6_dev_guide.md @@ -30,7 +30,7 @@ The latter is typically used with e.g. `python -m flopy.mf6.utils.generate_class Generated files are created in `flopy/mf6/modflow/` and contain interface classes, one file/class per input component. These can be used to initialize and access model/package data as well as the input specification itself. -**Note**: Code generation requires a few extra dependencies, grouped in the `codegen` optional dependency group: `Jinja2`, `boltons`, `tomlkit` and `modflow-devtools`. +**Note**: Code generation requires a few extra dependencies, grouped in the `codegen` optional dependency group: `Jinja2` and `modflow-devtools`. **Note**: Code generation scripts previously used `flopy/mf6/data/mfstructure.py` to read and represent definition files, and wrote Python by hand. They now use the `flopy.mf6.utils.codegen` module, which uses Jinja2. diff --git a/etc/environment.yml b/etc/environment.yml index 12f5775b7..6507d7d29 100644 --- a/etc/environment.yml +++ b/etc/environment.yml @@ -15,7 +15,6 @@ dependencies: - Jinja2>=3.0 - pip: - git+https://github.com/MODFLOW-USGS/modflow-devtools.git - - tomlkit # lint - cffconvert diff --git a/flopy/mf6/utils/codegen/__init__.py b/flopy/mf6/utils/codegen/__init__.py index a11ffbf42..42016091f 100644 --- a/flopy/mf6/utils/codegen/__init__.py +++ b/flopy/mf6/utils/codegen/__init__.py @@ -28,13 +28,10 @@ def _get_template_env(): env.filters["prefix"] = Filters.Cls.prefix env.filters["parent"] = Filters.Cls.parent env.filters["skip_init"] = Filters.Cls.skip_init - env.filters["attrs"] = Filters.Vars.attrs env.filters["init"] = Filters.Vars.init - env.filters["untag"] = Filters.Var.untag env.filters["type"] = Filters.Var.type - env.filters["safe_name"] = Filters.safe_name env.filters["escape_trailing_underscore"] = ( Filters.escape_trailing_underscore @@ -84,7 +81,7 @@ def _get_template_name(ctx_name) -> str: if ctx_name.l == "exg": return "exchange.py.jinja" return "package.py.jinja" - + for context in Context.from_dfn(dfn): name = context["name"] target_path = outdir / f"mf{Filters.Cls.title(name)}.py" @@ -96,12 +93,12 @@ def _get_template_name(ctx_name) -> str: print(f"Wrote {target_path}") -def make_all(dfndir: Path, outdir: PathLike, verbose: bool = False): +def make_all(dfndir: Path, outdir: PathLike, verbose: bool = False, version: int = 1): """Generate Python source files from the DFN files in the given location.""" - from flopy.mf6.utils.codegen.dfn import Dfn + from modflow_devtools.dfn import Dfn - dfns = Dfn.load_all(dfndir) + dfns = Dfn.load_all(dfndir, version=version) make_init(dfns, outdir, verbose) for dfn in dfns.values(): make_targets(dfn, outdir, verbose) diff --git a/flopy/mf6/utils/codegen/context.py b/flopy/mf6/utils/codegen/context.py index 61c334b83..3cbfb9b96 100644 --- a/flopy/mf6/utils/codegen/context.py +++ b/flopy/mf6/utils/codegen/context.py @@ -6,7 +6,7 @@ TypedDict, ) -from flopy.mf6.utils.codegen.dfn import Dfn, Vars +from modflow_devtools.dfn import Dfn, Vars class Context(TypedDict): @@ -42,17 +42,17 @@ def from_dfn(dfn: Dfn) -> List["Context.Name"]: Returns a list of context names this definition produces. An input definition may produce one or more input contexts. """ - name = dfn["name"] - if name.r == "nam": - if name.l == "sim": + name = dfn["name"].split("-") + if name[1] == "nam": + if name[0] == "sim": return [ - Context.Name(None, name.r), # nam pkg + Context.Name(None, name[1]), # nam pkg Context.Name(*name), # simulation ] else: return [ Context.Name(*name), # nam pkg - Context.Name(name.l, None), # model + Context.Name(name[0], None), # model ] elif name in [ ("gwf", "mvr"), @@ -62,7 +62,7 @@ def from_dfn(dfn: Dfn) -> List["Context.Name"]: # TODO: deduplicate mfmvr.py/mfgwfmvr.py etc and remove special cases return [ Context.Name(*name), - Context.Name(None, name.r), + Context.Name(None, name[1]), ] return [Context.Name(*name)] diff --git a/flopy/mf6/utils/codegen/dfn.py b/flopy/mf6/utils/codegen/dfn.py deleted file mode 100644 index 7717c41be..000000000 --- a/flopy/mf6/utils/codegen/dfn.py +++ /dev/null @@ -1,587 +0,0 @@ -from ast import literal_eval -from collections.abc import Mapping -from os import PathLike -from typing import ( - Any, - Dict, - List, - Literal, - NamedTuple, - Optional, - Tuple, - TypedDict, -) -from warnings import warn - -from boltons.dictutils import OMD - -_SCALARS = { - "keyword", - "integer", - "double precision", - "string", -} - - -def _try_literal_eval(value: str) -> Any: - """ - Try to parse a string as a literal. If this fails, - return the value unaltered. - """ - try: - return literal_eval(value) - except (SyntaxError, ValueError): - return value - - -def _try_parse_bool(value: Any) -> Any: - """ - Try to parse a boolean from a string as represented - in a DFN file, otherwise return the value unaltered. - """ - if isinstance(value, str): - value = value.lower() - if value in ["true", "false"]: - return value == "true" - return value - - -Vars = Dict[str, "Var"] -Refs = Dict[str, "Ref"] -Dfns = Dict[str, "Dfn"] - - -class Var(TypedDict): - """An input variable specification.""" - - name: str - type: str - shape: Optional[Any] = None - block: Optional[str] = None - default: Optional[Any] = None - children: Optional["Vars"] = None - description: Optional[str] = None - - -class Ref(TypedDict): - """ - This class is used to represent subpackage references: - a foreign-key-like reference between a file input variable - and another input definition. This allows an input context - to refer to another input context by including a filepath - variable as a foreign key. The former's `__init__` method - is modified such that the variable named `val` replaces - the `key` variable. - """ - - key: str - val: str - abbr: str - param: str - parent: str - description: Optional[str] - - -class Sln(TypedDict): - abbr: str - pattern: str - - -DfnFmtVersion = Literal[1] -"""DFN format version number.""" - - -class Dfn(TypedDict): - """ - MODFLOW 6 input definition. An input definition - file specifies a component of an MF6 simulation, - e.g. a model or package. - """ - - class Name(NamedTuple): - """ - Uniquely identifies an input definition. - Consists of a left term and a right term. - """ - - l: str - r: str - - @classmethod - def parse(cls, v: str) -> "Dfn.Name": - try: - return cls(*v.split("-")) - except: - raise ValueError(f"Bad DFN name format: {v}") - - def __str__(self) -> str: - return "-".join(self) - - name: Name - vars: Vars - - @staticmethod - def _load_v1_flat( - f, common: Optional[dict] = None - ) -> Tuple[Mapping, List[str]]: - var = dict() - flat = list() - meta = list() - common = common or dict() - - for line in f: - # remove whitespace/etc from the line - line = line.strip() - - # record context name and flopy metadata - # attributes, skip all other comment lines - if line.startswith("#"): - _, sep, tail = line.partition("flopy") - if sep == "flopy": - if ( - "multi-package" in tail - or "solution_package" in tail - or "subpackage" in tail - or "parent" in tail - ): - meta.append(tail.strip()) - _, sep, tail = line.partition("package-type") - if sep == "package-type": - meta.append(f"package-type {tail.strip()}") - continue - - # if we hit a newline and the parameter dict - # is nonempty, we've reached the end of its - # block of attributes - if not any(line): - if any(var): - flat.append((var["name"], var)) - var = dict() - continue - - # split the attribute's key and value and - # store it in the parameter dictionary - key, _, value = line.partition(" ") - if key == "default_value": - key = "default" - var[key] = value - - # make substitutions from common variable definitions, - # remove backslashes, TODO: generate/insert citations. - descr = var.get("description", None) - if descr: - descr = ( - descr.replace("\\", "") - .replace("``", "'") - .replace("''", "'") - ) - _, replace, tail = descr.strip().partition("REPLACE") - if replace: - key, _, subs = tail.strip().partition(" ") - subs = literal_eval(subs) - cvar = common.get(key, None) - if cvar is None: - warn( - "Can't substitute description text, " - f"common variable not found: {key}" - ) - else: - descr = cvar.get("description", "") - if any(subs): - descr = descr.replace("\\", "").replace( - "{#1}", subs["{#1}"] - ) - var["description"] = descr - - # add the final parameter - if any(var): - flat.append((var["name"], var)) - - # the point of the OMD is to losslessly handle duplicate variable names - return OMD(flat), meta - - @classmethod - def _load_v1(cls, f, name, **kwargs) -> "Dfn": - """ - Temporary load routine for the v1 DFN format. - This can go away once we convert to v2 (TOML). - """ - - # if we have any subpackage references - # we need to watch for foreign key vars - # (file input vars) and register matches - refs = kwargs.pop("refs", dict()) - fkeys = dict() - - # load dfn as flat multidict and str metadata - flat, meta = Dfn._load_v1_flat(f, **kwargs) - - # pass the original dfn representation on - # the dfn since it is reproduced verbatim - # in generated classes for now. drop this - # later when we figure out how to unravel - # mfstructure.py etc - def _meta(): - meta_ = list() - for m in meta: - if "multi" in m: - meta_.append(m) - elif "solution" in m: - s = m.split() - meta_.append([s[0], s[2]]) - elif "package-type" in m: - s = m.split() - meta_.append(" ".join(s)) - return meta_ - - dfn = list(flat.values(multi=True)), _meta() - - def _load_variable(var: Dict[str, Any]) -> Var: - """ - Convert an input variable from its original representation - in a definition file to a structured, Python-friendly form. - - This involves trimming unneeded attributes and setting - some others. - - Notes - ----- - If a variable does not have a `default` attribute, it will - default to `False` if it is a keyword, otherwise to `None`. - - A filepath variable whose name functions as a foreign key - for a separate context will be given a reference to it. - """ - - # parse booleans from strings. everything else can - # stay a string except default values, which we'll - # try to parse as arbitrary literals below, and at - # some point types, once we introduce type hinting - var = {k: _try_parse_bool(v) for k, v in var.items()} - - _name = var["name"] - _type = var.get("type", None) - shape = var.get("shape", None) - shape = None if shape == "" else shape - block = var.get("block", None) - children = dict() - default = var.get("default", None) - default = ( - _try_literal_eval(default) if _type != "string" else default - ) - description = var.get("description", "") - ref = refs.get(_name, None) - - # if var is a foreign key, register it - if ref: - fkeys[_name] = ref - - def _items() -> Vars: - """Load a list's children (items: record or union of records).""" - - names = _type.split()[1:] - types = [ - v["type"] - for v in flat.values(multi=True) - if v["name"] in names and v.get("in_record", False) - ] - n_names = len(names) - if n_names < 1: - raise ValueError(f"Missing recarray definition: {_type}") - - # list input can have records or unions as rows. lists - # that have a consistent item type can be considered - # tabular. lists that can possess multiple item types - # (unions) are considered irregular. regular lists can - # be defined with a nested record (explicit) or with a - # set of fields directly in the recarray (implicit). an - # irregular list is always defined with a nested union. - is_explicit = n_names == 1 and ( - types[0].startswith("record") - or types[0].startswith("keystring") - ) - - if is_explicit: - child = next(iter(flat.getlist(names[0]))) - return {names[0]: _load_variable(child)} - elif all(t in _SCALARS for t in types): - # implicit simple record (all fields are scalars) - fields = _fields() - return { - _name: Var( - name=_name, - type="record", - block=block, - children=fields, - description=description.replace( - "is the list of", "is the record of" - ), - ) - } - else: - # implicit complex record (some fields are records or unions) - fields = { - v["name"]: _load_variable(v) - for v in flat.values(multi=True) - if v["name"] in names and v.get("in_record", False) - } - first = list(fields.values())[0] - single = len(fields) == 1 - name_ = first["name"] if single else _name - child_type = ( - "union" - if single and "keystring" in first["type"] - else "record" - ) - return { - name_: Var( - name=name_, - type=child_type, - block=block, - children=first["children"] if single else fields, - description=description.replace( - "is the list of", f"is the {child_type} of" - ), - ) - } - - def _choices() -> Vars: - """Load a union's children (choices).""" - names = _type.split()[1:] - return { - v["name"]: _load_variable(v) - for v in flat.values(multi=True) - if v["name"] in names and v.get("in_record", False) - } - - def _fields() -> Vars: - """Load a record's children (fields).""" - names = _type.split()[1:] - fields = dict() - for name in names: - v = flat.get(name, None) - if ( - not v - or not v.get("in_record", False) - or v["type"].startswith("record") - ): - continue - fields[name] = v - return fields - - if _type.startswith("recarray"): - children = _items() - _type = "list" - - elif _type.startswith("keystring"): - children = _choices() - _type = "union" - - elif _type.startswith("record"): - children = _fields() - _type = "record" - - # for now, we can tell a var is an array if its type - # is scalar and it has a shape. once we have proper - # typing, this can be read off the type itself. - elif shape is not None and _type not in _SCALARS: - raise TypeError(f"Unsupported array type: {_type}") - - # if var is a foreign key, return subpkg var instead - if ref: - return Var( - name=ref["param" if name == ("sim", "nam") else "val"], - type=_type, - shape=shape, - block=block, - children=None, - description=( - f"Contains data for the {ref['abbr']} package. Data can be " - f"stored in a dictionary containing data for the {ref['abbr']} " - "package with variable names as keys and package data as " - f"values. Data just for the {ref['val']} variable is also " - f"acceptable. See {ref['abbr']} package documentation for more " - "information" - ), - default=None, - subpackage=ref, - ) - - return Var( - name=_name, - type=_type, - shape=shape, - block=block, - children=children, - description=description, - default=default, - ) - - # load top-level variables. any nested - # variables will be loaded recursively - vars_ = { - var["name"]: _load_variable(var) - for var in flat.values(multi=True) - if not var.get("in_record", False) - } - - def _package_type() -> Optional[str]: - line = next( - iter( - m - for m in meta - if isinstance(m, str) and m.startswith("package-type") - ), - None, - ) - return line.split()[-1] if line else None - - def _subpackage() -> Optional["Ref"]: - def _parent(): - line = next( - iter( - m - for m in meta - if isinstance(m, str) and m.startswith("parent") - ), - None, - ) - if not line: - return None - split = line.split() - return split[1] - - def _rest(): - line = next( - iter( - m - for m in meta - if isinstance(m, str) and m.startswith("subpac") - ), - None, - ) - if not line: - return None - _, key, abbr, param, val = line.split() - matches = [v for v in vars_.values() if v["name"] == val] - if not any(matches): - descr = None - else: - if len(matches) > 1: - warn(f"Multiple matches for referenced variable {val}") - match = matches[0] - descr = match["description"] - - return { - "key": key, - "val": val, - "abbr": abbr, - "param": param, - "description": descr, - } - - parent = _parent() - rest = _rest() - if parent and rest: - return Ref(parent=parent, **rest) - return None - - def _solution() -> Optional[Sln]: - sln = next( - iter( - m - for m in meta - if isinstance(m, str) and m.startswith("solution_package") - ), - None, - ) - if sln: - abbr, pattern = sln.split()[1:] - return Sln(abbr=abbr, pattern=pattern) - return None - - def _multi() -> bool: - return any("multi-package" in m for m in meta) - - package_type = _package_type() - subpackage = _subpackage() - solution = _solution() - multi = _multi() - - return cls( - name=name, - foreign_keys=fkeys, - package_type=package_type, - subpackage=subpackage, - solution=solution, - multi=multi, - vars=vars_, - dfn=dfn, - ) - - @classmethod - def load( - cls, - f, - name: Optional[Name] = None, - version: DfnFmtVersion = 1, - **kwargs, - ) -> "Dfn": - """ - Load an input definition from a DFN file. - """ - - if version == 1: - return cls._load_v1(f, name, **kwargs) - else: - raise ValueError( - f"Unsupported version, expected one of {version.__args__}" - ) - - @staticmethod - def _load_all_v1(dfndir: PathLike) -> Dfns: - # find definition files - paths = [ - p - for p in dfndir.glob("*.dfn") - if p.stem not in ["common", "flopy"] - ] - - # try to load common variables - common_path = dfndir / "common.dfn" - if not common_path.is_file: - common = None - else: - with open(common_path, "r") as f: - common, _ = Dfn._load_v1_flat(f) - - # load subpackage references first - refs: Refs = {} - for path in paths: - with open(path) as f: - name = Dfn.Name.parse(path.stem) - dfn = Dfn.load(f, name=name, common=common) - subpkg = dfn.get("subpackage", None) - if subpkg: - refs[subpkg["key"]] = subpkg - - # load all the input definitions - dfns: Dfns = {} - for path in paths: - with open(path) as f: - name = Dfn.Name.parse(path.stem) - dfn = Dfn.load(f, name=name, common=common, refs=refs) - dfns[name] = dfn - - return dfns - - @staticmethod - def load_all(dfndir: PathLike, version: DfnFmtVersion = 1) -> Dfns: - """Load all input definitions from the given directory.""" - - if version == 1: - return Dfn._load_all_v1(dfndir) - else: - raise ValueError( - f"Unsupported version, expected one of {version.__args__}" - ) diff --git a/flopy/mf6/utils/codegen/dfn2toml.py b/flopy/mf6/utils/codegen/dfn2toml.py deleted file mode 100644 index 8a18021a3..000000000 --- a/flopy/mf6/utils/codegen/dfn2toml.py +++ /dev/null @@ -1,54 +0,0 @@ -import argparse -from collections.abc import Mapping -from pathlib import Path - -from flopy.utils import import_optional_dependency - -_MF6_PATH = Path(__file__).parents[2] -_DFN_PATH = _MF6_PATH / "data" / "dfn" -_TOML_PATH = _MF6_PATH / "data" / "toml" - - -def _drop_none(d): - if isinstance(d, Mapping): - return { - k: _drop_none(v) - for k, v in d.items() - if (v or isinstance(v, bool)) - } - else: - return d - - -def _shim(d): - del d["dfn"] - del d["foreign_keys"] - d["name"] = str(d["name"]) - return d - - -if __name__ == "__main__": - """Convert DFN files to TOML.""" - - from flopy.mf6.utils.codegen.dfn import Dfn - - tomlkit = import_optional_dependency("tomlkit") - parser = argparse.ArgumentParser(description="Convert DFN files to TOML.") - parser.add_argument( - "--dfndir", - type=str, - default=_DFN_PATH, - help="Directory containing DFN files.", - ) - parser.add_argument( - "--outdir", - default=_TOML_PATH, - help="Output directory.", - ) - args = parser.parse_args() - dfndir = Path(args.dfndir) - outdir = Path(args.outdir) - outdir.mkdir(exist_ok=True, parents=True) - for dfn in Dfn.load_all(dfndir).values(): - with open(Path(outdir) / f"{dfn['name']}.toml", "w") as f: - tomlkit.dump(_drop_none(_shim(dfn)), f) diff --git a/flopy/mf6/utils/codegen/filters.py b/flopy/mf6/utils/codegen/filters.py index 9d19f18d6..439c436ed 100644 --- a/flopy/mf6/utils/codegen/filters.py +++ b/flopy/mf6/utils/codegen/filters.py @@ -5,8 +5,6 @@ from jinja2 import pass_context -from flopy.mf6.utils.codegen.dfn import _SCALARS - def try_get_enum_value(v: Any) -> Any: """ @@ -77,9 +75,9 @@ def parent(ctx, ctx_name) -> str: if ctx_name == ("sim", "nam"): return None elif ( - ctx_name.l is None - or ctx_name.r is None - or ctx_name.l in ["sim", "exg", "sln"] + ctx_name[0] is None + or ctx_name[1] is None + or ctx_name[0] in ["sim", "exg", "sln"] ): return "simulation" return "model" @@ -99,7 +97,7 @@ def skip_init(ctx, ctx_name) -> List[str]: elif base == "MFModel": return ["packages", "export_netcdf", "nc_filerecord"] else: - if ctx_name.r == "nam": + if ctx_name[1] == "nam": return ["export_netcdf", "nc_filerecord"] elif ctx_name == ("utl", "ts"): return ["method", "interpolation_method_single", "sfac"] @@ -183,6 +181,8 @@ def attrs(ctx, variables) -> List[str]: of just a class attr for each variable, with anything complicated happening in a decorator or base class. """ + from modflow_devtools.dfn import _MF6_SCALARS + name = ctx["name"] base = Filters.Cls.base(name) @@ -194,12 +194,12 @@ def _attr(var: dict) -> Optional[str]: var_subpkg = var.get("subpackage", None) if ( - (var_type in _SCALARS and not var_shape) + (var_type in _MF6_SCALARS and not var_shape) or var_name in ["cvoptions", "output"] - or (name.r == "dis" and var_name == "packagedata") + or (name[1] == "dis" and var_name == "packagedata") or ( var_name != "packages" - and (name.l is not None and name.r == "nam") + and (name[0] is not None and name[1] == "nam") ) ): return None @@ -218,32 +218,32 @@ def _attr(var: dict) -> Optional[str]: # if the variable is a subpackage reference, use the original key # (which has been replaced already with the referenced variable) args = [ - f"'{name.r}'", + f"'{name[1]}'", f"'{var_block}'", f"'{var_subpkg['key']}'", ] - if name.l is not None and name.l not in [ + if name[0] is not None and name[0] not in [ "sim", "sln", "utl", "exg", ]: - args.insert(0, f"'{name.l}6'") + args.insert(0, f"'{name[0]}6'") return f"{var_subpkg['key']} = ListTemplateGenerator(({', '.join(args)}))" def _args(): args = [ - f"'{name.r}'", + f"'{name[1]}'", f"'{var_block}'", f"'{var_name}'", ] - if name.l is not None and name.l not in [ + if name[0] is not None and name[0] not in [ "sim", "sln", "utl", "exg", ]: - args.insert(0, f"'{name.l}6'") + args.insert(0, f"'{name[0]}6'") return args kind = "array" if is_array else "list" @@ -251,51 +251,18 @@ def _args(): return None - def _dfn() -> List[List[str]]: - dfn, meta = ctx["dfn"] - - def _meta(): - exclude = ["subpackage", "parent_name_type"] - return [ - v for v in meta if not any(p in v for p in exclude) - ] - - def _dfn(): - def _var(var: dict) -> List[str]: - exclude = ["longname", "description"] - name = var["name"] - var_ = variables.get(name, None) - keys = [ - "construct_package", - "construct_data", - "parameter_name", - ] - if var_ and keys[0] in var_: - for k in keys: - var[k] = var_[k] - return [ - " ".join([k, v]).strip() - for k, v in var.items() - if k not in exclude - ] - - return [_var(var) for var in dfn] - - return [["header"] + _meta()] + _dfn() - attrs = list(filter(None, [_attr(v) for v in variables.values()])) if base == "MFPackage": attrs.extend( [ - f"package_abbr = '{name.r}'" - if name.l == "exg" - else f"package_abbr = '{'' if name.l in ['sln', 'sim', 'exg', None] else name.l}{name.r}'", - f"_package_type = '{name.r}'", - f"dfn_file_name = '{name.l}-{name.r}.dfn'" - if name.l == "exg" - else f"dfn_file_name = '{name.l or 'sim'}-{name.r}.dfn'", - f"dfn = {pformat(_dfn(), indent=10)}", + f"package_abbr = '{name[1]}'" + if name[0] == "exg" + else f"package_abbr = '{'' if name[0] in ['sln', 'sim', 'exg', None] else name[0]}{name[1]}'", + f"_package_type = '{name[1]}'", + f"dfn_file_name = '{name[0]}-{name[1]}.dfn'" + if name[0] == "exg" + else f"dfn_file_name = '{name[0] or 'sim'}-{name[1]}.dfn'", ] ) @@ -435,7 +402,7 @@ def _should_build(var: dict) -> bool: if ( subpkg and subpkg["key"] not in refs - and ctx["name"].r != "nam" + and ctx["name"][1] != "nam" ): refs[subpkg["key"]] = subpkg stmts.append( diff --git a/flopy/mf6/utils/generate_classes.py b/flopy/mf6/utils/generate_classes.py index 5f5913517..714418b64 100644 --- a/flopy/mf6/utils/generate_classes.py +++ b/flopy/mf6/utils/generate_classes.py @@ -5,6 +5,8 @@ from pathlib import Path from warnings import warn +from modflow_devtools.dfn2toml import convert as dfn2toml + from .createpackages import make_all thisfilepath = os.path.dirname(os.path.abspath(__file__)) @@ -17,6 +19,7 @@ _MF6_PATH = Path(__file__).parents[1] _DFN_PATH = _MF6_PATH / "data" / "dfn" +_TOML_PATH = _DFN_PATH / "toml" _TGT_PATH = _MF6_PATH / "modflow" @@ -193,8 +196,13 @@ def generate_classes( print(" Deleting existing mf6 classes.") delete_mf6_classes() + # convert dfns to toml.. when we + # do this upstream, remove this. + _TOML_PATH.mkdir(exist_ok=True) + dfn2toml(_DFN_PATH, _TOML_PATH) + print(" Create mf6 classes using the downloaded definition files.") - make_all(_DFN_PATH, _TGT_PATH) + make_all(_TOML_PATH, _TGT_PATH, version=2) list_files(os.path.join(flopypth, "mf6", "modflow")) diff --git a/pyproject.toml b/pyproject.toml index f6ca8cef3..6de09f846 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -38,10 +38,8 @@ dynamic = ["version", "readme"] [project.optional-dependencies] dev = ["flopy[codegen,lint,test,optional,doc]", "tach"] codegen = [ - "boltons>=1.0", "Jinja2>=3.0", "modflow-devtools", - "tomlkit", ] lint = ["cffconvert", "codespell[toml] >=2.2.2", "ruff"] test = [