diff --git a/attmap/_att_map_like.py b/attmap/_att_map_like.py index d6c0308..2ebab10 100644 --- a/attmap/_att_map_like.py +++ b/attmap/_att_map_like.py @@ -41,7 +41,7 @@ def __getattr__(self, item, default=None): raise AttributeError(item) @abc.abstractmethod - def __setitem__(self, key, value): + def __delitem__(self, item): pass @abc.abstractmethod @@ -49,7 +49,7 @@ def __getitem__(self, item): pass @abc.abstractmethod - def __delitem__(self, item): + def __setitem__(self, key, value): pass def __iter__(self): @@ -60,7 +60,8 @@ def __len__(self): def __repr__(self): base = self.__class__.__name__ - data = self._simplify_keyvalue(self._data_for_repr()) + data = self._simplify_keyvalue( + self._data_for_repr(), self._new_empty_basic_map) if data: return base + "\n" + "\n".join( get_data_lines(data, lambda obj: repr(obj).strip("'"))) @@ -91,6 +92,14 @@ def add_entries(self, entries): else self[k].add_entries(v) return self + def get_yaml_lines(self): + """ + Get collection of lines that define YAML text rep. of this instance. + + :return list[str]: YAML representation lines + """ + return ["{}"] if 0 == len(self) else repr(self).split("\n")[1:] + def is_null(self, item): """ Conjunction of presence in underlying mapping and value being None @@ -115,8 +124,23 @@ def to_map(self): :return dict[str, object]: this map's data, in a simpler container """ - return self._simplify_keyvalue( - self.items(), self._new_empty_basic_map()) + return self._simplify_keyvalue(self.items(), self._new_empty_basic_map) + + def to_dict(self): + """ + Return a builtin dict representation of this instance. + + :return dict: builtin dict representation of this instance + """ + return self._simplify_keyvalue(self.items(), dict) + + def to_yaml(self): + """ + Get text for YAML representation. + + :return str: YAML text representation of this instance. + """ + return "\n".join(self.get_yaml_lines()) def _data_for_repr(self): """ @@ -158,20 +182,21 @@ def _new_empty_basic_map(self): """ Return the empty collection builder for Mapping type simplification. """ pass - def _simplify_keyvalue(self, kvs, acc=None): + def _simplify_keyvalue(self, kvs, build, acc=None): """ Simplify a collection of key-value pairs, "reducing" to simpler types. :param Iterable[(object, object)] kvs: collection of key-value pairs + :param callable build: how to build an empty collection :param Iterable acc: accumulating collection of simplified data :return Iterable: collection of simplified data """ - acc = acc or self._new_empty_basic_map() + acc = acc or build() kvs = iter(kvs) try: k, v = next(kvs) except StopIteration: return acc acc[k] = self._simplify_keyvalue( - v.items(), self._new_empty_basic_map()) if is_custom_map(v) else v - return self._simplify_keyvalue(kvs, acc) + v.items(), build, build()) if is_custom_map(v) else v + return self._simplify_keyvalue(kvs, build, acc) diff --git a/attmap/_version.py b/attmap/_version.py index 91bf823..dc64b55 100644 --- a/attmap/_version.py +++ b/attmap/_version.py @@ -1 +1 @@ -__version__ = "0.10" +__version__ = "0.11" diff --git a/attmap/attmap.py b/attmap/attmap.py index e881a86..1340b72 100644 --- a/attmap/attmap.py +++ b/attmap/attmap.py @@ -41,7 +41,7 @@ def __setitem__(self, key, value): """ # TODO: consider enforcement of type constraint, that value of different # type may not overwrite existing. - self.__dict__[key] = self._finalize_value(value) + self.__dict__[key] = self._final_for_store(value) def __eq__(self, other): # TODO: check for equality across classes? @@ -78,14 +78,14 @@ def same_type(obj1, obj2, typenames=None): # have nonidentical labels. return False - def _finalize_value(self, v): + def _final_for_store(self, v): """ Before storing a value, apply any desired transformation. :param object v: value to potentially transform before storing :return object: finalized value """ - for p, f in self._transformations: + for p, f in self._insertion_mutations: if p(v): return f(v) return v @@ -124,9 +124,9 @@ def _repr_pretty_(self, p, cycle): return p.text(repr(self) if not cycle else '...') @property - def _transformations(self): + def _insertion_mutations(self): """ - Add path expansion behavior to more general attmap. + Hook for item transformation(s) to be applied upon insertion. :return list[(function, function)]: pairs in which first component is a predicate and second is a function to apply to a value if it diff --git a/attmap/ordattmap.py b/attmap/ordattmap.py index a049ec3..eedbd95 100644 --- a/attmap/ordattmap.py +++ b/attmap/ordattmap.py @@ -21,6 +21,7 @@ def __init__(self, entries=None): super(OrdAttMap, self).__init__(entries or {}) def __iter__(self): + """ Include in the iteration keys/atts added with setattr style. """ return itertools.chain( super(OrdAttMap, self).__iter__(), filter(lambda k: not self._is_od_member(k), self.__dict__.keys())) @@ -30,13 +31,21 @@ def __reversed__(self): return iter(reversed(list(self.keys()))) def __getitem__(self, item): + """ + Attempt ordinary access, then access to attributes. + + :param hashable item: key/attr for which to fetch value + :return object: value to which given key maps, perhaps modifed + according to the instance's finalization of retrieved values + """ try: return super(OrdAttMap, self).__getitem__(item) except KeyError: return AttMap.__getitem__(self, item) def __setitem__(self, key, value): - super(OrdAttMap, self).__setitem__(key, self._finalize_value(value)) + """ Support hook for value transformation before storage. """ + super(OrdAttMap, self).__setitem__(key, self._final_for_store(value)) def __delitem__(self, key): """ Make unmapped key deletion unexceptional. """ @@ -46,6 +55,7 @@ def __delitem__(self, key): _LOGGER.debug(safedel_message(key)) def __eq__(self, other): + """ Leverage base AttMap eq check, and check key order. """ return AttMap.__eq__(self, other) and \ list(self.keys()) == list(other.keys()) @@ -53,6 +63,7 @@ def __ne__(self, other): return not self == other def __repr__(self): + """ Leverage base AttMap text representation. """ return AttMap.__repr__(self) def keys(self): @@ -87,11 +98,14 @@ def popitem(self, last=True): @staticmethod def _is_od_member(name): + """ Assess whether name appears to be a protected OrderedDict member. """ return name.startswith("_OrderedDict") def _new_empty_basic_map(self): + """ For ordered maps, OrderedDict is the basic building block. """ return OrderedDict() @property def _lower_type_bound(self): + """ OrdAttMap is the type to which nested maps are converted. """ return OrdAttMap diff --git a/attmap/pathex_attmap.py b/attmap/pathex_attmap.py index bee30fa..52ee9cb 100644 --- a/attmap/pathex_attmap.py +++ b/attmap/pathex_attmap.py @@ -13,8 +13,103 @@ class PathExAttMap(OrdAttMap): """ Used in pepkit projects, with Mapping conversion and path expansion """ - @property - def _transformations(self): - """ Add path expansion behavior to more general attmap. """ - return super(PathExAttMap, self)._transformations + \ - [(lambda obj: isinstance(obj, str), expandpath)] + def __getattr__(self, item, default=None, expand=True): + """ + Get attribute, accessing stored key-value pairs as needed. + + :param str item: name of attribute/key + :param object default: value to return if requested attr/key is missing + :param bool expand: whether to attempt path expansion of string value + :return object: value bound to requested name + :raise AttributeError: if requested item is unavailable + """ + try: + v = super(PathExAttMap, self).__getattribute__(item) + except AttributeError: + try: + return self.__getitem__(item, expand) + except KeyError: + # Requested item is unknown, but request was made via + # __getitem__ syntax, not attribute-access syntax. + raise AttributeError(item) + else: + return expandpath(v) if expand else v + + def __getitem__(self, item, expand=True): + """ + Fetch the value of given key. + + :param hashable item: key for which to fetch value + :param bool expand: whether to expand string value as path + :return object: value mapped to given key, if available + :raise KeyError: if the requested key is unmapped. + """ + v = super(PathExAttMap, self).__getitem__(item) + return self._finalize_value(v) if expand else v + + def items(self, expand=False): + """ + Produce list of key-value pairs, optionally expanding paths. + + :param bool expand: whether to expand paths + :return Iterable[object]: stored key-value pairs, optionally expanded + """ + return [(k, self.__getitem__(k, expand)) for k in self] + + def values(self, expand=False): + """ + Produce list of values, optionally expanding paths. + + :param bool expand: whether to expand paths + :return Iterable[object]: stored values, optionally expanded + """ + return [self.__getitem__(k, expand) for k in self] + + def _data_for_repr(self, expand=False): + """ + Hook for extracting the data used in the object's text representation. + + :param bool expand: whether to expand paths + :return Iterable[(hashable, object)]: collection of key-value pairs + to include in object's text representation + """ + return filter(lambda kv: not self._excl_from_repr(kv[0], self.__class__), + self.items(expand)) + + def to_map(self, expand=False): + """ + Convert this instance to a dict. + + :return dict[str, object]: this map's data, in a simpler container + """ + return self._simplify_keyvalue(self.items(expand), self._new_empty_basic_map) + + def to_dict(self, expand=False): + """ + Return a builtin dict representation of this instance. + + :return dict: builtin dict representation of this instance + """ + return self._simplify_keyvalue(self.items(expand), dict) + + def _finalize_value(self, v): + """ + Make any modifications to a retrieved value before returning it. + + This hook accesses an instance's declaration of value mutations, or + transformations. That sequence may be empty (the base case), in which + case any value is simply always returned as-is. + + If an instances does declare retrieval modifications, though, the + declaration should be an iterable of pairs, in which each element's + first component is a single-argument predicate function evaluated on + the given value, and the second component of each element is the + modification to apply if the predicate is satisfied. + + At most one modification function will be called, and it would be the + first one for which the predicate was satisfied. + + :param object v: a value to consider for modification + :return object: the finalized value + """ + return expandpath(v) if isinstance(v, str) else v diff --git a/docs/changelog.md b/docs/changelog.md index f4466df..deceb9f 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -1,5 +1,13 @@ # Changelog +## [0.11] - 2019-05-16 +### Added +- `get_yaml_lines` to get collection of YAML-ready lines from any attmap +- `to_dict` to convert any attmap (and nested maps) to base `dict` +- `to_yaml` to represent any attmap as raw YAML text +### Changed +- `PathExAttMap` defers expansion behavior to retrieval time. + ## [0.10] - 2019-05-15 ### Fixed - `OrdAttMap` and descendants now have data updated via `__setattr__` syntax. diff --git a/tests/test_attmap_equality.py b/tests/test_equality.py similarity index 100% rename from tests/test_attmap_equality.py rename to tests/test_equality.py diff --git a/tests/test_path_expansion.py b/tests/test_path_expansion.py index 530b9de..20da64a 100644 --- a/tests/test_path_expansion.py +++ b/tests/test_path_expansion.py @@ -110,6 +110,8 @@ def test_PathExAttMap_substitution_is_selective(path, pres, repl, env, pam, fetc with TmpEnv(**env): pam[k] = path res = fetch(pam, k) + print("Inserted path: {}".format(path)) + print("Retrieved path: {}".format(res)) assert all(map(lambda s: s in res, pres)) assert all(map(lambda s: s not in res, repl)) diff --git a/tests/test_to_dict.py b/tests/test_to_dict.py new file mode 100644 index 0000000..8babe0d --- /dev/null +++ b/tests/test_to_dict.py @@ -0,0 +1,24 @@ +""" Tests for conversion to base/builtin dict type """ + +import pytest +from tests.helpers import get_att_map +import numpy as np +from pandas import Series + + +__author__ = "Vince Reuter" +__email__ = "vreuter@virginia.edu" + + +@pytest.mark.para +@pytest.mark.parametrize("entries", [ + {}, {"a": 1}, {"b": {"c": 3}}, {"A": [1, 2]}, + {"B": 1, "C": np.arange(3)}, {"E": Series(["a", "b"])}]) +def test_to_dict_type(attmap_type, entries): + """ Validate to_dict result is a base dict and that contents match. """ + m = get_att_map(attmap_type, entries) + assert type(m) is not dict + d = m.to_dict() + assert type(d) is dict + assert d == entries + assert entries == d diff --git a/tests/test_to_disk.py b/tests/test_to_disk.py new file mode 100644 index 0000000..af54afc --- /dev/null +++ b/tests/test_to_disk.py @@ -0,0 +1,146 @@ +""" Tests for YAML representation of instances. """ + +from collections import namedtuple +import json +import os +import sys +if sys.version_info < (3, 3): + from collections import MutableMapping +else: + from collections.abc import MutableMapping +import yaml +from attmap import * +import pytest +from tests.conftest import ALL_ATTMAPS + +__author__ = "Vince Reuter" +__email__ = "vreuter@virginia.edu" + +ENTRIES = [("b", 2), ("a", [("d", 4), ("c", [("f", 6), ("g", 7)])])] +EXPLINES = ["b: 2", "a:", " d: 4", " c:", " f: 6", " g: 7"] +FmtLib = namedtuple("FmtLib", ["parse", "write"]) +YAML_NAME = yaml.__name__ +JSON_NAME = json.__name__ +FORMATTER_LIBRARIES = { + YAML_NAME: FmtLib(lambda f: yaml.load(f, yaml.SafeLoader), + lambda m, f: f.write(m.to_yaml())), + JSON_NAME: FmtLib(lambda f: json.load(f), + lambda m, f: json.dump(m.to_map(), f)) +} + + +def pytest_generate_tests(metafunc): + """ Dynamic test case generation and parameterization for this module """ + if "maptype" in metafunc.fixturenames: + metafunc.parametrize("maptype", ALL_ATTMAPS) + + +def check_lines(m, explines, obs_fun, parse, check): + """ + Check a YAML expectation against observation. + + :param attmap.AttMap m: the map to convert to YAML + :param Iterable[str] explines: collection of expected lines + :param str obs_fun: name of attmap function to call to get observation + :param function(Iterable[str]) -> object parse: postprocess observed result + :param function(object, object) -> bool check: validation function + """ + obs = parse(getattr(m, obs_fun)()) + print("FUNCNAME: {}".format(obs_fun)) + print("EXPLINES (below):\n{}".format(explines)) + print("OBS (below):\n{}".format(obs)) + assert check(explines, obs) + + +@pytest.mark.parametrize( + ["funcname", "exp"], [("get_yaml_lines", ["{}"]), ("to_yaml", "{}")]) +def test_empty(funcname, exp, maptype): + """ Verify behavior for YAML of empty attmap. """ + assert exp == getattr(maptype({}), funcname)() + + +@pytest.mark.parametrize( + ["get_obs", "parse_obs"], + [("get_yaml_lines", lambda ls: ls), ("to_yaml", lambda ls: ls.split("\n"))]) +def test_yaml(maptype, get_obs, parse_obs): + """ Tests for attmap repr as YAML lines or full text chunk. """ + eq = lambda a, b: a == b + seteq = lambda a, b: len(a) == len(b) and set(a) == set(b) + checks = {OrdAttMap: eq, PathExAttMap: eq, AttMapEcho: eq, AttMap: seteq} + m = make_data(ENTRIES, maptype) + check_lines(m, EXPLINES, get_obs, parse_obs, check=checks[maptype]) + + +@pytest.mark.parametrize("fmtlib", [YAML_NAME, JSON_NAME]) +def test_disk_roundtrip(maptype, tmpdir, fmtlib): + """ Verify ability to parse, write, and reconstitute attmap. """ + m = make_data(ENTRIES, maptype) + fp = tmpdir.join("disked_attmap.out").strpath + assert not os.path.exists(fp) + fmtspec = FORMATTER_LIBRARIES[fmtlib] + parse, write = fmtspec.parse, fmtspec.write + with open(fp, 'w') as f: + write(m, f) + with open(fp, 'r') as f: + recons = parse(f) + assert recons == m.to_dict() + + +@pytest.mark.parametrize(["data", "env_var", "fmtlib", "exp_res"], [ + ({"arbkey": os.path.join("$HOME", "leaf.md")}, "HOME", JSON_NAME, + ["{{\"arbkey\": \"{}\"}}".format(os.path.join("$HOME", "leaf.md"))]), + ({"arbkey": os.path.join("$HOME", "leaf.md")}, "HOME", YAML_NAME, + ["arbkey: " + os.path.join("$HOME", "leaf.md")]), + ({"random": os.path.join("abc", "$HOME", "leaf.md")}, "HOME", JSON_NAME, + ["{{\"random\": \"{}\"}}".format(os.path.join("abc", "$HOME", "leaf.md"))]), + ({"random": os.path.join("abc", "$HOME", "leaf.md")}, "HOME", YAML_NAME, + ["random: " + os.path.join("abc", "$HOME", "leaf.md")]) +]) +def test_disk_path_expansion(tmpdir, data, env_var, fmtlib, exp_res, maptype): + """ Paths are not expanded when map goes to disk. """ + + # Pretests + assert len(data) == 1, \ + "To isolate focus, use just 1 item; got {} -- {}".format(len(data), data) + assert os.environ.get(env_var) is not None, \ + "Null or missing env var: {}".format(env_var) + + # Create the mapping. + m = make_data(list(data.items()), maptype) + assert type(m) is maptype + + # Set write/parse strategies. + fmtspec = FORMATTER_LIBRARIES[fmtlib] + parse, write = fmtspec.parse, fmtspec.write + + fp = tmpdir.join("disked_attmap.out").strpath + assert not os.path.exists(fp) + with open(fp, 'w') as f: + write(m, f) + with open(fp, 'r') as f: + assert exp_res == [l.strip("\n") for l in f.readlines()] + + +def make_data(entries, datatype): + """ + Create the base data used to populate an attmap. + + :param Iterable[(str, object)] entries: key-value pairs + :param type datatype: the type of mapping to build + :return MutableMapping: the newly built/populated mapping + """ + assert issubclass(datatype, MutableMapping) + + def go(items, acc): + try: + (k, v), t = items[0], items[1:] + except IndexError: + return acc + if type(v) is list and all(type(e) is tuple and len(e) == 2 for e in v): + v = go(v, datatype()) + acc[k] = v + return go(t, acc) + + res = go(entries, datatype()) + assert len(res) > 0, "Empty result" + return res