From 166a28b8b22dac2ed8ef1088989a6b58ad4a450f Mon Sep 17 00:00:00 2001 From: Vince Reuter Date: Wed, 15 May 2019 11:27:26 -0400 Subject: [PATCH 01/16] dev version --- attmap/_version.py | 2 +- docs/changelog.md | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/attmap/_version.py b/attmap/_version.py index 91bf823..9a52e92 100644 --- a/attmap/_version.py +++ b/attmap/_version.py @@ -1 +1 @@ -__version__ = "0.10" +__version__ = "0.11dev" diff --git a/docs/changelog.md b/docs/changelog.md index f4466df..acc5707 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -1,5 +1,7 @@ # Changelog +## Unreleased + ## [0.10] - 2019-05-15 ### Fixed - `OrdAttMap` and descendants now have data updated via `__setattr__` syntax. From 94f8cf3359ef667e1ab9e6b856f380f72861948f Mon Sep 17 00:00:00 2001 From: Vince Reuter Date: Wed, 15 May 2019 12:22:57 -0400 Subject: [PATCH 02/16] first pass at deferred expansion; 37 --- attmap/_att_map_like.py | 20 ++++++++++++++++++-- attmap/attmap.py | 12 ++++++------ attmap/ordattmap.py | 7 ++++--- attmap/pathex_attmap.py | 8 +++++--- docs/changelog.md | 2 ++ tests/test_path_expansion.py | 2 ++ 6 files changed, 37 insertions(+), 14 deletions(-) diff --git a/attmap/_att_map_like.py b/attmap/_att_map_like.py index d6c0308..5841442 100644 --- a/attmap/_att_map_like.py +++ b/attmap/_att_map_like.py @@ -31,14 +31,21 @@ def __init__(self, entries=None): def __getattr__(self, item, default=None): try: - return super(AttMapLike, self).__getattribute__(item) + v = super(AttMapLike, self).__getattribute__(item) except AttributeError: try: - return self.__getitem__(item) + v = self.__getitem__(item) except KeyError: # Requested item is unknown, but request was made via # __getitem__ syntax, not attribute-access syntax. raise AttributeError(item) + return self._finalize_value(v) + + def _finalize_value(self, v): + for p, f in self._retrieval_mutations: + if p(v): + return f(v) + return v @abc.abstractmethod def __setitem__(self, key, value): @@ -158,6 +165,15 @@ def _new_empty_basic_map(self): """ Return the empty collection builder for Mapping type simplification. """ pass + @property + def _retrieval_mutations(self): + """ + Hook for item transformation(s) to be applied upon retrieval. + + :return: + """ + return [] + def _simplify_keyvalue(self, kvs, acc=None): """ Simplify a collection of key-value pairs, "reducing" to simpler types. diff --git a/attmap/attmap.py b/attmap/attmap.py index e881a86..326712b 100644 --- a/attmap/attmap.py +++ b/attmap/attmap.py @@ -29,7 +29,7 @@ def __delitem__(self, key): _LOGGER.debug(safedel_message(key)) def __getitem__(self, item): - return self.__dict__[item] + return self._finalize_value(self.__dict__[item]) def __setitem__(self, key, value): """ @@ -41,7 +41,7 @@ def __setitem__(self, key, value): """ # TODO: consider enforcement of type constraint, that value of different # type may not overwrite existing. - self.__dict__[key] = self._finalize_value(value) + self.__dict__[key] = self._final_for_store(value) def __eq__(self, other): # TODO: check for equality across classes? @@ -78,14 +78,14 @@ def same_type(obj1, obj2, typenames=None): # have nonidentical labels. return False - def _finalize_value(self, v): + def _final_for_store(self, v): """ Before storing a value, apply any desired transformation. :param object v: value to potentially transform before storing :return object: finalized value """ - for p, f in self._transformations: + for p, f in self._insertion_mutations: if p(v): return f(v) return v @@ -124,9 +124,9 @@ def _repr_pretty_(self, p, cycle): return p.text(repr(self) if not cycle else '...') @property - def _transformations(self): + def _insertion_mutations(self): """ - Add path expansion behavior to more general attmap. + Hook for item transformation(s) to be applied upon insertion. :return list[(function, function)]: pairs in which first component is a predicate and second is a function to apply to a value if it diff --git a/attmap/ordattmap.py b/attmap/ordattmap.py index a049ec3..05d2945 100644 --- a/attmap/ordattmap.py +++ b/attmap/ordattmap.py @@ -31,12 +31,13 @@ def __reversed__(self): def __getitem__(self, item): try: - return super(OrdAttMap, self).__getitem__(item) + v = super(OrdAttMap, self).__getitem__(item) except KeyError: - return AttMap.__getitem__(self, item) + v = AttMap.__getitem__(self, item) + return self._finalize_value(v) def __setitem__(self, key, value): - super(OrdAttMap, self).__setitem__(key, self._finalize_value(value)) + super(OrdAttMap, self).__setitem__(key, self._final_for_store(value)) def __delitem__(self, key): """ Make unmapped key deletion unexceptional. """ diff --git a/attmap/pathex_attmap.py b/attmap/pathex_attmap.py index bee30fa..dd231f9 100644 --- a/attmap/pathex_attmap.py +++ b/attmap/pathex_attmap.py @@ -14,7 +14,9 @@ class PathExAttMap(OrdAttMap): """ Used in pepkit projects, with Mapping conversion and path expansion """ @property - def _transformations(self): + def _retrieval_mutations(self): """ Add path expansion behavior to more general attmap. """ - return super(PathExAttMap, self)._transformations + \ - [(lambda obj: isinstance(obj, str), expandpath)] + # DEBUG + print("HIT!") + return super(PathExAttMap, self)._retrieval_mutations + \ + [(lambda obj: isinstance(obj, str), expandpath)] diff --git a/docs/changelog.md b/docs/changelog.md index acc5707..7deb326 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -1,6 +1,8 @@ # Changelog ## Unreleased +### Changed +- `PathExAttMap` defers expansion behavior to retrieval time ## [0.10] - 2019-05-15 ### Fixed diff --git a/tests/test_path_expansion.py b/tests/test_path_expansion.py index 530b9de..20da64a 100644 --- a/tests/test_path_expansion.py +++ b/tests/test_path_expansion.py @@ -110,6 +110,8 @@ def test_PathExAttMap_substitution_is_selective(path, pres, repl, env, pam, fetc with TmpEnv(**env): pam[k] = path res = fetch(pam, k) + print("Inserted path: {}".format(path)) + print("Retrieved path: {}".format(res)) assert all(map(lambda s: s in res, pres)) assert all(map(lambda s: s not in res, repl)) From fcf39a82d45b547c2715525abee97c9f9bbc5d79 Mon Sep 17 00:00:00 2001 From: Vince Reuter Date: Wed, 15 May 2019 13:25:28 -0400 Subject: [PATCH 03/16] reorder --- attmap/_att_map_like.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/attmap/_att_map_like.py b/attmap/_att_map_like.py index 5841442..206687b 100644 --- a/attmap/_att_map_like.py +++ b/attmap/_att_map_like.py @@ -48,7 +48,7 @@ def _finalize_value(self, v): return v @abc.abstractmethod - def __setitem__(self, key, value): + def __delitem__(self, item): pass @abc.abstractmethod @@ -56,7 +56,7 @@ def __getitem__(self, item): pass @abc.abstractmethod - def __delitem__(self, item): + def __setitem__(self, key, value): pass def __iter__(self): From 530d0c8dea296d6f96d0b3d056a87e381ee24b86 Mon Sep 17 00:00:00 2001 From: Vince Reuter Date: Wed, 15 May 2019 14:05:06 -0400 Subject: [PATCH 04/16] first pass at serialization and YAML helpers --- attmap/_att_map_like.py | 76 +++++++++++++++++++++++++++++++++-------- 1 file changed, 62 insertions(+), 14 deletions(-) diff --git a/attmap/_att_map_like.py b/attmap/_att_map_like.py index 206687b..e70eab7 100644 --- a/attmap/_att_map_like.py +++ b/attmap/_att_map_like.py @@ -41,12 +41,6 @@ def __getattr__(self, item, default=None): raise AttributeError(item) return self._finalize_value(v) - def _finalize_value(self, v): - for p, f in self._retrieval_mutations: - if p(v): - return f(v) - return v - @abc.abstractmethod def __delitem__(self, item): pass @@ -67,7 +61,8 @@ def __len__(self): def __repr__(self): base = self.__class__.__name__ - data = self._simplify_keyvalue(self._data_for_repr()) + data = self._simplify_keyvalue( + self._data_for_repr(), self._new_empty_basic_map) if data: return base + "\n" + "\n".join( get_data_lines(data, lambda obj: repr(obj).strip("'"))) @@ -98,6 +93,14 @@ def add_entries(self, entries): else self[k].add_entries(v) return self + def get_yaml_lines(self): + """ + Get collection of lines that define YAML text rep. of this instance. + + :return list[str]: YAML representation lines + """ + return ["{}"] if 0 == len(self) else repr(self).split("\n")[1:] + def is_null(self, item): """ Conjunction of presence in underlying mapping and value being None @@ -122,8 +125,23 @@ def to_map(self): :return dict[str, object]: this map's data, in a simpler container """ - return self._simplify_keyvalue( - self.items(), self._new_empty_basic_map()) + return self._simplify_keyvalue(self.items(), self._new_empty_basic_map) + + def to_dict(self): + """ + Return a builtin dict representation of this instance. + + :return dict: builtin dict representation of this instance + """ + return self._simplify_keyvalue(self.items(), dict) + + def to_yaml(self): + """ + Get text for YAML representation. + + :return str: YAML text representation of this instance. + """ + return "\n".join(self.yaml_lines()) def _data_for_repr(self): """ @@ -155,6 +173,31 @@ def _excl_from_repr(self, k, cls): """ return False + def _finalize_value(self, v): + """ + Make any modifications to a retrieved value before returning it. + + This hook accesses an instance's declaration of value mutations, or + transformations. That sequence may be empty (the base case), in which + case any value is simply always returned as-is. + + If an instances does declare retrieval modifications, though, the + declaration should be an iterable of pairs, in which each element's + first component is a single-argument predicate function evaluated on + the given value, and the second component of each element is the + modification to apply if the predicate is satisfied. + + At most one modification function will be called, and it would be the + first one for which the predicate was satisfied. + + :param object v: a value to consider for modification + :return object: the finalized value + """ + for p, f in self._retrieval_mutations: + if p(v): + return f(v) + return v + @abc.abstractproperty def _lower_type_bound(self): """ Most specific type to which stored Mapping should be transformed """ @@ -170,24 +213,29 @@ def _retrieval_mutations(self): """ Hook for item transformation(s) to be applied upon retrieval. - :return: + :return Iterable[(function(object) -> bool, function(object) -> object)]: + collection in which each element has a one-arg predicate function + and a one-arg transformation function, used to determine a + retrieved value's final form (i.e., transform it according to the + first predicate that it satisfies.) """ return [] - def _simplify_keyvalue(self, kvs, acc=None): + def _simplify_keyvalue(self, kvs, build, acc=None): """ Simplify a collection of key-value pairs, "reducing" to simpler types. :param Iterable[(object, object)] kvs: collection of key-value pairs + :param callable build: how to build an empty collection :param Iterable acc: accumulating collection of simplified data :return Iterable: collection of simplified data """ - acc = acc or self._new_empty_basic_map() + acc = acc or build() kvs = iter(kvs) try: k, v = next(kvs) except StopIteration: return acc acc[k] = self._simplify_keyvalue( - v.items(), self._new_empty_basic_map()) if is_custom_map(v) else v - return self._simplify_keyvalue(kvs, acc) + v.items(), build, build()) if is_custom_map(v) else v + return self._simplify_keyvalue(kvs, build, acc) From 5d62e363f0d2a9ec1de00685ac144f71930f8ede Mon Sep 17 00:00:00 2001 From: Vince Reuter Date: Wed, 15 May 2019 14:22:48 -0400 Subject: [PATCH 05/16] to_dict tests and changelog --- docs/changelog.md | 4 ++++ tests/test_to_dict.py | 24 ++++++++++++++++++++++++ 2 files changed, 28 insertions(+) create mode 100644 tests/test_to_dict.py diff --git a/docs/changelog.md b/docs/changelog.md index 7deb326..5b9f30a 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -1,6 +1,10 @@ # Changelog ## Unreleased +### Added +- `get_yaml_lines` to get collection of YAML-ready lines from any attmap +- `to_dict` to convert any attmap (and nested maps) to base `dict` +- `to_yaml` to represent any attmap as raw YAML text ### Changed - `PathExAttMap` defers expansion behavior to retrieval time diff --git a/tests/test_to_dict.py b/tests/test_to_dict.py new file mode 100644 index 0000000..489fed1 --- /dev/null +++ b/tests/test_to_dict.py @@ -0,0 +1,24 @@ +""" Tests for conversion to base/builtin dict type """ + +import pytest +from tests.helpers import get_att_map +import numpy as np +from pandas import Series + + +__author__ = "Vince Reuter" +__email__ = "vreuter@virginia.edu" + + +@pytest.mark.para +@pytest.mark.parametrize("entries", [ + {}, {"a": 1}, {"b": {"c": 3}}, {"A": [1, 2]}, + {"B": 1, "C": np.arange(3)}, {"E": Series(["a", "b"])}]) +def test_to_dict_type(attmap_type, entries): + """ Validate to_dict result. """ + m = get_att_map(attmap_type, entries) + assert type(m) is not dict + d = m.to_dict() + assert type(d) is dict + assert d == entries + assert entries == d From aee775fa574b6c14d7b598b399daa5c5417ac58f Mon Sep 17 00:00:00 2001 From: Vince Reuter Date: Wed, 15 May 2019 16:34:47 -0400 Subject: [PATCH 06/16] fix method name --- attmap/_att_map_like.py | 2 +- attmap/pathex_attmap.py | 2 -- tests/{test_attmap_equality.py => test_equality.py} | 0 tests/test_yaml.py | 0 4 files changed, 1 insertion(+), 3 deletions(-) rename tests/{test_attmap_equality.py => test_equality.py} (100%) create mode 100644 tests/test_yaml.py diff --git a/attmap/_att_map_like.py b/attmap/_att_map_like.py index e70eab7..21268de 100644 --- a/attmap/_att_map_like.py +++ b/attmap/_att_map_like.py @@ -141,7 +141,7 @@ def to_yaml(self): :return str: YAML text representation of this instance. """ - return "\n".join(self.yaml_lines()) + return "\n".join(self.get_yaml_lines()) def _data_for_repr(self): """ diff --git a/attmap/pathex_attmap.py b/attmap/pathex_attmap.py index dd231f9..0b1a2e8 100644 --- a/attmap/pathex_attmap.py +++ b/attmap/pathex_attmap.py @@ -16,7 +16,5 @@ class PathExAttMap(OrdAttMap): @property def _retrieval_mutations(self): """ Add path expansion behavior to more general attmap. """ - # DEBUG - print("HIT!") return super(PathExAttMap, self)._retrieval_mutations + \ [(lambda obj: isinstance(obj, str), expandpath)] diff --git a/tests/test_attmap_equality.py b/tests/test_equality.py similarity index 100% rename from tests/test_attmap_equality.py rename to tests/test_equality.py diff --git a/tests/test_yaml.py b/tests/test_yaml.py new file mode 100644 index 0000000..e69de29 From 6fd7953075ad66e1d085c0873270751a8d20f943 Mon Sep 17 00:00:00 2001 From: Vince Reuter Date: Wed, 15 May 2019 16:34:57 -0400 Subject: [PATCH 07/16] YAML tests --- tests/test_to_dict.py | 2 +- tests/test_yaml.py | 63 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 64 insertions(+), 1 deletion(-) diff --git a/tests/test_to_dict.py b/tests/test_to_dict.py index 489fed1..8babe0d 100644 --- a/tests/test_to_dict.py +++ b/tests/test_to_dict.py @@ -15,7 +15,7 @@ {}, {"a": 1}, {"b": {"c": 3}}, {"A": [1, 2]}, {"B": 1, "C": np.arange(3)}, {"E": Series(["a", "b"])}]) def test_to_dict_type(attmap_type, entries): - """ Validate to_dict result. """ + """ Validate to_dict result is a base dict and that contents match. """ m = get_att_map(attmap_type, entries) assert type(m) is not dict d = m.to_dict() diff --git a/tests/test_yaml.py b/tests/test_yaml.py index e69de29..4702fe8 100644 --- a/tests/test_yaml.py +++ b/tests/test_yaml.py @@ -0,0 +1,63 @@ +""" Tests for YAML representation of instances. """ + +from collections import OrderedDict +from attmap import * +import pytest +from tests.conftest import ALL_ATTMAPS + +__author__ = "Vince Reuter" +__email__ = "vreuter@virginia.edu" + + +def check_lines(m, explines, obs_fun, parse, check): + """ + Check a YAML expectation against observation. + + :param attmap.AttMap m: the map to convert to YAML + :param Iterable[str] explines: collection of expected lines + :param str obs_fun: name of attmap function to call to get observation + :param function(Iterable[str]) -> object parse: postprocess observed result + :param function(object, object) -> bool check: validation function + """ + obs = parse(getattr(m, obs_fun)()) + print("FUNCNAME: {}".format(obs_fun)) + print("EXPLINES (below):\n{}".format(explines)) + print("OBS (below):\n{}".format(obs)) + assert check(explines, obs) + + +@pytest.mark.parametrize( + ["get_obs", "parse_obs"], + [("get_yaml_lines", lambda ls: ls), ("to_yaml", lambda ls: ls.split("\n"))]) +@pytest.mark.parametrize("maptype", ALL_ATTMAPS) +def test_yaml(maptype, get_obs, parse_obs): + eq = lambda a, b: a == b + seteq = lambda a, b: len(a) == len(b) and set(a) == set(b) + checks = {OrdAttMap: eq, PathExAttMap: eq, AttMapEcho: eq, AttMap: seteq} + entries = [("b", 2), ("a", [("d", 4), ("c", [("f", 6), ("g", 7)])])] + explines = ["b: 2", "a:", " d: 4", " c:", " f: 6", " g: 7"] + m = make_data(entries, maptype) + check_lines(m, explines, get_obs, parse_obs, check=checks[maptype]) + + +def make_data(entries, datatype): + """ + Create the base data used to populate an attmap. + + :param Iterable[(str, object)] entries: key-value pairs + :param type datatype: the type of ma + :return: + """ + assert datatype in [dict, OrderedDict] or issubclass(datatype, AttMap) + + def go(items, acc): + try: + (k, v), t = items[0], items[1:] + except IndexError: + return acc + if type(v) is list and all(type(e) is tuple and len(e) == 2 for e in v): + v = go(v, datatype()) + acc[k] = v + return go(t, acc) + + return go(entries, datatype()) From bb2b7191104f948bf5fd4fe6054562cddcfd6def Mon Sep 17 00:00:00 2001 From: Vince Reuter Date: Wed, 15 May 2019 16:42:43 -0400 Subject: [PATCH 08/16] tests for empty map as YAML --- tests/test_yaml.py | 26 +++++++++++++++++++++----- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/tests/test_yaml.py b/tests/test_yaml.py index 4702fe8..eba0d9e 100644 --- a/tests/test_yaml.py +++ b/tests/test_yaml.py @@ -1,6 +1,10 @@ """ Tests for YAML representation of instances. """ -from collections import OrderedDict +import sys +if sys.version_info < (3, 3): + from collections import MutableMapping +else: + from collections.abc import MutableMapping from attmap import * import pytest from tests.conftest import ALL_ATTMAPS @@ -26,11 +30,21 @@ def check_lines(m, explines, obs_fun, parse, check): assert check(explines, obs) + +@pytest.mark.parametrize( + ["funcname", "exp"], [("get_yaml_lines", ["{}"]), ("to_yaml", "{}")]) +@pytest.mark.parametrize("maptype", ALL_ATTMAPS) +def test_empty(funcname, exp, maptype): + """ Verify behavior for YAML of empty attmap. """ + assert getattr(maptype({}), funcname)() == exp + + @pytest.mark.parametrize( ["get_obs", "parse_obs"], [("get_yaml_lines", lambda ls: ls), ("to_yaml", lambda ls: ls.split("\n"))]) @pytest.mark.parametrize("maptype", ALL_ATTMAPS) def test_yaml(maptype, get_obs, parse_obs): + """ Tests for attmap repr as YAML lines or full text chunk. """ eq = lambda a, b: a == b seteq = lambda a, b: len(a) == len(b) and set(a) == set(b) checks = {OrdAttMap: eq, PathExAttMap: eq, AttMapEcho: eq, AttMap: seteq} @@ -45,10 +59,10 @@ def make_data(entries, datatype): Create the base data used to populate an attmap. :param Iterable[(str, object)] entries: key-value pairs - :param type datatype: the type of ma - :return: + :param type datatype: the type of mapping to build + :return MutableMapping: the newly built/populated mapping """ - assert datatype in [dict, OrderedDict] or issubclass(datatype, AttMap) + assert issubclass(datatype, MutableMapping) def go(items, acc): try: @@ -60,4 +74,6 @@ def go(items, acc): acc[k] = v return go(t, acc) - return go(entries, datatype()) + res = go(entries, datatype()) + assert len(res) > 0, "Empty result" + return res From fea2117b9a84918c20ca67c4aee11af8a19d042a Mon Sep 17 00:00:00 2001 From: Vince Reuter Date: Wed, 15 May 2019 17:15:34 -0400 Subject: [PATCH 09/16] roundtrip tests; #43 --- tests/{test_yaml.py => test_to_disk.py} | 37 +++++++++++++++++++------ 1 file changed, 29 insertions(+), 8 deletions(-) rename tests/{test_yaml.py => test_to_disk.py} (68%) diff --git a/tests/test_yaml.py b/tests/test_to_disk.py similarity index 68% rename from tests/test_yaml.py rename to tests/test_to_disk.py index eba0d9e..74123d7 100644 --- a/tests/test_yaml.py +++ b/tests/test_to_disk.py @@ -1,10 +1,12 @@ """ Tests for YAML representation of instances. """ +import json import sys if sys.version_info < (3, 3): from collections import MutableMapping else: from collections.abc import MutableMapping +import yaml from attmap import * import pytest from tests.conftest import ALL_ATTMAPS @@ -12,6 +14,15 @@ __author__ = "Vince Reuter" __email__ = "vreuter@virginia.edu" +ENTRIES = [("b", 2), ("a", [("d", 4), ("c", [("f", 6), ("g", 7)])])] +EXPLINES = ["b: 2", "a:", " d: 4", " c:", " f: 6", " g: 7"] + + +def pytest_generate_tests(metafunc): + """ Dynamic test case generation and parameterization for this module """ + if "maptype" in metafunc.fixturenames: + metafunc.parametrize("maptype", ALL_ATTMAPS) + def check_lines(m, explines, obs_fun, parse, check): """ @@ -30,28 +41,38 @@ def check_lines(m, explines, obs_fun, parse, check): assert check(explines, obs) - @pytest.mark.parametrize( ["funcname", "exp"], [("get_yaml_lines", ["{}"]), ("to_yaml", "{}")]) -@pytest.mark.parametrize("maptype", ALL_ATTMAPS) def test_empty(funcname, exp, maptype): """ Verify behavior for YAML of empty attmap. """ - assert getattr(maptype({}), funcname)() == exp + assert exp == getattr(maptype({}), funcname)() @pytest.mark.parametrize( ["get_obs", "parse_obs"], [("get_yaml_lines", lambda ls: ls), ("to_yaml", lambda ls: ls.split("\n"))]) -@pytest.mark.parametrize("maptype", ALL_ATTMAPS) def test_yaml(maptype, get_obs, parse_obs): """ Tests for attmap repr as YAML lines or full text chunk. """ eq = lambda a, b: a == b seteq = lambda a, b: len(a) == len(b) and set(a) == set(b) checks = {OrdAttMap: eq, PathExAttMap: eq, AttMapEcho: eq, AttMap: seteq} - entries = [("b", 2), ("a", [("d", 4), ("c", [("f", 6), ("g", 7)])])] - explines = ["b: 2", "a:", " d: 4", " c:", " f: 6", " g: 7"] - m = make_data(entries, maptype) - check_lines(m, explines, get_obs, parse_obs, check=checks[maptype]) + m = make_data(ENTRIES, maptype) + check_lines(m, EXPLINES, get_obs, parse_obs, check=checks[maptype]) + + +@pytest.mark.parametrize(["write", "parse"], [ + (lambda m, f: f.write(m.to_yaml()), lambda f: yaml.load(f, yaml.SafeLoader)), + (lambda m, f: json.dump(m.to_map(), f), lambda f: json.load(f)) +]) +def test_disk_roundtrip(maptype, tmpdir, write, parse): + """ Verify ability to parse, write, and reconstitute attmap. """ + m = make_data(ENTRIES, maptype) + fp = tmpdir.join("disked_attmap.out").strpath + with open(fp, 'w') as f: + write(m, f) + with open(fp, 'r') as f: + recons = parse(f) + assert recons == m.to_dict() def make_data(entries, datatype): From fe4703acdb82a01cf65d742708761896a28d9e19 Mon Sep 17 00:00:00 2001 From: Vince Reuter Date: Wed, 15 May 2019 17:15:47 -0400 Subject: [PATCH 10/16] comments --- attmap/ordattmap.py | 14 ++++++++++++++ docs/changelog.md | 2 +- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/attmap/ordattmap.py b/attmap/ordattmap.py index 05d2945..aa672f1 100644 --- a/attmap/ordattmap.py +++ b/attmap/ordattmap.py @@ -21,6 +21,7 @@ def __init__(self, entries=None): super(OrdAttMap, self).__init__(entries or {}) def __iter__(self): + """ Include in the iteration keys/atts added with setattr style. """ return itertools.chain( super(OrdAttMap, self).__iter__(), filter(lambda k: not self._is_od_member(k), self.__dict__.keys())) @@ -30,6 +31,13 @@ def __reversed__(self): return iter(reversed(list(self.keys()))) def __getitem__(self, item): + """ + Attempt ordinary access, then access to attributes. + + :param hashable item: key/attr for which to fetch value + :return object: value to which given key maps, perhaps modifed + according to the instance's finalization of retrieved values + """ try: v = super(OrdAttMap, self).__getitem__(item) except KeyError: @@ -37,6 +45,7 @@ def __getitem__(self, item): return self._finalize_value(v) def __setitem__(self, key, value): + """ Support hook for value transformation before storage. """ super(OrdAttMap, self).__setitem__(key, self._final_for_store(value)) def __delitem__(self, key): @@ -47,6 +56,7 @@ def __delitem__(self, key): _LOGGER.debug(safedel_message(key)) def __eq__(self, other): + """ Leverage base AttMap eq check, and check key order. """ return AttMap.__eq__(self, other) and \ list(self.keys()) == list(other.keys()) @@ -54,6 +64,7 @@ def __ne__(self, other): return not self == other def __repr__(self): + """ Leverage base AttMap text representation. """ return AttMap.__repr__(self) def keys(self): @@ -88,11 +99,14 @@ def popitem(self, last=True): @staticmethod def _is_od_member(name): + """ Assess whether name appears to be a protected OrderedDict member. """ return name.startswith("_OrderedDict") def _new_empty_basic_map(self): + """ For ordered maps, OrderedDict is the basic building block. """ return OrderedDict() @property def _lower_type_bound(self): + """ OrdAttMap is the type to which nested maps are converted. """ return OrdAttMap diff --git a/docs/changelog.md b/docs/changelog.md index 5b9f30a..97b2a6b 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -6,7 +6,7 @@ - `to_dict` to convert any attmap (and nested maps) to base `dict` - `to_yaml` to represent any attmap as raw YAML text ### Changed -- `PathExAttMap` defers expansion behavior to retrieval time +- `PathExAttMap` defers expansion behavior to retrieval time. ## [0.10] - 2019-05-15 ### Fixed From 9856110c7162dc100f2f434650d550184f3e19ba Mon Sep 17 00:00:00 2001 From: Vince Reuter Date: Wed, 15 May 2019 17:38:16 -0400 Subject: [PATCH 11/16] better format lib param --- tests/test_to_disk.py | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/tests/test_to_disk.py b/tests/test_to_disk.py index 74123d7..8bb2ab5 100644 --- a/tests/test_to_disk.py +++ b/tests/test_to_disk.py @@ -1,5 +1,6 @@ """ Tests for YAML representation of instances. """ +from collections import namedtuple import json import sys if sys.version_info < (3, 3): @@ -16,6 +17,15 @@ ENTRIES = [("b", 2), ("a", [("d", 4), ("c", [("f", 6), ("g", 7)])])] EXPLINES = ["b: 2", "a:", " d: 4", " c:", " f: 6", " g: 7"] +FmtLib = namedtuple("FmtLib", ["parse", "write"]) +YAML_NAME = yaml.__name__ +JSON_NAME = json.__name__ +FORMATTER_LIBRARIES = { + YAML_NAME: FmtLib(lambda f: yaml.load(f, yaml.SafeLoader), + lambda m, f: f.write(m.to_yaml())), + JSON_NAME: FmtLib(lambda f: json.load(f), + lambda m, f: json.dump(m.to_map(), f)) +} def pytest_generate_tests(metafunc): @@ -60,14 +70,13 @@ def test_yaml(maptype, get_obs, parse_obs): check_lines(m, EXPLINES, get_obs, parse_obs, check=checks[maptype]) -@pytest.mark.parametrize(["write", "parse"], [ - (lambda m, f: f.write(m.to_yaml()), lambda f: yaml.load(f, yaml.SafeLoader)), - (lambda m, f: json.dump(m.to_map(), f), lambda f: json.load(f)) -]) -def test_disk_roundtrip(maptype, tmpdir, write, parse): +@pytest.mark.parametrize("fmtlib", [YAML_NAME, JSON_NAME]) +def test_disk_roundtrip(maptype, tmpdir, fmtlib): """ Verify ability to parse, write, and reconstitute attmap. """ m = make_data(ENTRIES, maptype) fp = tmpdir.join("disked_attmap.out").strpath + fmtspec = FORMATTER_LIBRARIES[fmtlib] + parse, write = fmtspec.parse, fmtspec.write with open(fp, 'w') as f: write(m, f) with open(fp, 'r') as f: @@ -75,6 +84,11 @@ def test_disk_roundtrip(maptype, tmpdir, write, parse): assert recons == m.to_dict() +@pytest.mark.skip("not implemented") +def test_disk_path_expansion(): + pass + + def make_data(entries, datatype): """ Create the base data used to populate an attmap. From 654c2183175979ca5746cefa2601c662b9e9f014 Mon Sep 17 00:00:00 2001 From: Vince Reuter Date: Wed, 15 May 2019 17:53:29 -0400 Subject: [PATCH 12/16] start skip-expand-to-disk testing --- tests/test_to_disk.py | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/tests/test_to_disk.py b/tests/test_to_disk.py index 8bb2ab5..044ca29 100644 --- a/tests/test_to_disk.py +++ b/tests/test_to_disk.py @@ -2,6 +2,7 @@ from collections import namedtuple import json +import os import sys if sys.version_info < (3, 3): from collections import MutableMapping @@ -75,6 +76,7 @@ def test_disk_roundtrip(maptype, tmpdir, fmtlib): """ Verify ability to parse, write, and reconstitute attmap. """ m = make_data(ENTRIES, maptype) fp = tmpdir.join("disked_attmap.out").strpath + assert not os.path.exists(fp) fmtspec = FORMATTER_LIBRARIES[fmtlib] parse, write = fmtspec.parse, fmtspec.write with open(fp, 'w') as f: @@ -85,8 +87,27 @@ def test_disk_roundtrip(maptype, tmpdir, fmtlib): @pytest.mark.skip("not implemented") -def test_disk_path_expansion(): - pass +@pytest.mark.parametrize(["data", "env_var"], [ + ({"arbkey": os.path.join("$HOME", "leaf.md")}, "HOME"), + ({"random": os.path.join("abc/$HOME/leaf.md")}, "HOME") +]) +@pytest.mark.parametrize("fmtlib", [JSON_NAME, YAML_NAME]) +@pytest.mark.parametrize("maptype", ALL_ATTMAPS) +def test_disk_path_expansion(tmpdir, data, env_var, fmtlib, maptype): + """ Paths are not expanded when map goes to disk. """ + assert len(data) == 1, \ + "To isolate focus, use just 1 item; got {} -- {}".format(len(data), data) + fmtspec = FORMATTER_LIBRARIES[fmtlib] + parse, write = fmtspec.parse, fmtspec.write + m = make_data(list(data.items()), maptype) + assert isinstance(m, maptype) + fp = tmpdir.join("disked_attmap.out").strpath + assert not os.path.exists(fp) + with open(fp, 'w') as f: + write(data, f) + with open(fp, 'r') as f: + res = parse(f) + assert res == data def make_data(entries, datatype): From b95a6a4c6284db180483b338fe117e4c751837c1 Mon Sep 17 00:00:00 2001 From: Vince Date: Thu, 16 May 2019 01:11:17 -0400 Subject: [PATCH 13/16] isolate pathex stuff to most specific type --- attmap/_att_map_like.py | 43 ++--------------------------------------- attmap/attmap.py | 2 +- attmap/ordattmap.py | 5 ++--- attmap/pathex_attmap.py | 43 ++++++++++++++++++++++++++++++++++++----- tests/test_to_disk.py | 35 +++++++++++++++++++++------------ 5 files changed, 66 insertions(+), 62 deletions(-) diff --git a/attmap/_att_map_like.py b/attmap/_att_map_like.py index 21268de..2ebab10 100644 --- a/attmap/_att_map_like.py +++ b/attmap/_att_map_like.py @@ -31,15 +31,14 @@ def __init__(self, entries=None): def __getattr__(self, item, default=None): try: - v = super(AttMapLike, self).__getattribute__(item) + return super(AttMapLike, self).__getattribute__(item) except AttributeError: try: - v = self.__getitem__(item) + return self.__getitem__(item) except KeyError: # Requested item is unknown, but request was made via # __getitem__ syntax, not attribute-access syntax. raise AttributeError(item) - return self._finalize_value(v) @abc.abstractmethod def __delitem__(self, item): @@ -173,31 +172,6 @@ def _excl_from_repr(self, k, cls): """ return False - def _finalize_value(self, v): - """ - Make any modifications to a retrieved value before returning it. - - This hook accesses an instance's declaration of value mutations, or - transformations. That sequence may be empty (the base case), in which - case any value is simply always returned as-is. - - If an instances does declare retrieval modifications, though, the - declaration should be an iterable of pairs, in which each element's - first component is a single-argument predicate function evaluated on - the given value, and the second component of each element is the - modification to apply if the predicate is satisfied. - - At most one modification function will be called, and it would be the - first one for which the predicate was satisfied. - - :param object v: a value to consider for modification - :return object: the finalized value - """ - for p, f in self._retrieval_mutations: - if p(v): - return f(v) - return v - @abc.abstractproperty def _lower_type_bound(self): """ Most specific type to which stored Mapping should be transformed """ @@ -208,19 +182,6 @@ def _new_empty_basic_map(self): """ Return the empty collection builder for Mapping type simplification. """ pass - @property - def _retrieval_mutations(self): - """ - Hook for item transformation(s) to be applied upon retrieval. - - :return Iterable[(function(object) -> bool, function(object) -> object)]: - collection in which each element has a one-arg predicate function - and a one-arg transformation function, used to determine a - retrieved value's final form (i.e., transform it according to the - first predicate that it satisfies.) - """ - return [] - def _simplify_keyvalue(self, kvs, build, acc=None): """ Simplify a collection of key-value pairs, "reducing" to simpler types. diff --git a/attmap/attmap.py b/attmap/attmap.py index 326712b..1340b72 100644 --- a/attmap/attmap.py +++ b/attmap/attmap.py @@ -29,7 +29,7 @@ def __delitem__(self, key): _LOGGER.debug(safedel_message(key)) def __getitem__(self, item): - return self._finalize_value(self.__dict__[item]) + return self.__dict__[item] def __setitem__(self, key, value): """ diff --git a/attmap/ordattmap.py b/attmap/ordattmap.py index aa672f1..eedbd95 100644 --- a/attmap/ordattmap.py +++ b/attmap/ordattmap.py @@ -39,10 +39,9 @@ def __getitem__(self, item): according to the instance's finalization of retrieved values """ try: - v = super(OrdAttMap, self).__getitem__(item) + return super(OrdAttMap, self).__getitem__(item) except KeyError: - v = AttMap.__getitem__(self, item) - return self._finalize_value(v) + return AttMap.__getitem__(self, item) def __setitem__(self, key, value): """ Support hook for value transformation before storage. """ diff --git a/attmap/pathex_attmap.py b/attmap/pathex_attmap.py index 0b1a2e8..1328671 100644 --- a/attmap/pathex_attmap.py +++ b/attmap/pathex_attmap.py @@ -13,8 +13,41 @@ class PathExAttMap(OrdAttMap): """ Used in pepkit projects, with Mapping conversion and path expansion """ - @property - def _retrieval_mutations(self): - """ Add path expansion behavior to more general attmap. """ - return super(PathExAttMap, self)._retrieval_mutations + \ - [(lambda obj: isinstance(obj, str), expandpath)] + def __getattr__(self, item, default=None, expand=True): + try: + v = super(PathExAttMap, self).__getattribute__(item) + except AttributeError: + try: + return self.__getitem__(item, expand) + except KeyError: + # Requested item is unknown, but request was made via + # __getitem__ syntax, not attribute-access syntax. + raise AttributeError(item) + else: + return expandpath(v) if expand else v + + def __getitem__(self, item, expand=True): + v = super(PathExAttMap, self).__getitem__(item) + return self._finalize_value(v) if expand else v + + def _finalize_value(self, v): + """ + Make any modifications to a retrieved value before returning it. + + This hook accesses an instance's declaration of value mutations, or + transformations. That sequence may be empty (the base case), in which + case any value is simply always returned as-is. + + If an instances does declare retrieval modifications, though, the + declaration should be an iterable of pairs, in which each element's + first component is a single-argument predicate function evaluated on + the given value, and the second component of each element is the + modification to apply if the predicate is satisfied. + + At most one modification function will be called, and it would be the + first one for which the predicate was satisfied. + + :param object v: a value to consider for modification + :return object: the finalized value + """ + return expandpath(v) if isinstance(v, str) else v diff --git a/tests/test_to_disk.py b/tests/test_to_disk.py index 044ca29..8df544f 100644 --- a/tests/test_to_disk.py +++ b/tests/test_to_disk.py @@ -86,28 +86,39 @@ def test_disk_roundtrip(maptype, tmpdir, fmtlib): assert recons == m.to_dict() -@pytest.mark.skip("not implemented") -@pytest.mark.parametrize(["data", "env_var"], [ - ({"arbkey": os.path.join("$HOME", "leaf.md")}, "HOME"), - ({"random": os.path.join("abc/$HOME/leaf.md")}, "HOME") +@pytest.mark.parametrize(["data", "env_var", "fmtlib", "exp_res"], [ + ({"arbkey": os.path.join("$HOME", "leaf.md")}, "HOME", JSON_NAME, + ["{{arbkey: {}}}".format(os.path.join("$HOME", "leaf.md"))]), + ({"arbkey": os.path.join("$HOME", "leaf.md")}, "HOME", YAML_NAME, + ["arbkey:", " " + os.path.join("$HOME", "leaf.md")]), + ({"random": os.path.join("abc", "$HOME", "leaf.md")}, "HOME", JSON_NAME, + ["{{random: {}}}".format(os.path.join("abc", "$HOME", "leaf.md"))]), + ({"random": os.path.join("abc", "$HOME", "leaf.md")}, "HOME", YAML_NAME, + ["random:", " " + os.path.join("abc", "$HOME", "leaf.md")]) ]) -@pytest.mark.parametrize("fmtlib", [JSON_NAME, YAML_NAME]) -@pytest.mark.parametrize("maptype", ALL_ATTMAPS) -def test_disk_path_expansion(tmpdir, data, env_var, fmtlib, maptype): +def test_disk_path_expansion(tmpdir, data, env_var, fmtlib, exp_res, maptype): """ Paths are not expanded when map goes to disk. """ + + # Pretests assert len(data) == 1, \ "To isolate focus, use just 1 item; got {} -- {}".format(len(data), data) + assert os.environ.get(env_var) is not None, \ + "Null or missing env var: {}".format(env_var) + + # Create the mapping. + m = make_data(list(data.items()), maptype) + assert type(m) is maptype + + # Set write/parse strategies. fmtspec = FORMATTER_LIBRARIES[fmtlib] parse, write = fmtspec.parse, fmtspec.write - m = make_data(list(data.items()), maptype) - assert isinstance(m, maptype) + fp = tmpdir.join("disked_attmap.out").strpath assert not os.path.exists(fp) with open(fp, 'w') as f: - write(data, f) + write(m, f) with open(fp, 'r') as f: - res = parse(f) - assert res == data + assert exp_res == [l.strip("\n") for l in f.readlines()] def make_data(entries, datatype): From eae3d1539adfcda2f747e95c53565e3249914ba4 Mon Sep 17 00:00:00 2001 From: Vince Date: Thu, 16 May 2019 01:42:10 -0400 Subject: [PATCH 14/16] provide the conditional path expansion hooks; #37, #43 --- attmap/pathex_attmap.py | 32 ++++++++++++++++++++++++++++++++ tests/test_to_disk.py | 8 ++++---- 2 files changed, 36 insertions(+), 4 deletions(-) diff --git a/attmap/pathex_attmap.py b/attmap/pathex_attmap.py index 1328671..ab8c4c8 100644 --- a/attmap/pathex_attmap.py +++ b/attmap/pathex_attmap.py @@ -30,6 +30,38 @@ def __getitem__(self, item, expand=True): v = super(PathExAttMap, self).__getitem__(item) return self._finalize_value(v) if expand else v + def items(self, expand=False): + return [(k, self.__getitem__(k, expand)) for k in self] + + def values(self, expand=False): + return [self.__getitem__(k, expand) for k in self] + + def _data_for_repr(self): + """ + Hook for extracting the data used in the object's text representation. + + :return Iterable[(hashable, object)]: collection of key-value pairs + to include in object's text representation + """ + return filter(lambda kv: not self._excl_from_repr(kv[0], self.__class__), + self.items(expand=False)) + + def to_map(self, expand=False): + """ + Convert this instance to a dict. + + :return dict[str, object]: this map's data, in a simpler container + """ + return self._simplify_keyvalue(self.items(expand), self._new_empty_basic_map) + + def to_dict(self, expand=False): + """ + Return a builtin dict representation of this instance. + + :return dict: builtin dict representation of this instance + """ + return self._simplify_keyvalue(self.items(expand), dict) + def _finalize_value(self, v): """ Make any modifications to a retrieved value before returning it. diff --git a/tests/test_to_disk.py b/tests/test_to_disk.py index 8df544f..af54afc 100644 --- a/tests/test_to_disk.py +++ b/tests/test_to_disk.py @@ -88,13 +88,13 @@ def test_disk_roundtrip(maptype, tmpdir, fmtlib): @pytest.mark.parametrize(["data", "env_var", "fmtlib", "exp_res"], [ ({"arbkey": os.path.join("$HOME", "leaf.md")}, "HOME", JSON_NAME, - ["{{arbkey: {}}}".format(os.path.join("$HOME", "leaf.md"))]), + ["{{\"arbkey\": \"{}\"}}".format(os.path.join("$HOME", "leaf.md"))]), ({"arbkey": os.path.join("$HOME", "leaf.md")}, "HOME", YAML_NAME, - ["arbkey:", " " + os.path.join("$HOME", "leaf.md")]), + ["arbkey: " + os.path.join("$HOME", "leaf.md")]), ({"random": os.path.join("abc", "$HOME", "leaf.md")}, "HOME", JSON_NAME, - ["{{random: {}}}".format(os.path.join("abc", "$HOME", "leaf.md"))]), + ["{{\"random\": \"{}\"}}".format(os.path.join("abc", "$HOME", "leaf.md"))]), ({"random": os.path.join("abc", "$HOME", "leaf.md")}, "HOME", YAML_NAME, - ["random:", " " + os.path.join("abc", "$HOME", "leaf.md")]) + ["random: " + os.path.join("abc", "$HOME", "leaf.md")]) ]) def test_disk_path_expansion(tmpdir, data, env_var, fmtlib, exp_res, maptype): """ Paths are not expanded when map goes to disk. """ From ef926ec967e3cad49bef6f80e54d1cdf09868107 Mon Sep 17 00:00:00 2001 From: Vince Date: Thu, 16 May 2019 01:48:26 -0400 Subject: [PATCH 15/16] comments --- attmap/pathex_attmap.py | 34 ++++++++++++++++++++++++++++++++-- 1 file changed, 32 insertions(+), 2 deletions(-) diff --git a/attmap/pathex_attmap.py b/attmap/pathex_attmap.py index ab8c4c8..52ee9cb 100644 --- a/attmap/pathex_attmap.py +++ b/attmap/pathex_attmap.py @@ -14,6 +14,15 @@ class PathExAttMap(OrdAttMap): """ Used in pepkit projects, with Mapping conversion and path expansion """ def __getattr__(self, item, default=None, expand=True): + """ + Get attribute, accessing stored key-value pairs as needed. + + :param str item: name of attribute/key + :param object default: value to return if requested attr/key is missing + :param bool expand: whether to attempt path expansion of string value + :return object: value bound to requested name + :raise AttributeError: if requested item is unavailable + """ try: v = super(PathExAttMap, self).__getattribute__(item) except AttributeError: @@ -27,24 +36,45 @@ def __getattr__(self, item, default=None, expand=True): return expandpath(v) if expand else v def __getitem__(self, item, expand=True): + """ + Fetch the value of given key. + + :param hashable item: key for which to fetch value + :param bool expand: whether to expand string value as path + :return object: value mapped to given key, if available + :raise KeyError: if the requested key is unmapped. + """ v = super(PathExAttMap, self).__getitem__(item) return self._finalize_value(v) if expand else v def items(self, expand=False): + """ + Produce list of key-value pairs, optionally expanding paths. + + :param bool expand: whether to expand paths + :return Iterable[object]: stored key-value pairs, optionally expanded + """ return [(k, self.__getitem__(k, expand)) for k in self] def values(self, expand=False): + """ + Produce list of values, optionally expanding paths. + + :param bool expand: whether to expand paths + :return Iterable[object]: stored values, optionally expanded + """ return [self.__getitem__(k, expand) for k in self] - def _data_for_repr(self): + def _data_for_repr(self, expand=False): """ Hook for extracting the data used in the object's text representation. + :param bool expand: whether to expand paths :return Iterable[(hashable, object)]: collection of key-value pairs to include in object's text representation """ return filter(lambda kv: not self._excl_from_repr(kv[0], self.__class__), - self.items(expand=False)) + self.items(expand)) def to_map(self, expand=False): """ From 1ab3ad367a8f7c48747559a924186be910f3575b Mon Sep 17 00:00:00 2001 From: Vince Date: Thu, 16 May 2019 01:52:33 -0400 Subject: [PATCH 16/16] version finalization --- attmap/_version.py | 2 +- docs/changelog.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/attmap/_version.py b/attmap/_version.py index 9a52e92..dc64b55 100644 --- a/attmap/_version.py +++ b/attmap/_version.py @@ -1 +1 @@ -__version__ = "0.11dev" +__version__ = "0.11" diff --git a/docs/changelog.md b/docs/changelog.md index 97b2a6b..deceb9f 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -1,6 +1,6 @@ # Changelog -## Unreleased +## [0.11] - 2019-05-16 ### Added - `get_yaml_lines` to get collection of YAML-ready lines from any attmap - `to_dict` to convert any attmap (and nested maps) to base `dict`