diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bd341a5b..379b7974 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -129,7 +129,7 @@ jobs: strategy: fail-fast: true matrix: - python-version: ["3.7", "3.8", "3.9", "3.10"] + python-version: ["3.8", "3.9", "3.10", "3.11"] runs-on: "ubuntu-20.04" env: PYTHON_VER: "${{ matrix.python-version }}" diff --git a/CHANGELOG.md b/CHANGELOG.md index 3b9b4114..ed4de59f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,16 @@ # Changelog +## v1.10.0 - 2023-11-16 + +### Fixed + +- #249 - Fixes natural deletion order flag +- #247 - Fixes underspecified typing_extensions dependency + +### Changed + +- #247 - Deprecates Python 3.7 + ## v1.9.0 - 2023-10-16 ### Added diff --git a/diffsync/__init__.py b/diffsync/__init__.py index 412270d8..5669b250 100644 --- a/diffsync/__init__.py +++ b/diffsync/__init__.py @@ -14,9 +14,9 @@ See the License for the specific language governing permissions and limitations under the License. """ +import sys from inspect import isclass from typing import Callable, ClassVar, Dict, List, Optional, Tuple, Type, Union, Any, Set -from typing_extensions import Self from pydantic import BaseModel, PrivateAttr import structlog # type: ignore @@ -29,6 +29,11 @@ from diffsync.store.local import LocalStore from diffsync.utils import get_path, set_key, tree_string +if sys.version_info >= (3, 11): + from typing import Self +else: + from typing_extensions import Self + # This workaround is used because we are defining a method called `str` in our class definition, which therefore renders # the builtin `str` type unusable. StrType = str diff --git a/diffsync/helpers.py b/diffsync/helpers.py index 96882bae..c4e64c53 100644 --- a/diffsync/helpers.py +++ b/diffsync/helpers.py @@ -369,11 +369,13 @@ def sync_diff_element(self, element: DiffElement, parent_model: Optional["DiffSy natural_deletion_order = bool(dst_model.model_flags & DiffSyncModelFlags.NATURAL_DELETION_ORDER) skip_children = bool(dst_model.model_flags & DiffSyncModelFlags.SKIP_CHILDREN_ON_DELETE) + # Recurse through children to delete if we are supposed to delete the current diff element changed = False if natural_deletion_order and self.action == DiffSyncActions.DELETE and not skip_children: for child in element.get_children(): changed |= self.sync_diff_element(child, parent_model=dst_model) + # Sync the current model - this will delete the current model if self.action is DELETE changed, modified_model = self.sync_model(src_model=src_model, dst_model=dst_model, ids=ids, attrs=attrs) dst_model = modified_model or dst_model @@ -396,7 +398,7 @@ def sync_diff_element(self, element: DiffElement, parent_model: Optional["DiffSy self.incr_elements_processed() - if not natural_deletion_order: + if not natural_deletion_order or self.action is not DiffSyncActions.DELETE: for child in element.get_children(): changed |= self.sync_diff_element(child, parent_model=dst_model) diff --git a/poetry.lock b/poetry.lock index dc0752c7..50f846cf 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.5.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.6.1 and should not be changed by hand. [[package]] name = "alabaster" @@ -25,7 +25,6 @@ files = [ [package.dependencies] lazy-object-proxy = ">=1.4.0" setuptools = ">=20.0" -typed-ast = {version = ">=1.4.0,<2.0", markers = "implementation_name == \"cpython\" and python_version < \"3.8\""} typing-extensions = {version = ">=3.10", markers = "python_version < \"3.10\""} wrapt = ">=1.11,<2" @@ -40,9 +39,6 @@ files = [ {file = "async_timeout-4.0.2-py3-none-any.whl", hash = "sha256:8ca1e4fcf50d07413d66d1a5e416e42cfdf5851c981d679a09851a6853383b3c"}, ] -[package.dependencies] -typing-extensions = {version = ">=3.6.5", markers = "python_version < \"3.8\""} - [[package]] name = "attrs" version = "22.2.0" @@ -138,7 +134,6 @@ packaging = ">=22.0" pathspec = ">=0.9.0" platformdirs = ">=2" tomli = {version = ">=1.1.0", markers = "python_version < \"3.11\""} -typed-ast = {version = ">=1.4.2", markers = "python_version < \"3.8\" and implementation_name == \"cpython\""} typing-extensions = {version = ">=3.10.0.0", markers = "python_version < \"3.10\""} [package.extras] @@ -344,7 +339,6 @@ files = [ [package.dependencies] colorama = {version = "*", markers = "platform_system == \"Windows\""} -importlib-metadata = {version = "*", markers = "python_version < \"3.8\""} [[package]] name = "colorama" @@ -519,7 +513,6 @@ files = [ ] [package.dependencies] -importlib-metadata = {version = ">=1.1.0,<4.3", markers = "python_version < \"3.8\""} mccabe = ">=0.7.0,<0.8.0" pycodestyle = ">=2.9.0,<2.10.0" pyflakes = ">=2.5.0,<2.6.0" @@ -551,7 +544,6 @@ files = [ [package.dependencies] gitdb = ">=4.0.1,<5" -typing-extensions = {version = ">=3.7.4.3", markers = "python_version < \"3.8\""} [[package]] name = "idna" @@ -575,25 +567,6 @@ files = [ {file = "imagesize-1.4.1.tar.gz", hash = "sha256:69150444affb9cb0d5cc5a92b3676f0b2fb7cd9ae39e947a5e11a36b4497cd4a"}, ] -[[package]] -name = "importlib-metadata" -version = "4.2.0" -description = "Read metadata from Python packages" -optional = false -python-versions = ">=3.6" -files = [ - {file = "importlib_metadata-4.2.0-py3-none-any.whl", hash = "sha256:057e92c15bc8d9e8109738a48db0ccb31b4d9d5cfbee5a8670879a30be66304b"}, - {file = "importlib_metadata-4.2.0.tar.gz", hash = "sha256:b7e52a1f8dec14a75ea73e0891f3060099ca1d8e6a462a4dff11c3e119ea1b31"}, -] - -[package.dependencies] -typing-extensions = {version = ">=3.6.4", markers = "python_version < \"3.8\""} -zipp = ">=0.5" - -[package.extras] -docs = ["jaraco.packaging (>=8.2)", "rst.linker (>=1.9)", "sphinx"] -testing = ["flufl.flake8", "importlib-resources (>=1.3)", "packaging", "pep517", "pyfakefs", "pytest (>=4.6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.0.1)", "pytest-flake8", "pytest-mypy"] - [[package]] name = "iniconfig" version = "2.0.0" @@ -843,7 +816,6 @@ files = [ [package.dependencies] mypy-extensions = ">=1.0.0" tomli = {version = ">=1.1.0", markers = "python_version < \"3.11\""} -typed-ast = {version = ">=1.4.0,<2", markers = "python_version < \"3.8\""} typing-extensions = ">=4.1.0" [package.extras] @@ -907,9 +879,6 @@ files = [ {file = "platformdirs-3.0.0.tar.gz", hash = "sha256:8a1228abb1ef82d788f74139988b137e78692984ec7b08eaa6c65f1723af28f9"}, ] -[package.dependencies] -typing-extensions = {version = ">=4.4", markers = "python_version < \"3.8\""} - [package.extras] docs = ["furo (>=2022.12.7)", "proselint (>=0.13)", "sphinx (>=6.1.3)", "sphinx-autodoc-typehints (>=1.22,!=1.23.4)"] test = ["appdirs (==1.4.4)", "covdefaults (>=2.2.2)", "pytest (>=7.2.1)", "pytest-cov (>=4)", "pytest-mock (>=3.10)"] @@ -925,9 +894,6 @@ files = [ {file = "pluggy-1.0.0.tar.gz", hash = "sha256:4224373bacce55f955a878bf9cfa763c1e360858e330072059e10bad68531159"}, ] -[package.dependencies] -importlib-metadata = {version = ">=0.12", markers = "python_version < \"3.8\""} - [package.extras] dev = ["pre-commit", "tox"] testing = ["pytest", "pytest-benchmark"] @@ -1055,7 +1021,6 @@ files = [ ] [package.dependencies] -importlib-metadata = {version = ">=2.0.0,<5.0.0", markers = "python_version < \"3.8\""} snowballstemmer = ">=2.2.0" [package.extras] @@ -1125,7 +1090,6 @@ files = [ attrs = ">=19.2.0" colorama = {version = "*", markers = "sys_platform == \"win32\""} exceptiongroup = {version = ">=1.0.0rc8", markers = "python_version < \"3.11\""} -importlib-metadata = {version = ">=0.12", markers = "python_version < \"3.8\""} iniconfig = "*" packaging = "*" pluggy = ">=0.12,<2.0" @@ -1260,8 +1224,6 @@ files = [ [package.dependencies] async-timeout = ">=4.0.2" -importlib-metadata = {version = ">=1.0", markers = "python_version < \"3.8\""} -typing-extensions = {version = "*", markers = "python_version < \"3.8\""} [package.extras] hiredis = ["hiredis (>=1.0.0)"] @@ -1479,7 +1441,6 @@ files = [ ] [package.dependencies] -importlib-metadata = {version = ">=1.7.0", markers = "python_version < \"3.8\""} pbr = ">=2.0.0,<2.1.0 || >2.1.0" [[package]] @@ -1493,10 +1454,6 @@ files = [ {file = "structlog-22.3.0.tar.gz", hash = "sha256:e7509391f215e4afb88b1b80fa3ea074be57a5a17d794bd436a5c949da023333"}, ] -[package.dependencies] -importlib-metadata = {version = "*", markers = "python_version < \"3.8\""} -typing-extensions = {version = "*", markers = "python_version < \"3.8\""} - [package.extras] dev = ["structlog[docs,tests,typing]"] docs = ["furo", "myst-parser", "sphinx", "sphinx-notfound-page", "sphinxcontrib-mermaid", "twisted"] @@ -1525,39 +1482,6 @@ files = [ {file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"}, ] -[[package]] -name = "typed-ast" -version = "1.5.4" -description = "a fork of Python 2 and 3 ast modules with type comment support" -optional = false -python-versions = ">=3.6" -files = [ - {file = "typed_ast-1.5.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:669dd0c4167f6f2cd9f57041e03c3c2ebf9063d0757dc89f79ba1daa2bfca9d4"}, - {file = "typed_ast-1.5.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:211260621ab1cd7324e0798d6be953d00b74e0428382991adfddb352252f1d62"}, - {file = "typed_ast-1.5.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:267e3f78697a6c00c689c03db4876dd1efdfea2f251a5ad6555e82a26847b4ac"}, - {file = "typed_ast-1.5.4-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:c542eeda69212fa10a7ada75e668876fdec5f856cd3d06829e6aa64ad17c8dfe"}, - {file = "typed_ast-1.5.4-cp310-cp310-win_amd64.whl", hash = "sha256:a9916d2bb8865f973824fb47436fa45e1ebf2efd920f2b9f99342cb7fab93f72"}, - {file = "typed_ast-1.5.4-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:79b1e0869db7c830ba6a981d58711c88b6677506e648496b1f64ac7d15633aec"}, - {file = "typed_ast-1.5.4-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a94d55d142c9265f4ea46fab70977a1944ecae359ae867397757d836ea5a3f47"}, - {file = "typed_ast-1.5.4-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:183afdf0ec5b1b211724dfef3d2cad2d767cbefac291f24d69b00546c1837fb6"}, - {file = "typed_ast-1.5.4-cp36-cp36m-win_amd64.whl", hash = "sha256:639c5f0b21776605dd6c9dbe592d5228f021404dafd377e2b7ac046b0349b1a1"}, - {file = "typed_ast-1.5.4-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:cf4afcfac006ece570e32d6fa90ab74a17245b83dfd6655a6f68568098345ff6"}, - {file = "typed_ast-1.5.4-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ed855bbe3eb3715fca349c80174cfcfd699c2f9de574d40527b8429acae23a66"}, - {file = "typed_ast-1.5.4-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:6778e1b2f81dfc7bc58e4b259363b83d2e509a65198e85d5700dfae4c6c8ff1c"}, - {file = "typed_ast-1.5.4-cp37-cp37m-win_amd64.whl", hash = "sha256:0261195c2062caf107831e92a76764c81227dae162c4f75192c0d489faf751a2"}, - {file = "typed_ast-1.5.4-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:2efae9db7a8c05ad5547d522e7dbe62c83d838d3906a3716d1478b6c1d61388d"}, - {file = "typed_ast-1.5.4-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:7d5d014b7daa8b0bf2eaef684295acae12b036d79f54178b92a2b6a56f92278f"}, - {file = "typed_ast-1.5.4-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:370788a63915e82fd6f212865a596a0fefcbb7d408bbbb13dea723d971ed8bdc"}, - {file = "typed_ast-1.5.4-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:4e964b4ff86550a7a7d56345c7864b18f403f5bd7380edf44a3c1fb4ee7ac6c6"}, - {file = "typed_ast-1.5.4-cp38-cp38-win_amd64.whl", hash = "sha256:683407d92dc953c8a7347119596f0b0e6c55eb98ebebd9b23437501b28dcbb8e"}, - {file = "typed_ast-1.5.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:4879da6c9b73443f97e731b617184a596ac1235fe91f98d279a7af36c796da35"}, - {file = "typed_ast-1.5.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:3e123d878ba170397916557d31c8f589951e353cc95fb7f24f6bb69adc1a8a97"}, - {file = "typed_ast-1.5.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ebd9d7f80ccf7a82ac5f88c521115cc55d84e35bf8b446fcd7836eb6b98929a3"}, - {file = "typed_ast-1.5.4-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:98f80dee3c03455e92796b58b98ff6ca0b2a6f652120c263efdba4d6c5e58f72"}, - {file = "typed_ast-1.5.4-cp39-cp39-win_amd64.whl", hash = "sha256:0fdbcf2fef0ca421a3f5912555804296f0b0960f0418c440f5d6d3abb549f3e1"}, - {file = "typed_ast-1.5.4.tar.gz", hash = "sha256:39e21ceb7388e4bb37f4c679d72707ed46c2fbf2a5609b8b8ebc4b067d977df2"}, -] - [[package]] name = "types-pyopenssl" version = "23.0.0.4" @@ -1636,13 +1560,13 @@ files = [ [[package]] name = "typing-extensions" -version = "4.5.0" -description = "Backported and Experimental Type Hints for Python 3.7+" +version = "4.8.0" +description = "Backported and Experimental Type Hints for Python 3.8+" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "typing_extensions-4.5.0-py3-none-any.whl", hash = "sha256:fb33085c39dd998ac16d1431ebc293a8b3eedd00fd4a32de0ff79002c19511b4"}, - {file = "typing_extensions-4.5.0.tar.gz", hash = "sha256:5cb5f4a79139d699607b3ef622a1dedafa84e115ab0024e0d9c044a9479ca7cb"}, + {file = "typing_extensions-4.8.0-py3-none-any.whl", hash = "sha256:8f92fc8806f9a6b641eaa5318da32b44d401efaac0f6678c9bc448ba3605faa0"}, + {file = "typing_extensions-4.8.0.tar.gz", hash = "sha256:df8e4339e9cb77357558cbdbceca33c303714cf861d1eef15e1070055ae8b7ef"}, ] [[package]] @@ -1761,25 +1685,10 @@ pathspec = ">=0.5.3" pyyaml = "*" setuptools = "*" -[[package]] -name = "zipp" -version = "3.15.0" -description = "Backport of pathlib-compatible object wrapper for zip files" -optional = false -python-versions = ">=3.7" -files = [ - {file = "zipp-3.15.0-py3-none-any.whl", hash = "sha256:48904fc76a60e542af151aded95726c1a5c34ed43ab4134b597665c86d7ad556"}, - {file = "zipp-3.15.0.tar.gz", hash = "sha256:112929ad649da941c23de50f356a2b5570c954b65150642bccdd66bf194d224b"}, -] - -[package.extras] -docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] -testing = ["big-O", "flake8 (<5)", "jaraco.functools", "jaraco.itertools", "more-itertools", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)"] - [extras] redis = ["redis"] [metadata] lock-version = "2.0" -python-versions = "^3.7" -content-hash = "2213a75a00f27293bb23f2a74792c2f8e46b97148e9197ddf620c9239d88b634" +python-versions = ">=3.8,<4.0" +content-hash = "d9fde9c831f492bddd25a304bee6047d730b94b2e3240325cf456e24b84aa811" diff --git a/pyproject.toml b/pyproject.toml index c215ca1e..ed09e16d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "diffsync" -version = "1.9.0" +version = "1.10.0" description = "Library to easily sync/diff/update 2 different data sources" authors = ["Network to Code, LLC "] license = "Apache-2.0" @@ -16,12 +16,14 @@ include = [ ] [tool.poetry.dependencies] -python = "^3.7" +python = ">=3.8,<4.0" pydantic = "^1.7.4,!=1.8,!=1.8.1" structlog = ">= 20.1.0, < 23.0.0" packaging = ">= 21.3, < 24.0" colorama = {version = "^0.4.3", optional = true} redis = {version = "^4.3", optional = true} +# typing.Self introduced in 3.11 +typing-extensions = { version = ">=4.0.1", python = "<3.11" } [tool.poetry.extras] redis = ["redis"] diff --git a/tasks.py b/tasks.py index c1efdf3c..2ae20b3b 100644 --- a/tasks.py +++ b/tasks.py @@ -47,7 +47,7 @@ def is_truthy(arg): # Can be set to a separate Python version to be used for launching or building image -PYTHON_VER = os.getenv("PYTHON_VER", os.getenv("TRAVIS_PYTHON_VERSION", "3.7")) +PYTHON_VER = os.getenv("PYTHON_VER", "3.8") # Name of the docker image/image NAME = os.getenv("IMAGE_NAME", f"diffsync-py{PYTHON_VER}") # Tag for the image diff --git a/tests/unit/test_diffsync_model_flags.py b/tests/unit/test_diffsync_model_flags.py index 76457ccf..8c10499d 100644 --- a/tests/unit/test_diffsync_model_flags.py +++ b/tests/unit/test_diffsync_model_flags.py @@ -162,3 +162,67 @@ def load(self): source.remove(source.get("parent", {"name": "Test-Parent"}), remove_children=True) source.sync_to(destination) assert call_order == ["Test-Child", "Test-Parent"] + + +def test_natural_deletion_order_with_noop_parent(): + """Test whether children are recursed through when natural deletion order is set and the parent has no changes.""" + call_order = [] + + class ChildModel(DiffSyncModel): + """Test child model that reports when its update method is called.""" + + _modelname = "child" + _identifiers = ("name",) + _attributes = ("attribute",) + + name: str + attribute: str + + def update(self, attrs): + call_order.append("Update on child") + return super().update(attrs) + + class ParentModel(DiffSyncModel): + """Test parent model.""" + + _modelname = "parent" + _identifiers = ("name",) + _attributes = ("attribute",) + _children = {"child": "children"} + + name: str + attribute: str + children: List[ChildModel] = [] + + class Adapter(DiffSync): + """Test adapter.""" + + top_level = ["parent"] + + parent = ParentModel + child = ChildModel + + def load(self, is_source=False) -> None: + """Test load method. Generate a difference with the is_source parameter.""" + parent = self.parent(name="Test Parent", attribute="This doesn't change") + parent.model_flags |= DiffSyncModelFlags.NATURAL_DELETION_ORDER + self.add(parent) + if is_source: + child = self.child(name="Test Child", attribute="Attribute from source") + child.model_flags |= DiffSyncModelFlags.NATURAL_DELETION_ORDER + parent.add_child(child) + self.add(child) + else: + child = self.child(name="Test Child", attribute="Attribute from destination") + child.model_flags |= DiffSyncModelFlags.NATURAL_DELETION_ORDER + parent.add_child(child) + self.add(child) + + source_adapter = Adapter() + source_adapter.load(is_source=True) + destination_adapter = Adapter() + destination_adapter.load() + + source_adapter.sync_to(destination_adapter) + + assert "Update on child" in call_order