From 03c83b87041b73a29e6666ecbeddca83420639b2 Mon Sep 17 00:00:00 2001 From: dluftspring Date: Sun, 29 May 2022 14:20:53 -0400 Subject: [PATCH 1/3] LICENSE and scaffolding file updates --- .gitignore | 5 ++- LICENSE | 13 +++++++ poetry.lock | 96 +++++++++++++++++++++++++++++++++++++++++++++++++- pyproject.toml | 4 ++- 4 files changed, 115 insertions(+), 3 deletions(-) create mode 100644 LICENSE diff --git a/.gitignore b/.gitignore index 9a869a7..1004c25 100644 --- a/.gitignore +++ b/.gitignore @@ -133,4 +133,7 @@ dmypy.json #don't send configs to version control .secrets/* -config.json \ No newline at end of file +config.json + +#local notebooks directory +notebooks/ \ No newline at end of file diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..41cf3a9 --- /dev/null +++ b/LICENSE @@ -0,0 +1,13 @@ +Copyright 2021 Daniel Luftspring + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. \ No newline at end of file diff --git a/poetry.lock b/poetry.lock index 6e26ac0..90af831 100644 --- a/poetry.lock +++ b/poetry.lock @@ -138,6 +138,20 @@ category = "main" optional = false python-versions = ">=3.5" +[[package]] +name = "dictdiffer" +version = "0.9.0" +description = "Dictdiffer is a library that helps you to diff and patch dictionaries." +category = "main" +optional = false +python-versions = "*" + +[package.extras] +all = ["Sphinx (>=3)", "sphinx-rtd-theme (>=0.2)", "check-manifest (>=0.42)", "mock (>=1.3.0)", "pytest-cov (>=2.10.1)", "pytest-isort (>=1.2.0)", "sphinx (>=3)", "tox (>=3.7.0)", "numpy (>=1.13.0)", "numpy (>=1.15.0)", "numpy (>=1.18.0)", "pytest (==5.4.3)", "pytest-pycodestyle (>=2)", "pytest-pydocstyle (>=2)", "pytest (>=6)", "pytest-pycodestyle (>=2.2.0)", "pytest-pydocstyle (>=2.2.0)", "numpy (>=1.20.0)"] +docs = ["Sphinx (>=3)", "sphinx-rtd-theme (>=0.2)"] +numpy = ["numpy (>=1.13.0)", "numpy (>=1.15.0)", "numpy (>=1.18.0)", "numpy (>=1.20.0)"] +tests = ["check-manifest (>=0.42)", "mock (>=1.3.0)", "pytest-cov (>=2.10.1)", "pytest-isort (>=1.2.0)", "sphinx (>=3)", "tox (>=3.7.0)", "pytest (==5.4.3)", "pytest-pycodestyle (>=2)", "pytest-pydocstyle (>=2)", "pytest (>=6)", "pytest-pycodestyle (>=2.2.0)", "pytest-pydocstyle (>=2.2.0)"] + [[package]] name = "greenlet" version = "1.1.2" @@ -244,6 +258,14 @@ category = "main" optional = false python-versions = "*" +[[package]] +name = "numpy" +version = "1.22.4" +description = "NumPy is the fundamental package for array computing with Python." +category = "main" +optional = false +python-versions = ">=3.8" + [[package]] name = "packaging" version = "21.3" @@ -255,6 +277,27 @@ python-versions = ">=3.6" [package.dependencies] pyparsing = ">=2.0.2,<3.0.5 || >3.0.5" +[[package]] +name = "pandas" +version = "1.4.2" +description = "Powerful data structures for data analysis, time series, and statistics" +category = "main" +optional = false +python-versions = ">=3.8" + +[package.dependencies] +numpy = [ + {version = ">=1.18.5", markers = "platform_machine != \"aarch64\" and platform_machine != \"arm64\" and python_version < \"3.10\""}, + {version = ">=1.19.2", markers = "platform_machine == \"aarch64\" and python_version < \"3.10\""}, + {version = ">=1.20.0", markers = "platform_machine == \"arm64\" and python_version < \"3.10\""}, + {version = ">=1.21.0", markers = "python_version >= \"3.10\""}, +] +python-dateutil = ">=2.8.1" +pytz = ">=2020.1" + +[package.extras] +test = ["hypothesis (>=5.5.3)", "pytest (>=6.0)", "pytest-xdist (>=1.31)"] + [[package]] name = "pathspec" version = "0.9.0" @@ -572,7 +615,7 @@ socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] [metadata] lock-version = "1.1" python-versions = ">3.8, <3.11" -content-hash = "11443bf4e80379ea33e5ef2f8371e73c43e499402917095c0a9520068b712d35" +content-hash = "9d2a32e901a6c9f6926afae17de5e19b1c3add4e6bddcb76f3ce354c81f11bdb" [metadata.files] atomicwrites = [ @@ -687,6 +730,10 @@ decorator = [ {file = "decorator-5.1.1-py3-none-any.whl", hash = "sha256:b8c3f85900b9dc423225913c5aace94729fe1fa9763b38939a95226f02d37186"}, {file = "decorator-5.1.1.tar.gz", hash = "sha256:637996211036b6385ef91435e4fae22989472f9d571faba8927ba8253acbc330"}, ] +dictdiffer = [ + {file = "dictdiffer-0.9.0-py2.py3-none-any.whl", hash = "sha256:442bfc693cfcadaf46674575d2eba1c53b42f5e404218ca2c2ff549f2df56595"}, + {file = "dictdiffer-0.9.0.tar.gz", hash = "sha256:17bacf5fbfe613ccf1b6d512bd766e6b21fb798822a133aa86098b8ac9997578"}, +] greenlet = [ {file = "greenlet-1.1.2-cp27-cp27m-macosx_10_14_x86_64.whl", hash = "sha256:58df5c2a0e293bf665a51f8a100d3e9956febfbf1d9aaf8c0677cf70218910c6"}, {file = "greenlet-1.1.2-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:aec52725173bd3a7b56fe91bc56eccb26fbdff1386ef123abb63c84c5b43b63a"}, @@ -799,10 +846,57 @@ mypy-extensions = [ {file = "mypy_extensions-0.4.3-py2.py3-none-any.whl", hash = "sha256:090fedd75945a69ae91ce1303b5824f428daf5a028d2f6ab8a299250a846f15d"}, {file = "mypy_extensions-0.4.3.tar.gz", hash = "sha256:2d82818f5bb3e369420cb3c4060a7970edba416647068eb4c5343488a6c604a8"}, ] +numpy = [ + {file = "numpy-1.22.4-cp310-cp310-macosx_10_14_x86_64.whl", hash = "sha256:ba9ead61dfb5d971d77b6c131a9dbee62294a932bf6a356e48c75ae684e635b3"}, + {file = "numpy-1.22.4-cp310-cp310-macosx_10_15_x86_64.whl", hash = "sha256:1ce7ab2053e36c0a71e7a13a7475bd3b1f54750b4b433adc96313e127b870887"}, + {file = "numpy-1.22.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:7228ad13744f63575b3a972d7ee4fd61815b2879998e70930d4ccf9ec721dce0"}, + {file = "numpy-1.22.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:43a8ca7391b626b4c4fe20aefe79fec683279e31e7c79716863b4b25021e0e74"}, + {file = "numpy-1.22.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a911e317e8c826ea632205e63ed8507e0dc877dcdc49744584dfc363df9ca08c"}, + {file = "numpy-1.22.4-cp310-cp310-win32.whl", hash = "sha256:9ce7df0abeabe7fbd8ccbf343dc0db72f68549856b863ae3dd580255d009648e"}, + {file = "numpy-1.22.4-cp310-cp310-win_amd64.whl", hash = "sha256:3e1ffa4748168e1cc8d3cde93f006fe92b5421396221a02f2274aab6ac83b077"}, + {file = "numpy-1.22.4-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:59d55e634968b8f77d3fd674a3cf0b96e85147cd6556ec64ade018f27e9479e1"}, + {file = "numpy-1.22.4-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:c1d937820db6e43bec43e8d016b9b3165dcb42892ea9f106c70fb13d430ffe72"}, + {file = "numpy-1.22.4-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d4c5d5eb2ec8da0b4f50c9a843393971f31f1d60be87e0fb0917a49133d257d6"}, + {file = "numpy-1.22.4-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:64f56fc53a2d18b1924abd15745e30d82a5782b2cab3429aceecc6875bd5add0"}, + {file = "numpy-1.22.4-cp38-cp38-win32.whl", hash = "sha256:fb7a980c81dd932381f8228a426df8aeb70d59bbcda2af075b627bbc50207cba"}, + {file = "numpy-1.22.4-cp38-cp38-win_amd64.whl", hash = "sha256:e96d7f3096a36c8754207ab89d4b3282ba7b49ea140e4973591852c77d09eb76"}, + {file = "numpy-1.22.4-cp39-cp39-macosx_10_14_x86_64.whl", hash = "sha256:4c6036521f11a731ce0648f10c18ae66d7143865f19f7299943c985cdc95afb5"}, + {file = "numpy-1.22.4-cp39-cp39-macosx_10_15_x86_64.whl", hash = "sha256:b89bf9b94b3d624e7bb480344e91f68c1c6c75f026ed6755955117de00917a7c"}, + {file = "numpy-1.22.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:2d487e06ecbf1dc2f18e7efce82ded4f705f4bd0cd02677ffccfb39e5c284c7e"}, + {file = "numpy-1.22.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f3eb268dbd5cfaffd9448113539e44e2dd1c5ca9ce25576f7c04a5453edc26fa"}, + {file = "numpy-1.22.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:37431a77ceb9307c28382c9773da9f306435135fae6b80b62a11c53cfedd8802"}, + {file = "numpy-1.22.4-cp39-cp39-win32.whl", hash = "sha256:cc7f00008eb7d3f2489fca6f334ec19ca63e31371be28fd5dad955b16ec285bd"}, + {file = "numpy-1.22.4-cp39-cp39-win_amd64.whl", hash = "sha256:f0725df166cf4785c0bc4cbfb320203182b1ecd30fee6e541c8752a92df6aa32"}, + {file = "numpy-1.22.4-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0791fbd1e43bf74b3502133207e378901272f3c156c4df4954cad833b1380207"}, + {file = "numpy-1.22.4.zip", hash = "sha256:425b390e4619f58d8526b3dcf656dde069133ae5c240229821f01b5f44ea07af"}, +] packaging = [ {file = "packaging-21.3-py3-none-any.whl", hash = "sha256:ef103e05f519cdc783ae24ea4e2e0f508a9c99b2d4969652eed6a2e1ea5bd522"}, {file = "packaging-21.3.tar.gz", hash = "sha256:dd47c42927d89ab911e606518907cc2d3a1f38bbd026385970643f9c5b8ecfeb"}, ] +pandas = [ + {file = "pandas-1.4.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:be67c782c4f1b1f24c2f16a157e12c2693fd510f8df18e3287c77f33d124ed07"}, + {file = "pandas-1.4.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:5a206afa84ed20e07603f50d22b5f0db3fb556486d8c2462d8bc364831a4b417"}, + {file = "pandas-1.4.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:0010771bd9223f7afe5f051eb47c4a49534345dfa144f2f5470b27189a4dd3b5"}, + {file = "pandas-1.4.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3228198333dd13c90b6434ddf61aa6d57deaca98cf7b654f4ad68a2db84f8cfe"}, + {file = "pandas-1.4.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5b79af3a69e5175c6fa7b4e046b21a646c8b74e92c6581a9d825687d92071b51"}, + {file = "pandas-1.4.2-cp310-cp310-win_amd64.whl", hash = "sha256:5586cc95692564b441f4747c47c8a9746792e87b40a4680a2feb7794defb1ce3"}, + {file = "pandas-1.4.2-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:061609334a8182ab500a90fe66d46f6f387de62d3a9cb9aa7e62e3146c712167"}, + {file = "pandas-1.4.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:b8134651258bce418cb79c71adeff0a44090c98d955f6953168ba16cc285d9f7"}, + {file = "pandas-1.4.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:df82739e00bb6daf4bba4479a40f38c718b598a84654cbd8bb498fd6b0aa8c16"}, + {file = "pandas-1.4.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:385c52e85aaa8ea6a4c600a9b2821181a51f8be0aee3af6f2dcb41dafc4fc1d0"}, + {file = "pandas-1.4.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:295872bf1a09758aba199992c3ecde455f01caf32266d50abc1a073e828a7b9d"}, + {file = "pandas-1.4.2-cp38-cp38-win32.whl", hash = "sha256:95c1e422ced0199cf4a34385ff124b69412c4bc912011ce895582bee620dfcaa"}, + {file = "pandas-1.4.2-cp38-cp38-win_amd64.whl", hash = "sha256:5c54ea4ef3823108cd4ec7fb27ccba4c3a775e0f83e39c5e17f5094cb17748bc"}, + {file = "pandas-1.4.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:c072c7f06b9242c855ed8021ff970c0e8f8b10b35e2640c657d2a541c5950f59"}, + {file = "pandas-1.4.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:f549097993744ff8c41b5e8f2f0d3cbfaabe89b4ae32c8c08ead6cc535b80139"}, + {file = "pandas-1.4.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:ff08a14ef21d94cdf18eef7c569d66f2e24e0bc89350bcd7d243dd804e3b5eb2"}, + {file = "pandas-1.4.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8c5bf555b6b0075294b73965adaafb39cf71c312e38c5935c93d78f41c19828a"}, + {file = "pandas-1.4.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:51649ef604a945f781105a6d2ecf88db7da0f4868ac5d45c51cb66081c4d9c73"}, + {file = "pandas-1.4.2-cp39-cp39-win32.whl", hash = "sha256:d0d4f13e4be7ce89d7057a786023c461dd9370040bdb5efa0a7fe76b556867a0"}, + {file = "pandas-1.4.2-cp39-cp39-win_amd64.whl", hash = "sha256:09d8be7dd9e1c4c98224c4dfe8abd60d145d934e9fc1f5f411266308ae683e6a"}, + {file = "pandas-1.4.2.tar.gz", hash = "sha256:92bc1fc585f1463ca827b45535957815b7deb218c549b7c18402c322c7549a12"}, +] pathspec = [ {file = "pathspec-0.9.0-py2.py3-none-any.whl", hash = "sha256:7d15c4ddb0b5c802d161efc417ec1a2558ea2653c2e8ad9c19098201dc1c993a"}, {file = "pathspec-0.9.0.tar.gz", hash = "sha256:e564499435a2673d586f6b2130bb5b95f04a3ba06f81b8f895b651a3c76aabb1"}, diff --git a/pyproject.toml b/pyproject.toml index 831ee94..f1e24fa 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "tap-hellobaton" -description = "Singer.io tap for extracting data from the Essesnys api" +description = "Singer tap for extracting data from the Baton api" version = "0.0.3" license = "Apache 2.0" authors = ["Daniel Luftspring"] @@ -21,6 +21,8 @@ pytest = "^6.2.5" mypy = "^0.931" types-requests = "^2.27.7" black = "^21.12b0" +dictdiffer = "^0.9.0" +pandas = "^1.4.2" [tool.poetry.scripts] tap-hellobaton = "tap_hellobaton.tap:Taphellobaton.cli" From 0f21082c9cd492c8b981fd4628cfce709a6115e5 Mon Sep 17 00:00:00 2001 From: dluftspring Date: Fri, 3 Jun 2022 17:42:20 -0400 Subject: [PATCH 2/3] Working sync CLI without making edits to necessary python files --- poetry.lock | 17 ++- pyproject.toml | 5 +- scripts/__init__.py | 0 scripts/add_schema.py | 4 + scripts/swagger_sync.py | 253 ++++++++++++++++++++++++++++++++++++++++ 5 files changed, 277 insertions(+), 2 deletions(-) create mode 100644 scripts/__init__.py create mode 100644 scripts/add_schema.py create mode 100644 scripts/swagger_sync.py diff --git a/poetry.lock b/poetry.lock index 90af831..b7b1512 100644 --- a/poetry.lock +++ b/poetry.lock @@ -556,6 +556,17 @@ postgresql_psycopg2cffi = ["psycopg2cffi"] pymysql = ["pymysql (<1)", "pymysql"] sqlcipher = ["sqlcipher3-binary"] +[[package]] +name = "tabulate" +version = "0.8.9" +description = "Pretty-print tabular data" +category = "main" +optional = false +python-versions = "*" + +[package.extras] +widechars = ["wcwidth"] + [[package]] name = "toml" version = "0.10.2" @@ -615,7 +626,7 @@ socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] [metadata] lock-version = "1.1" python-versions = ">3.8, <3.11" -content-hash = "9d2a32e901a6c9f6926afae17de5e19b1c3add4e6bddcb76f3ce354c81f11bdb" +content-hash = "b2b92929c59b5780c1426c4cc1ee85151edb1f5dd3611020069f2cd88e62732c" [metadata.files] atomicwrites = [ @@ -1064,6 +1075,10 @@ sqlalchemy = [ {file = "SQLAlchemy-1.4.31-cp39-cp39-win_amd64.whl", hash = "sha256:9e4fb2895b83993831ba2401b6404de953fdbfa9d7d4fa6a4756294a83bbc94f"}, {file = "SQLAlchemy-1.4.31.tar.gz", hash = "sha256:582b59d1e5780a447aada22b461e50b404a9dc05768da1d87368ad8190468418"}, ] +tabulate = [ + {file = "tabulate-0.8.9-py3-none-any.whl", hash = "sha256:d7c013fe7abbc5e491394e10fa845f8f32fe54f8dc60c6622c6cf482d25d47e4"}, + {file = "tabulate-0.8.9.tar.gz", hash = "sha256:eb1d13f25760052e8931f2ef80aaf6045a6cceb47514db8beab24cded16f13a7"}, +] toml = [ {file = "toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b"}, {file = "toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"}, diff --git a/pyproject.toml b/pyproject.toml index f1e24fa..3dc9dbf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,7 +8,8 @@ readme = "README.md" repository = "https://github.com/dluftspring/tap-hellobaton" packages = [ - { include = "tap_hellobaton"} + { include = "tap_hellobaton" }, + { include = "scripts" } ] [tool.poetry.dependencies] @@ -23,9 +24,11 @@ types-requests = "^2.27.7" black = "^21.12b0" dictdiffer = "^0.9.0" pandas = "^1.4.2" +tabulate = "^0.8.9" [tool.poetry.scripts] tap-hellobaton = "tap_hellobaton.tap:Taphellobaton.cli" +sync-tap = "scripts.swagger_sync:main" [build-system] requires = ["poetry"] diff --git a/scripts/__init__.py b/scripts/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/scripts/add_schema.py b/scripts/add_schema.py new file mode 100644 index 0000000..b4fcaba --- /dev/null +++ b/scripts/add_schema.py @@ -0,0 +1,4 @@ +""" +Command line utility to generate schema files from JSON response files. +""" +#@TODO implement this \ No newline at end of file diff --git a/scripts/swagger_sync.py b/scripts/swagger_sync.py new file mode 100644 index 0000000..a09e671 --- /dev/null +++ b/scripts/swagger_sync.py @@ -0,0 +1,253 @@ +""" +Command line utility to generate schema files from the baton swagger open API spec https://app.hellobaton.com/api/swagger.json. +""" + +import glob +import json +import operator +from os import PathLike +from typing import List, Mapping, Tuple, Generator +from pathlib import Path +from functools import reduce +from jsonschema import Draft7Validator +import requests +import dictdiffer +import pandas as pd +import tabulate +import click + + +SWAGGER_URL: str = 'https://app.hellobaton.com/api/swagger.json' +# for the first few cycles of this implementation, we will only allow the swagger spec to add properties to our schemas +# we will not allow the swagger spec to remove properties or change existing properties that a human has already validated +ALLOWED_CHANGES = ['add'] + +class OpenApiSpec(object): + + """ + Base object that processes the open api spec from swagger + """ + + def __init__(self): + + self.spec = self._get_spec() + self.url = SWAGGER_URL + + def _get_spec(self) -> dict: + response = requests.get(SWAGGER_URL) + assert response.status_code == 200, "Failed to get swagger spec from {}".format(SWAGGER_URL) + return response.json() + + def openapi_to_jsonschema(self, openapi_schema: dict) -> dict: + + """ + Convert OpenAPI schema into valid json schema + """ + + json_schema = dict(required=list(), type='object', properties=dict()) + json_schema['required'] = openapi_schema.get('required', list()) + + # restrict openAPI schema metadata we extract + valid_keys = ['type', 'format'] + for key, value in openapi_schema['properties'].items(): + + # if the value is a reference to another schema, we need to extract the object properties of the reference + ref_value = value.get('$ref', None) + complex_type = None + properties = None + if ref_value: + schema_ref = value.get('$ref').split('/')[-1] + complex_type = self.spec['definitions'][schema_ref]['type'] + # Recurse here because the nested objects conform to openAPI not jsonschema + properties = self.openapi_to_jsonschema(self.spec['definitions'][schema_ref])['properties'] + + type = value.get('type', complex_type or 'string') + format = value.get('format', None) + + if 'x-nullable' in value.keys(): + type = [type] + type.append("null") + + json_schema['properties'][key] = dict(type=type) + + # We have to add format only where it exists. Not all entries of json schema have a format string + if format: + json_schema['properties'][key]['format'] = format + + # Same idea as above but for the properties of complex json objects like arrays and nested json blobs + if properties: + json_schema['properties'][key]['properties'] = properties + + + return json_schema + + + def convert_to_schemas(self) -> Mapping[str, dict]: + + """ + Parse the spec for all available endpoints and return a dictionary of valid json schemas. This method validates the schema against Draft7. + NOTE: This method will not validate json instances against the schema produced it only checks for internal validity of the schema itself. + """ + + schemas = {} + for path in self.spec['paths']: + if not ('{' in path or '}' in path): + for method in self.spec['paths'][path]: + if method == 'get': + # Convert the openAPI paths e.g. /{endpoint}/ to 'endpoint' so they can be valid dictionary keys + schema_key = path.replace('/','') + # Get the relevant schema definition path from the response + definition = self.spec['paths'][path][method]['responses']['200']['schema']['properties']['results']['items']['$ref'].split('/')[-1] + valid_schema = self.openapi_to_jsonschema(self.spec['definitions'][definition]) + + schemas[schema_key] = valid_schema + + # This check should return None if it passes + assert Draft7Validator.check_schema(valid_schema) is None, "Schema {} is not valid".format(schema_key) + + return schemas + +class SchemaUpdater(object): + + """ + Determines which schemas to update and writes them to the schemas directory + """ + + def __init__(self, new_schemas: Mapping[str, dict], allowed_changes: List[str] = ALLOWED_CHANGES): + self.new_schemas = new_schemas + self.schema_dir: PathLike = Path(__file__).parent.parent / Path('tap_hellobaton') / Path('schemas') + self.existing_schemas: Mapping[str, dict] = self._read_schemas() + self.allowed_changes: List[str] = allowed_changes + self.updated_schemas: Mapping[str, dict] = { k:{} for k in self.new_schemas.keys() } + + def _read_schemas(self) -> Mapping[str, dict]: + """ + Reads all schemas from the tap schemas directory + """ + + schemas = {} + for schema_file in glob.glob(str(self.schema_dir) + '/*.json'): + with open(schema_file) as f: + dict_key = schema_file.split('/')[-1].split('.')[0] + schemas[dict_key] = json.load(f) + + return schemas + + def _process_changes(self, diff: Generator[Tuple, None, None]) -> pd.DataFrame: + + """ + Process the incoming changeset into a more readable dataframe with the follow structure + + | type | path | change | + |-------|-----|--------| + | add | actor.name | John Doe | + + where + type - the type of change e.g. add, change, remove + path - the path of the schema being changed separated by dots e.g. key1.key2.key3 + old_value - the existing value of the schema at the path specified before the change + new_value - the incoming change to the schema at the path specified + """ + + init_dict = dict(type=list(), path=list(), old_value=list(), new_value=list()) + + for t, p, c in diff: + if t in ['add', 'remove']: + change_type = t + + for change in c: + # If a completely new schema is being added, the path is going to be missing or root so we get the path from the first entry of the changeset + json_path = '.'.join([p, change[0]]) if p else change[0] + new_value = None if t == 'remove' else change[1] + old_value = None if t == 'add' else change[1] + + elif t == 'change': + change_type = t + json_path = p + + + else: + raise ValueError("Unknown change type {}".format(t)) + + init_dict['type'].append(change_type) + init_dict['path'].append(json_path) + init_dict['old_value'].append(old_value) + init_dict['new_value'].append(new_value) + + diff_df = pd.DataFrame(init_dict) + return diff_df + + def update_schemas(self) -> None: + """ + Update schemas and write to a new directory called schemas + """ + + for key in self.existing_schemas.keys(): + self.updated_schemas[key] = self.existing_schemas[key] + + for path in self.calculate_diff(): + schema_path = path.split('.') + leaf_to_set = schema_path[-1] + new_val = reduce(operator.getitem, schema_path, self.new_schemas) + reduce(operator.getitem, schema_path[:-1], self.updated_schemas)[leaf_to_set] = new_val + + for key in self.updated_schemas.keys(): + with open(str(self.schema_dir) + '/' + key + '.json', 'w') as f: + json.dump(self.updated_schemas[key], f, indent=4) + + + def calculate_diff(self) -> List[str]: + """ + Summarize the changes being written into the schemas directory + + :returns: A list of keys that should be updated with the values in self.new_schemas + """ + + raw_diff = dictdiffer.diff(self.existing_schemas, self.new_schemas) + keys_to_update = [] + + for type, path, changes in raw_diff: + if type in self.allowed_changes: + for change in changes: + dict_path = '.'.join([path, change[0]]) + if not path: + dict_path = change[0] + keys_to_update.append(dict_path) + + return keys_to_update + + def change_report(self) -> None: + """ + Print a report of the changes being written into the schemas directory + """ + raw_diff = dictdiffer.diff(self.existing_schemas, self.new_schemas) + diff = self._process_changes(raw_diff) + + print("INFO - Your allowed changes are {}".format(self.allowed_changes)) + print(diff.loc[diff.type.isin(self.allowed_changes)].to_markdown(index=False)) + + + +class AstSchemaTemplate(object): + + """ + AST template class required for safely modifying this repositories python files + """ + def __init__(self): + pass + +@click.command() +def main(): + """ + Update schemas in the schemas directory based on the current swagger spec and open api definition + """ + + spec = OpenApiSpec() + incoming_schemas = spec.convert_to_schemas() + + updater = SchemaUpdater(incoming_schemas) + print(f"Updating schemas... \n{updater.change_report()}") + updater.update_schemas() + +if __name__ == "__main__": + main() \ No newline at end of file From 57293eca3d35328027faa14329bf31482710dc82 Mon Sep 17 00:00:00 2001 From: dluftspring Date: Thu, 29 Dec 2022 11:03:36 -0500 Subject: [PATCH 3/3] Sync script from swagger spec --- scripts/swagger_sync.py | 174 ++++++++++-- scripts/templates.py | 12 + tap_hellobaton/schemas/activity.json | 233 +++++++--------- tap_hellobaton/schemas/comments.json | 19 +- tap_hellobaton/schemas/companies.json | 15 +- .../schemas/custom_field_options.json | 18 +- .../schemas/custom_field_values.json | 41 ++- tap_hellobaton/schemas/custom_fields.json | 36 ++- tap_hellobaton/schemas/external_tasks.json | 42 +++ .../schemas/milestone_feedback.json | 44 +++ tap_hellobaton/schemas/milestones.json | 59 +++- tap_hellobaton/schemas/phases.json | 12 +- .../schemas/project_attachments.json | 30 +- tap_hellobaton/schemas/project_phases.json | 41 +++ tap_hellobaton/schemas/project_users.json | 27 +- tap_hellobaton/schemas/projects.json | 131 +++++++-- tap_hellobaton/schemas/task_attachments.json | 48 +++- tap_hellobaton/schemas/task_deliverables.json | 24 ++ tap_hellobaton/schemas/tasks.json | 261 +++++++++++------- tap_hellobaton/schemas/templates.json | 82 ++++-- tap_hellobaton/schemas/time_entries.json | 54 +++- tap_hellobaton/schemas/users.json | 30 +- tap_hellobaton/streams.py | 160 ++++++----- tap_hellobaton/tap.py | 31 ++- 24 files changed, 1144 insertions(+), 480 deletions(-) create mode 100644 scripts/templates.py create mode 100644 tap_hellobaton/schemas/external_tasks.json create mode 100644 tap_hellobaton/schemas/milestone_feedback.json create mode 100644 tap_hellobaton/schemas/project_phases.json create mode 100644 tap_hellobaton/schemas/task_deliverables.json diff --git a/scripts/swagger_sync.py b/scripts/swagger_sync.py index a09e671..a7a0f46 100644 --- a/scripts/swagger_sync.py +++ b/scripts/swagger_sync.py @@ -4,16 +4,17 @@ import glob import json +import ast import operator from os import PathLike -from typing import List, Mapping, Tuple, Generator +from typing import Any, Iterable, List, Mapping, Tuple, Generator from pathlib import Path from functools import reduce from jsonschema import Draft7Validator +from .templates import STREAM_TEMPLATE import requests import dictdiffer import pandas as pd -import tabulate import click @@ -46,7 +47,7 @@ def openapi_to_jsonschema(self, openapi_schema: dict) -> dict: json_schema = dict(required=list(), type='object', properties=dict()) json_schema['required'] = openapi_schema.get('required', list()) - + # restrict openAPI schema metadata we extract valid_keys = ['type', 'format'] for key, value in openapi_schema['properties'].items(): @@ -73,14 +74,14 @@ def openapi_to_jsonschema(self, openapi_schema: dict) -> dict: # We have to add format only where it exists. Not all entries of json schema have a format string if format: json_schema['properties'][key]['format'] = format - + # Same idea as above but for the properties of complex json objects like arrays and nested json blobs if properties: json_schema['properties'][key]['properties'] = properties return json_schema - + def convert_to_schemas(self) -> Mapping[str, dict]: @@ -104,7 +105,7 @@ def convert_to_schemas(self) -> Mapping[str, dict]: # This check should return None if it passes assert Draft7Validator.check_schema(valid_schema) is None, "Schema {} is not valid".format(schema_key) - + return schemas class SchemaUpdater(object): @@ -119,6 +120,8 @@ def __init__(self, new_schemas: Mapping[str, dict], allowed_changes: List[str] = self.existing_schemas: Mapping[str, dict] = self._read_schemas() self.allowed_changes: List[str] = allowed_changes self.updated_schemas: Mapping[str, dict] = { k:{} for k in self.new_schemas.keys() } + self.list_of_modified_schemas: List[str] = list() + self.list_of_new_schemas: List[str] = list() def _read_schemas(self) -> Mapping[str, dict]: """ @@ -130,17 +133,35 @@ def _read_schemas(self) -> Mapping[str, dict]: with open(schema_file) as f: dict_key = schema_file.split('/')[-1].split('.')[0] schemas[dict_key] = json.load(f) - + return schemas - + + def _track_schema_changes(self, json_path: str, raw_path: str) -> None: + """ + Tracking whether or not schemas are being updated or newly created + based on the output dictdiffer.diff(, ) + """ + if isinstance(json_path, list): + schema = json_path[0] + #@TODO - clunky to construct this then split it + else: + schema = json_path.split('.')[0] + + if schema not in self.list_of_modified_schemas: + #@TODO - a bit clunky to check for this twice + if not raw_path: + self.list_of_new_schemas.append(schema) + self.list_of_modified_schemas.append(schema) + + def _process_changes(self, diff: Generator[Tuple, None, None]) -> pd.DataFrame: """ Process the incoming changeset into a more readable dataframe with the follow structure - | type | path | change | - |-------|-----|--------| - | add | actor.name | John Doe | + | type | path | change | old_value | new_value | + |------|------|--------|-----------|-----------| + | add | actor.name | | None | John Doe | where type - the type of change e.g. add, change, remove @@ -154,34 +175,36 @@ def _process_changes(self, diff: Generator[Tuple, None, None]) -> pd.DataFrame: for t, p, c in diff: if t in ['add', 'remove']: change_type = t - + for change in c: # If a completely new schema is being added, the path is going to be missing or root so we get the path from the first entry of the changeset - json_path = '.'.join([p, change[0]]) if p else change[0] + print(p, change) + json_path = '.'.join([p, str(change[0])]) if p else change[0] new_value = None if t == 'remove' else change[1] old_value = None if t == 'add' else change[1] + self._track_schema_changes(json_path, p) elif t == 'change': change_type = t json_path = p - - + old_value = c[0] + new_value = c[1] else: raise ValueError("Unknown change type {}".format(t)) - + init_dict['type'].append(change_type) init_dict['path'].append(json_path) init_dict['old_value'].append(old_value) init_dict['new_value'].append(new_value) - + diff_df = pd.DataFrame(init_dict) return diff_df - + def update_schemas(self) -> None: """ Update schemas and write to a new directory called schemas """ - + for key in self.existing_schemas.keys(): self.updated_schemas[key] = self.existing_schemas[key] @@ -202,7 +225,7 @@ def calculate_diff(self) -> List[str]: :returns: A list of keys that should be updated with the values in self.new_schemas """ - + raw_diff = dictdiffer.diff(self.existing_schemas, self.new_schemas) keys_to_update = [] @@ -213,16 +236,16 @@ def calculate_diff(self) -> List[str]: if not path: dict_path = change[0] keys_to_update.append(dict_path) - + return keys_to_update - + def change_report(self) -> None: """ Print a report of the changes being written into the schemas directory """ raw_diff = dictdiffer.diff(self.existing_schemas, self.new_schemas) diff = self._process_changes(raw_diff) - + print("INFO - Your allowed changes are {}".format(self.allowed_changes)) print(diff.loc[diff.type.isin(self.allowed_changes)].to_markdown(index=False)) @@ -233,21 +256,118 @@ class AstSchemaTemplate(object): """ AST template class required for safely modifying this repositories python files """ - def __init__(self): - pass + def __init__(self, new_streams: List[str]): + self.stream_template: str = STREAM_TEMPLATE + self.stream_path: PathLike = Path(__file__).parent.parent / Path('tap_hellobaton') / Path('streams.py') + self.tap_path: PathLike = Path(__file__).parent.parent / Path('tap_hellobaton') / Path('tap.py') + self.new_streams = new_streams + + def read_streams(self) -> ast.Module: + """ + Reads the streams.py file and returns the contents of the abstract syntax tree + """ + with open(self.stream_path) as f: + return ast.parse(f.read()) + + def read_tap(self) -> ast.Module: + """ + Reads the tap.py file and returns the contents + """ + with open(self.tap_path) as f: + return ast.parse(f.read()) + + def update_streams(self) -> None: + """ + Updates the tap stream objects in place + """ + stream_tree = self.read_streams() + + for new_stream in self.new_streams: + stream_object_name = self._snake_to_pascal(new_stream) + stream_name = new_stream + unparsed_class_def = self.stream_template.format( + stream_name=stream_name, + stream_object_name=stream_object_name + ) + #Dump the contents into the ast syntax tree + stream_tree.body.append(ast.parse(unparsed_class_def)) + + with open(self.stream_path, 'w') as out: + out.write(ast.unparse(stream_tree)) + + + def update_tap(self) -> None: + """ + Updates the tap object in place. It's really important that we initialize the proper + ast objects to avoid a compilation error. This is why we don't template this out + """ + tap_tree = self.read_tap() + + for new_stream in self.new_streams: + object_stream_name = self._snake_to_pascal(new_stream) + idx = self._find_tap_import_index(tap_tree) + #Edit the import statement + tap_tree.body[idx].names.append(ast.alias(name=object_stream_name)) + #Edit the assignment statement + tap_tree.body[idx+1].value.elts.append(ast.Name(object_stream_name,ast.Load)) + + with open(self.tap_path, 'w') as out: + out.write(ast.unparse(tap_tree)) + + def update(self) -> None: + """ + Updates the streams.py and tap.py files in place. + """ + self.update_streams() + self.update_tap() + + def _snake_to_pascal(self, s: str) -> str: + """ + Converts a snake case string to pascal case + + >>> _snake_to_pascal('hello_world') + 'HelloWorld' + """ + return s.replace("_"," ").title().replace(" ","") + + def _find_tap_import_index(self, tree: ast.Module) -> int: + """ + Helper method that parses a generic tap syntax tree and returns + the indexer for the import statement which looks something like + + from .streams import [ + Stream1, + Stream2, + ] + + This is the import we are trying to modify in place + """ + + #Our premise is that the generic import must be followed by an assignment + #There are more exhaustive ways to find these statments but this should work + #Most of the time + for i, obj in enumerate(tree.body): + if isinstance(obj, ast.ImportFrom): + if isinstance(tree.body[i+1], ast.Assign): + return i + @click.command() def main(): """ Update schemas in the schemas directory based on the current swagger spec and open api definition """ - + spec = OpenApiSpec() incoming_schemas = spec.convert_to_schemas() updater = SchemaUpdater(incoming_schemas) - print(f"Updating schemas... \n{updater.change_report()}") + print(f"Updating schemas with changes... \n{updater.change_report()}") updater.update_schemas() + code_generator = AstSchemaTemplate(new_streams = updater.list_of_new_schemas) + print(f"Generating new stream objects... \n{updater.list_of_new_schemas}") + code_generator.update() + if __name__ == "__main__": main() \ No newline at end of file diff --git a/scripts/templates.py b/scripts/templates.py new file mode 100644 index 0000000..a930a6c --- /dev/null +++ b/scripts/templates.py @@ -0,0 +1,12 @@ +""" +Templating out python objects so ast can be generated +""" + +STREAM_TEMPLATE = """ +class {stream_object_name}(hellobatonStream): + \"""Define custom stream.\""" + name = "{stream_name}" + path = "/{stream_name}/" + primary_keys = ["id"] + schema_filepath = SCHEMAS_DIR / "{stream_name}.json" +""" \ No newline at end of file diff --git a/tap_hellobaton/schemas/activity.json b/tap_hellobaton/schemas/activity.json index ffba94a..277b1f9 100644 --- a/tap_hellobaton/schemas/activity.json +++ b/tap_hellobaton/schemas/activity.json @@ -1,138 +1,111 @@ { - "properties": - { - "id": - { - "type": "integer" - }, - "_self": - { - "type": "string" - }, - "type": - { - "type": - [ - "string", - "null" - ] - }, - "group": - { - "type": "string" - }, - "parent": - { - "type": - [ - "string", - "null" - ] - }, - "child": - { - "type": - [ - "string", - "null" - ] - }, - "actor": - { - "type": - [ - "string", - "null" - ] - }, - "project": - { - "type": "string" - }, - "parent_type": - { - "type": - [ - "string", - "null" - ] - }, - "child_type": - { - "type": - [ - "string", - "null" - ] - }, - "meta": - { - "type": - [ - "object", - "null" - ], - "properties": - { - "new_status": - { - "type": - [ - "integer", - "null" - ] + "properties": { + "id": { + "type": "integer" + }, + "_self": { + "type": "string", + "format": "uri" + }, + "type": { + "type": [ + "string", + "null" + ] + }, + "group": { + "type": "string" + }, + "parent": { + "type": [ + "string", + "null" + ], + "format": "uri" }, - "previous_status": - { - "type": - [ - "integer", - "null" - ] + "child": { + "type": [ + "string", + "null" + ], + "format": "uri" }, - "new_assignee": - { - "type": - [ - "integer", - "null" - ] + "actor": { + "type": [ + "string", + "null" + ], + "format": "uri" }, - "previous_assignee": - { - "type": - [ - "integer", - "null" - ] + "project": { + "type": "string", + "format": "uri" }, - "new_due_date": - { - "type": - [ - "date-time", - "null" - ] + "parent_type": { + "type": [ + "string", + "null" + ] }, - "previous_due_date": - { - "type": - [ - "date-time", - "null" - ] + "child_type": { + "type": [ + "string", + "null" + ] + }, + "meta": { + "type": [ + "object", + "null" + ], + "properties": { + "new_status": { + "type": [ + "integer", + "null" + ] + }, + "previous_status": { + "type": [ + "integer", + "null" + ] + }, + "new_assignee": { + "type": [ + "integer", + "null" + ] + }, + "previous_assignee": { + "type": [ + "integer", + "null" + ] + }, + "new_due_date": { + "type": [ + "date-time", + "null" + ] + }, + "previous_due_date": { + "type": [ + "date-time", + "null" + ] + } + } + }, + "created": { + "type": "string", + "format": "date-time" + }, + "modified": { + "type": "string", + "format": "date-time" } - } - }, - "created": - { - "type": "string", - "format": "date-time" }, - "modified": - { - "type": "string", - "format": "date-time" - } - } -} + "required": [], + "type": "object" +} \ No newline at end of file diff --git a/tap_hellobaton/schemas/comments.json b/tap_hellobaton/schemas/comments.json index 6e68fa1..e23624b 100644 --- a/tap_hellobaton/schemas/comments.json +++ b/tap_hellobaton/schemas/comments.json @@ -4,16 +4,23 @@ "type": "integer" }, "_self": { - "type": "string" + "type": "string", + "format": "uri" }, "author": { - "type": ["string", "null"] + "type": [ + "string", + "null" + ], + "format": "uri" }, "task": { - "type": "string" + "type": "string", + "format": "uri" }, "project": { - "type": "string" + "type": "string", + "format": "uri" }, "content": { "type": "string" @@ -29,5 +36,7 @@ "type": "string", "format": "date-time" } - } + }, + "required": [], + "type": "object" } \ No newline at end of file diff --git a/tap_hellobaton/schemas/companies.json b/tap_hellobaton/schemas/companies.json index a7b2919..3b9bdee 100644 --- a/tap_hellobaton/schemas/companies.json +++ b/tap_hellobaton/schemas/companies.json @@ -1,30 +1,29 @@ { "properties": { - "id": { "type": "integer" }, - "_self": { - "type": "string" + "type": "string", + "format": "uri" }, - "name": { "type": "string" }, - "type": { "type": "string" }, - "created": { "type": "string", "format": "date-time" }, - "modified": { "type": "string", "format": "date-time" } - } + }, + "required": [ + "name" + ], + "type": "object" } \ No newline at end of file diff --git a/tap_hellobaton/schemas/custom_field_options.json b/tap_hellobaton/schemas/custom_field_options.json index 9cd2542..ecc28d7 100644 --- a/tap_hellobaton/schemas/custom_field_options.json +++ b/tap_hellobaton/schemas/custom_field_options.json @@ -4,10 +4,12 @@ "type": "integer" }, "_self": { - "type": "string" + "type": "string", + "format": "uri" }, "field": { - "type": "string" + "type": "string", + "format": "uri" }, "value": { "type": "string" @@ -19,7 +21,11 @@ "type": "boolean" }, "created_by": { - "type": ["string", "null"] + "type": [ + "string", + "null" + ], + "format": "uri" }, "created": { "type": "string", @@ -29,5 +35,9 @@ "type": "string", "format": "date-time" } - } + }, + "required": [ + "value" + ], + "type": "object" } \ No newline at end of file diff --git a/tap_hellobaton/schemas/custom_field_values.json b/tap_hellobaton/schemas/custom_field_values.json index 1a21f96..f431d2a 100644 --- a/tap_hellobaton/schemas/custom_field_values.json +++ b/tap_hellobaton/schemas/custom_field_values.json @@ -4,10 +4,12 @@ "type": "integer" }, "_self": { - "type": "string" + "type": "string", + "format": "uri" }, "project": { - "type": "string" + "type": "string", + "format": "uri" }, "field": { "type": "object", @@ -16,7 +18,8 @@ "type": "integer" }, "_self": { - "type": "string" + "type": "string", + "format": "uri" }, "title": { "type": "string" @@ -27,19 +30,29 @@ } }, "value_numeric": { - "type": ["number", "null"] + "type": [ + "number", + "null" + ] }, "value_text": { - "type": ["string", "null"] + "type": [ + "string", + "null" + ] }, "selected_option": { - "type": ["object", "null"], + "type": [ + "object", + "null" + ], "properties": { "id": { "type": "integer" }, "_self": { - "type": "string" + "type": "string", + "format": "uri" }, "value": { "type": "string" @@ -56,6 +69,18 @@ "modified": { "type": "string", "format": "date-time" + }, + "value_date": { + "type": [ + "string", + "null" + ], + "format": "date" } - } + }, + "required": [ + "field", + "selected_option" + ], + "type": "object" } \ No newline at end of file diff --git a/tap_hellobaton/schemas/custom_fields.json b/tap_hellobaton/schemas/custom_fields.json index ce33676..e84fff2 100644 --- a/tap_hellobaton/schemas/custom_fields.json +++ b/tap_hellobaton/schemas/custom_fields.json @@ -4,13 +4,17 @@ "type": "integer" }, "_self": { - "type": "string" + "type": "string", + "format": "uri" }, "title": { "type": "string" }, "description": { - "type": ["string", "null"] + "type": [ + "string", + "null" + ] }, "visibility": { "type": "string" @@ -25,16 +29,30 @@ "type": "boolean" }, "number_scale": { - "type": ["integer", "null"] + "type": [ + "integer", + "null" + ] }, "number_display": { - "type": ["string", "null"] + "type": [ + "string", + "null" + ] }, "options": { - "type": ["string", "null"] + "type": [ + "string", + "null" + ], + "format": "uri" }, "created_by": { - "type": ["string", "null"] + "type": [ + "string", + "null" + ], + "format": "uri" }, "created": { "type": "string", @@ -44,5 +62,9 @@ "type": "string", "format": "date-time" } - } + }, + "required": [ + "title" + ], + "type": "object" } \ No newline at end of file diff --git a/tap_hellobaton/schemas/external_tasks.json b/tap_hellobaton/schemas/external_tasks.json new file mode 100644 index 0000000..5f61469 --- /dev/null +++ b/tap_hellobaton/schemas/external_tasks.json @@ -0,0 +1,42 @@ +{ + "required": [ + "url" + ], + "type": "object", + "properties": { + "id": { + "type": "integer" + }, + "_self": { + "type": "string", + "format": "uri" + }, + "type": { + "type": "string" + }, + "task": { + "type": "string", + "format": "uri" + }, + "url": { + "type": "string", + "format": "uri" + }, + "created_by": { + "type": "string", + "format": "uri" + }, + "project": { + "type": "string", + "format": "uri" + }, + "created": { + "type": "string", + "format": "date-time" + }, + "modified": { + "type": "string", + "format": "date-time" + } + } +} \ No newline at end of file diff --git a/tap_hellobaton/schemas/milestone_feedback.json b/tap_hellobaton/schemas/milestone_feedback.json new file mode 100644 index 0000000..e6c5024 --- /dev/null +++ b/tap_hellobaton/schemas/milestone_feedback.json @@ -0,0 +1,44 @@ +{ + "required": [ + "rating" + ], + "type": "object", + "properties": { + "_self": { + "type": "string", + "format": "uri" + }, + "id": { + "type": "integer" + }, + "rating": { + "type": "integer" + }, + "comment": { + "type": [ + "string", + "null" + ] + }, + "project": { + "type": "string", + "format": "uri" + }, + "milestone": { + "type": "string", + "format": "uri" + }, + "created_by": { + "type": "string", + "format": "uri" + }, + "created": { + "type": "string", + "format": "date-time" + }, + "modified": { + "type": "string", + "format": "date-time" + } + } +} \ No newline at end of file diff --git a/tap_hellobaton/schemas/milestones.json b/tap_hellobaton/schemas/milestones.json index d6f368e..d6d2bbf 100644 --- a/tap_hellobaton/schemas/milestones.json +++ b/tap_hellobaton/schemas/milestones.json @@ -4,28 +4,38 @@ "type": "integer" }, "_self": { - "type": "string" + "type": "string", + "format": "uri" }, "title": { "type": "string" }, "description": { - "type": ["string", "null"] + "type": [ + "string", + "null" + ] }, "project": { - "type": "string" + "type": "string", + "format": "uri" }, "task_list": { - "type": "string" + "type": "string", + "format": "uri" }, "phase": { - "type": ["object", "null"], + "type": [ + "object", + "null" + ], "properties": { "id": { "type": "integer" }, "_self": { - "type": "string" + "type": "string", + "format": "uri" }, "name": { "type": "string" @@ -37,6 +47,9 @@ "modified": { "type": "string", "format": "date-time" + }, + "order": { + "type": "integer" } } }, @@ -44,7 +57,10 @@ "type": "boolean" }, "deadline_datetime": { - "type": ["string", "null"], + "type": [ + "string", + "null" + ], "format": "date-time" }, "risk_profiles": { @@ -68,14 +84,24 @@ } }, "start_datetime": { - "type": ["string", "null"] + "type": [ + "string", + "null" + ] }, "finish_datetime": { - "type": ["string", "null"], + "type": [ + "string", + "null" + ], "format": "date-time" }, "created_from": { - "type": ["string", "null"] + "type": [ + "string", + "null" + ], + "format": "uri" }, "duration": { "type": "integer" @@ -87,6 +113,15 @@ "modified": { "type": "string", "format": "date-time" + }, + "feedback_list": { + "type": "string", + "format": "uri" } - } -} + }, + "required": [ + "title", + "deadline_fixed" + ], + "type": "object" +} \ No newline at end of file diff --git a/tap_hellobaton/schemas/phases.json b/tap_hellobaton/schemas/phases.json index 08ba90e..928f0cb 100644 --- a/tap_hellobaton/schemas/phases.json +++ b/tap_hellobaton/schemas/phases.json @@ -4,10 +4,14 @@ "type": "integer" }, "_self": { - "type": "string" + "type": "string", + "format": "uri" }, "name": { - "type": ["string", "null"] + "type": [ + "string", + "null" + ] }, "order": { "type": "integer" @@ -20,5 +24,7 @@ "type": "string", "format": "date-time" } - } + }, + "required": [], + "type": "object" } \ No newline at end of file diff --git a/tap_hellobaton/schemas/project_attachments.json b/tap_hellobaton/schemas/project_attachments.json index 680c108..d6098f3 100644 --- a/tap_hellobaton/schemas/project_attachments.json +++ b/tap_hellobaton/schemas/project_attachments.json @@ -4,19 +4,26 @@ "type": "integer" }, "_self": { - "type": "string" + "type": "string", + "format": "uri" }, "project": { - "type": "string" + "type": "string", + "format": "uri" }, "url": { - "type": "string" + "type": "string", + "format": "uri" }, "label": { - "type": ["string", "null"] + "type": [ + "string", + "null" + ] }, "created_by": { - "type": "string" + "type": "string", + "format": "uri" }, "type": { "type": "string" @@ -25,7 +32,10 @@ "type": "boolean" }, "original_filename": { - "type": ["string", "null"] + "type": [ + "string", + "null" + ] }, "created": { "type": "string", @@ -35,5 +45,9 @@ "type": "string", "format": "date-time" } - } -} + }, + "required": [ + "url" + ], + "type": "object" +} \ No newline at end of file diff --git a/tap_hellobaton/schemas/project_phases.json b/tap_hellobaton/schemas/project_phases.json new file mode 100644 index 0000000..cc66255 --- /dev/null +++ b/tap_hellobaton/schemas/project_phases.json @@ -0,0 +1,41 @@ +{ + "required": [], + "type": "object", + "properties": { + "id": { + "type": "integer" + }, + "_self": { + "type": "string", + "format": "uri" + }, + "project": { + "type": "string", + "format": "uri" + }, + "phase": { + "type": "string", + "format": "uri" + }, + "start_datetime": { + "type": [ + "string", + "null" + ], + "format": "date-time" + }, + "end_datetime": { + "type": [ + "string", + "null" + ], + "format": "date-time" + }, + "template_offset_days": { + "type": "integer" + }, + "template_duration_days": { + "type": "integer" + } + } +} \ No newline at end of file diff --git a/tap_hellobaton/schemas/project_users.json b/tap_hellobaton/schemas/project_users.json index e9344c3..9b71fef 100644 --- a/tap_hellobaton/schemas/project_users.json +++ b/tap_hellobaton/schemas/project_users.json @@ -4,13 +4,16 @@ "type": "integer" }, "_self": { - "type": "string" + "type": "string", + "format": "uri" }, "project": { - "type": "string" + "type": "string", + "format": "uri" }, "user": { - "type": "string" + "type": "string", + "format": "uri" }, "role": { "type": "string" @@ -19,13 +22,21 @@ "type": "boolean" }, "joined_on": { - "type": ["string", "null"] + "type": [ + "string", + "null" + ], + "format": "date-time" }, "invited": { "type": "boolean" }, "invited_on": { - "type": ["string", "null"] + "type": [ + "string", + "null" + ], + "format": "date-time" }, "created": { "type": "string", @@ -35,5 +46,7 @@ "type": "string", "format": "date-time" } - } -} + }, + "required": [], + "type": "object" +} \ No newline at end of file diff --git a/tap_hellobaton/schemas/projects.json b/tap_hellobaton/schemas/projects.json index 7cfd83c..6611f4c 100644 --- a/tap_hellobaton/schemas/projects.json +++ b/tap_hellobaton/schemas/projects.json @@ -1,17 +1,25 @@ { "properties": { - "_self": { - "type": "string" + "type": "string", + "format": "uri" }, "annual_contract_value": { - "type": ["string", "null"] + "type": [ + "string", + "null" + ], + "format": "decimal" }, "attachment_list": { - "type": "string" + "type": "string", + "format": "uri" }, "client_systems": { - "type": ["string", "null"] + "type": [ + "string", + "null" + ] }, "companies": { "type": "array", @@ -20,56 +28,93 @@ } }, "completed_datetime": { - "type": ["string", "null"], + "type": [ + "string", + "null" + ], "format": "date-time" }, "cost": { - "type": ["number", "null"] + "type": [ + "number", + "null" + ], + "format": "decimal" }, "created": { "type": "string", "format": "date-time" }, "created_from": { - "type": ["string", "null"] + "type": [ + "string", + "null" + ], + "format": "uri" }, "created_from_template": { - "type": ["string", "null"] + "type": [ + "string", + "null" + ], + "format": "uri" }, "creator": { - "type": ["string", "null"] + "type": [ + "string", + "null" + ], + "format": "uri" }, "deadline_datetime": { - "type": ["string", "null"], + "type": [ + "string", + "null" + ], "format": "date-time" }, "estimated_duration": { - "type": ["integer", "null"] + "type": [ + "integer", + "null" + ] }, "id": { "type": "integer" }, "implementation_budget": { - "type": "string" + "type": "string", + "format": "decimal" }, "milestone_list": { - "type": "string" + "type": "string", + "format": "uri" }, "modified": { - "type": ["string", "null"], + "type": [ + "string", + "null" + ], "format": "date-time" }, "phase": { - "type": ["object", "null"], + "type": [ + "object", + "null" + ], "properties": { "id": { "type": "integer" }, "_self": { - "type": "string" + "type": "string", + "format": "uri" }, "name": { - "type": ["string", "null"] + "type": [ + "string", + "null" + ] }, "order": { "type": "integer" @@ -105,33 +150,65 @@ "type": "string" }, "projected_golive_datetime": { - "type": ["string", "null"], + "type": [ + "string", + "null" + ], "format": "date-time" } } } }, "start_datetime": { - "type": ["string", "null"], + "type": [ + "string", + "null" + ], "format": "date-time" }, "started_datetime": { - "type": ["string", "null"], + "type": [ + "string", + "null" + ], "format": "date-time" }, "status": { "type": "string" }, "task_list": { - "type": "string" + "type": "string", + "format": "uri" }, "time_entry_list": { - "type": "string" + "type": "string", + "format": "uri" }, "title": { "type": "string" + }, + "milestone_feedback_list": { + "type": "string" + }, + "custom_field_list": { + "type": "string", + "format": "uri" + }, + "project_user_list": { + "type": "string", + "format": "uri" + }, + "archived": { + "type": "boolean" + }, + "project_phases_list": { + "type": "string", + "format": "uri" } - - } - -} + }, + "required": [ + "title", + "risk_profiles" + ], + "type": "object" +} \ No newline at end of file diff --git a/tap_hellobaton/schemas/task_attachments.json b/tap_hellobaton/schemas/task_attachments.json index 8ed7bcd..6631bc8 100644 --- a/tap_hellobaton/schemas/task_attachments.json +++ b/tap_hellobaton/schemas/task_attachments.json @@ -4,19 +4,25 @@ "type": "integer" }, "_self": { - "type": "string" + "type": "string", + "format": "uri" }, "task": { - "type": "string" + "type": "string", + "format": "uri" }, "url": { - "type": "string" + "type": "string", + "format": "uri" }, "type": { "type": "string" }, "label": { - "type": ["string", "null"] + "type": [ + "string", + "null" + ] }, "deliverable": { "type": "boolean" @@ -25,22 +31,42 @@ "type": "boolean" }, "approved": { - "type": ["boolean", "null"] + "type": [ + "boolean", + "null" + ] }, "revision_task": { - "type": ["string", "null"] + "type": [ + "string", + "null" + ], + "format": "uri" }, "original_filename": { - "type": ["string", "null"] + "type": [ + "string", + "null" + ] }, "created_by": { - "type": ["string", "null"] + "type": [ + "string", + "null" + ], + "format": "uri" }, "created": { - "type": "string" + "type": "string", + "format": "date-time" }, "modified": { - "type": "string" + "type": "string", + "format": "date-time" } - } + }, + "required": [ + "url" + ], + "type": "object" } \ No newline at end of file diff --git a/tap_hellobaton/schemas/task_deliverables.json b/tap_hellobaton/schemas/task_deliverables.json new file mode 100644 index 0000000..89f8e67 --- /dev/null +++ b/tap_hellobaton/schemas/task_deliverables.json @@ -0,0 +1,24 @@ +{ + "required": [], + "type": "object", + "properties": { + "id": { + "type": "integer" + }, + "_self": { + "type": "string", + "format": "uri" + }, + "task": { + "type": "string", + "format": "uri" + }, + "created_by": { + "type": "string", + "format": "uri" + }, + "type": { + "type": "string" + } + } +} \ No newline at end of file diff --git a/tap_hellobaton/schemas/tasks.json b/tap_hellobaton/schemas/tasks.json index 6a89c63..82260a4 100644 --- a/tap_hellobaton/schemas/tasks.json +++ b/tap_hellobaton/schemas/tasks.json @@ -1,113 +1,162 @@ { - "properties": { - "id": { - "type": "integer" - }, - "_self": { - "type": "string" - }, - "title": { - "type": "string" - }, - "description": { - "type": ["string", "null"] - }, - "project": { - "type": "string" - }, - "status": { - "type": "string" - }, - "dependency": { - "type": ["string", "null"] - }, - "start_datetime": { - "type": ["string", "null"], - "format": "date-time" - }, - "due_datetime": { - "type": "string", - "format": "date-time" - }, - "started_datetime": { - "type": ["string", "null"], - "format": "date-time" - }, - "finished_datetime": { - "type": ["string", "null"], - "format": "date-time" - }, - "started_overridden_datetime": { - "type": ["string", "null"], - "format": "date-time" - }, - "finished_overridden_datetime": { - "type": ["string", "null"], - "format": "date-time" - }, - "estimated_duration": { - "type": ["integer", "null"] - }, - "milestone": { - "type": "string" - }, - "created_by": { - "type": ["string", "null"] - }, - "assigned_to": { - "type": ["string", "null"] - }, - "created_from": { - "type": ["string", "null"] - }, - "risk_profiles": { - "type": "array", - "items": { - "type": "object", - "properties": { - "id": { + "properties": { + "id": { "type": "integer" - }, - "risk_level": { + }, + "_self": { + "type": "string", + "format": "uri" + }, + "title": { "type": "string" - }, - "formula": { + }, + "description": { + "type": [ + "string", + "null" + ] + }, + "project": { + "type": "string", + "format": "uri" + }, + "status": { "type": "string" - }, - "over_run": { - "type": "integer" - }, - "task_variance": { - "type": "integer" - }, - "cool_down": { - "type": "integer" - }, - "reason": { - "type": "integer" - }, - "duration": { - "type": "integer" - }, - "estimated_duration": { - "type": "integer" - } + }, + "dependency": { + "type": [ + "string", + "null" + ], + "format": "uri" + }, + "start_datetime": { + "type": [ + "string", + "null" + ], + "format": "date-time" + }, + "due_datetime": { + "type": "string", + "format": "date-time" + }, + "started_datetime": { + "type": [ + "string", + "null" + ], + "format": "date-time" + }, + "finished_datetime": { + "type": [ + "string", + "null" + ], + "format": "date-time" + }, + "started_overridden_datetime": { + "type": [ + "string", + "null" + ], + "format": "date-time" + }, + "finished_overridden_datetime": { + "type": [ + "string", + "null" + ], + "format": "date-time" + }, + "estimated_duration": { + "type": [ + "integer", + "null" + ] + }, + "milestone": { + "type": "string", + "format": "uri" + }, + "created_by": { + "type": [ + "string", + "null" + ], + "format": "uri" + }, + "assigned_to": { + "type": [ + "string", + "null" + ], + "format": "uri" + }, + "created_from": { + "type": [ + "string", + "null" + ], + "format": "uri" + }, + "risk_profiles": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "integer" + }, + "risk_level": { + "type": "string" + }, + "formula": { + "type": "string" + }, + "over_run": { + "type": "integer" + }, + "task_variance": { + "type": "integer" + }, + "cool_down": { + "type": "integer" + }, + "reason": { + "type": "integer" + }, + "duration": { + "type": "integer" + }, + "estimated_duration": { + "type": "integer" + } + } + } + }, + "time_entry_list": { + "type": "string", + "format": "uri" + }, + "attachment_list": { + "type": "string", + "format": "uri" + }, + "created": { + "type": "string", + "format": "date-time" + }, + "modified": { + "type": "string", + "format": "date-time" + }, + "dependencies": { + "type": "array" } - } - }, - "time_entry_list": { - "type": "string" - }, - "attachment_list": { - "type": "string" - }, - "created": { - "type": "string", - "format": "date-time" }, - "modified": { - "type": "string", - "format": "date-time" - } - } + "required": [ + "title" + ], + "type": "object" } \ No newline at end of file diff --git a/tap_hellobaton/schemas/templates.json b/tap_hellobaton/schemas/templates.json index cc317d9..428c49a 100644 --- a/tap_hellobaton/schemas/templates.json +++ b/tap_hellobaton/schemas/templates.json @@ -4,7 +4,8 @@ "type": "integer" }, "_self": { - "type": "string" + "type": "string", + "format": "uri" }, "title": { "type": "string" @@ -16,19 +17,22 @@ "type": [ "string", "null" - ] + ], + "format": "decimal" }, "annual_contract_value": { "type": [ "string", "null" - ] + ], + "format": "decimal" }, "implementation_budget": { "type": [ "string", "null" - ] + ], + "format": "decimal" }, "estimated_duration": { "type": [ @@ -37,38 +41,65 @@ ] }, "created_from_template": { - "type": ["string", "null"] + "type": [ + "string", + "null" + ], + "format": "uri" }, "created_from": { - "type": ["string", "null"] + "type": [ + "string", + "null" + ], + "format": "uri" }, "start_datetime": { - "type": ["string", "null"], + "type": [ + "string", + "null" + ], "format": "date-time" }, "started_datetime": { - "type": ["string", "null"], + "type": [ + "string", + "null" + ], "format": "date-time" }, "deadline_datetime": { - "type": ["string", "null"], + "type": [ + "string", + "null" + ], "format": "date-time" }, "completed_datetime": { - "type": ["string", "null"], + "type": [ + "string", + "null" + ], "format": "date-time" }, "client_systems": { - "type": ["string", "null"] + "type": [ + "string", + "null" + ] }, "phase": { - "type": ["object", "null"], + "type": [ + "object", + "null" + ], "properties": { "id": { "type": "integer" }, "_self": { - "type": "string" + "type": "string", + "format": "uri" }, "order": { "type": "integer" @@ -80,14 +111,33 @@ "modified": { "type": "string", "format": "date-time" + }, + "name": { + "type": [ + "string", + "null" + ] } } }, "creator": { - "type": ["string", "null"] + "type": [ + "string", + "null" + ], + "format": "uri" }, "task_list": { - "type": "string" + "type": "string", + "format": "uri" + }, + "milestone_list": { + "type": "string", + "format": "uri" } - } + }, + "required": [ + "title" + ], + "type": "object" } \ No newline at end of file diff --git a/tap_hellobaton/schemas/time_entries.json b/tap_hellobaton/schemas/time_entries.json index fa0f6e6..9429518 100644 --- a/tap_hellobaton/schemas/time_entries.json +++ b/tap_hellobaton/schemas/time_entries.json @@ -4,37 +4,55 @@ "type": "integer" }, "_self": { - "type": "string" + "type": "string", + "format": "uri" }, "user": { - "type": "string" + "type": "string", + "format": "uri" }, "created_by": { - "type": "string" + "type": "string", + "format": "uri" }, "project": { - "type": "string" + "type": "string", + "format": "uri" }, "task": { - "type": ["string", "null"] + "type": [ + "string", + "null" + ], + "format": "uri" }, "rate": { - "type": ["object", "null"], + "type": [ + "object", + "null" + ], "properties": { "id": { "type": "integer" }, "hourly_rate": { - "type": "string" + "type": "string", + "format": "decimal" } } }, "started_at": { - "type": ["string", "null"], + "type": [ + "string", + "null" + ], "format": "date-time" }, "ended_at": { - "type": ["string", "null"], + "type": [ + "string", + "null" + ], "format": "date-time" }, "reference_date": { @@ -45,13 +63,23 @@ "type": "boolean" }, "calculated_duration": { - "type": ["integer", "null"] + "type": [ + "integer", + "null" + ] }, "inputted_duration": { - "type": ["integer", "null"] + "type": [ + "integer", + "null" + ] }, "notes": { "type": "string" } - } -} + }, + "required": [ + "billable" + ], + "type": "object" +} \ No newline at end of file diff --git a/tap_hellobaton/schemas/users.json b/tap_hellobaton/schemas/users.json index 7ddb75c..a5922de 100644 --- a/tap_hellobaton/schemas/users.json +++ b/tap_hellobaton/schemas/users.json @@ -4,7 +4,8 @@ "type": "integer" }, "_self": { - "type": "string" + "type": "string", + "format": "uri" }, "first_name": { "type": "string" @@ -13,7 +14,8 @@ "type": "string" }, "email": { - "type": "string" + "type": "string", + "format": "email" }, "account_type": { "type": "string" @@ -22,16 +24,28 @@ "type": "string" }, "company": { - "type": "string" + "type": "string", + "format": "uri" }, "avatar_url": { - "type": ["string", "null"] + "type": [ + "string", + "null" + ], + "format": "uri" }, "created_by": { - "type": ["string", "null"] + "type": [ + "string", + "null" + ], + "format": "uri" }, "signed_up_at": { - "type": ["string", "null"], + "type": [ + "string", + "null" + ], "format": "date-time" }, "created": { @@ -42,5 +56,7 @@ "type": "string", "format": "date-time" } - } + }, + "required": [], + "type": "object" } \ No newline at end of file diff --git a/tap_hellobaton/streams.py b/tap_hellobaton/streams.py index bfe7802..9fc4031 100644 --- a/tap_hellobaton/streams.py +++ b/tap_hellobaton/streams.py @@ -1,118 +1,144 @@ """Stream type classes for tap-hellobaton.""" - from pathlib import Path from tap_hellobaton.client import hellobatonStream - -SCHEMAS_DIR = Path(__file__).parent / Path("./schemas") +SCHEMAS_DIR = Path(__file__).parent / Path('./schemas') class ProjectsStream(hellobatonStream): """Define custom stream.""" - name = "projects" - path = "/projects/" - primary_keys = ["id"] - schema_filepath = SCHEMAS_DIR / "projects.json" + name = 'projects' + path = '/projects/' + primary_keys = ['id'] + schema_filepath = SCHEMAS_DIR / 'projects.json' class CompaniesStream(hellobatonStream): """Define custom stream.""" - name = "companies" - path = "/companies/" - primary_keys = ["id"] - schema_filepath = SCHEMAS_DIR / "companies.json" + name = 'companies' + path = '/companies/' + primary_keys = ['id'] + schema_filepath = SCHEMAS_DIR / 'companies.json' class MilestonesStream(hellobatonStream): """Define custom stream.""" - name = "milestones" - path = "/milestones/" - primary_keys = ["id"] - schema_filepath = SCHEMAS_DIR / "milestones.json" + name = 'milestones' + path = '/milestones/' + primary_keys = ['id'] + schema_filepath = SCHEMAS_DIR / 'milestones.json' class PhasesStream(hellobatonStream): """Define custom stream.""" - name = "phases" - path = "/phases/" - primary_keys = ["id"] - schema_filepath = SCHEMAS_DIR / "phases.json" + name = 'phases' + path = '/phases/' + primary_keys = ['id'] + schema_filepath = SCHEMAS_DIR / 'phases.json' class ProjectAttachementsStream(hellobatonStream): """Define custom stream.""" - name = "project_attachments" - path = "/project_attachments/" - primary_keys = ["id"] - schema_filepath = SCHEMAS_DIR / "project_attachments.json" + name = 'project_attachments' + path = '/project_attachments/' + primary_keys = ['id'] + schema_filepath = SCHEMAS_DIR / 'project_attachments.json' class ProjectUsersStream(hellobatonStream): """Define custom stream.""" - name = "project_users" - path = "/project_users/" - primary_keys = ["id"] - schema_filepath = SCHEMAS_DIR / "project_users.json" + name = 'project_users' + path = '/project_users/' + primary_keys = ['id'] + schema_filepath = SCHEMAS_DIR / 'project_users.json' class TasksStream(hellobatonStream): """Define custom stream.""" - name = "tasks" - path = "/tasks/" - primary_keys = ["id"] - schema_filepath = SCHEMAS_DIR / "tasks.json" + name = 'tasks' + path = '/tasks/' + primary_keys = ['id'] + schema_filepath = SCHEMAS_DIR / 'tasks.json' class TaskAttachmentsStream(hellobatonStream): """Define custom stream.""" - name = "task_attachments" - path = "/task_attachments/" - primary_keys = ["id"] - schema_filepath = SCHEMAS_DIR / "task_attachments.json" + name = 'task_attachments' + path = '/task_attachments/' + primary_keys = ['id'] + schema_filepath = SCHEMAS_DIR / 'task_attachments.json' class TemplatesStream(hellobatonStream): """Define custom stream.""" - name = "templates" - path = "/templates/" - primary_keys = ["id"] - schema_filepath = SCHEMAS_DIR / "templates.json" + name = 'templates' + path = '/templates/' + primary_keys = ['id'] + schema_filepath = SCHEMAS_DIR / 'templates.json' class TimeEntriesStream(hellobatonStream): """Define custom stream.""" - name = "time_entries" - path = "/time_entries/" - primary_keys = ["id"] - schema_filepath = SCHEMAS_DIR / "time_entries.json" + name = 'time_entries' + path = '/time_entries/' + primary_keys = ['id'] + schema_filepath = SCHEMAS_DIR / 'time_entries.json' class UsersStream(hellobatonStream): """Define custom stream.""" - name = "users" - path = "/users/" - primary_keys = ["id"] - schema_filepath = SCHEMAS_DIR / "users.json" + name = 'users' + path = '/users/' + primary_keys = ['id'] + schema_filepath = SCHEMAS_DIR / 'users.json' class ActivityStream(hellobatonStream): """Define custom stream.""" - name = "activity" - path = "/activity/" - primary_keys = ["id"] - schema_filepath = SCHEMAS_DIR / "activity.json" + name = 'activity' + path = '/activity/' + primary_keys = ['id'] + schema_filepath = SCHEMAS_DIR / 'activity.json' class CommentsStream(hellobatonStream): """Define custom stream.""" - name = "comments" - path = "/comments/" - primary_keys = ["id"] - schema_filepath = SCHEMAS_DIR / "comments.json" + name = 'comments' + path = '/comments/' + primary_keys = ['id'] + schema_filepath = SCHEMAS_DIR / 'comments.json' class CustomFieldsStream(hellobatonStream): """Define custom stream.""" - name = "custom_fields" - path = "/custom_fields/" - primary_keys = ["id"] - schema_filepath = SCHEMAS_DIR / "custom_fields.json" + name = 'custom_fields' + path = '/custom_fields/' + primary_keys = ['id'] + schema_filepath = SCHEMAS_DIR / 'custom_fields.json' class CustomFieldOptionsStream(hellobatonStream): """Define custom stream.""" - name = "custom_field_options" - path = "/custom_field_options/" - primary_keys = ["id"] - schema_filepath = SCHEMAS_DIR / "custom_field_options.json" + name = 'custom_field_options' + path = '/custom_field_options/' + primary_keys = ['id'] + schema_filepath = SCHEMAS_DIR / 'custom_field_options.json' class CustomFieldValuesStream(hellobatonStream): """Define custom stream.""" - name = "custom_field_values" - path = "/custom_field_values/" - primary_keys = ["id"] - schema_filepath = SCHEMAS_DIR / "custom_field_values.json" + name = 'custom_field_values' + path = '/custom_field_values/' + primary_keys = ['id'] + schema_filepath = SCHEMAS_DIR / 'custom_field_values.json' + +class ExternalTasks(hellobatonStream): + """Define custom stream.""" + name = 'external_tasks' + path = '/external_tasks/' + primary_keys = ['id'] + schema_filepath = SCHEMAS_DIR / 'external_tasks.json' + +class MilestoneFeedback(hellobatonStream): + """Define custom stream.""" + name = 'milestone_feedback' + path = '/milestone_feedback/' + primary_keys = ['id'] + schema_filepath = SCHEMAS_DIR / 'milestone_feedback.json' + +class TaskDeliverables(hellobatonStream): + """Define custom stream.""" + name = 'task_deliverables' + path = '/task_deliverables/' + primary_keys = ['id'] + schema_filepath = SCHEMAS_DIR / 'task_deliverables.json' + +class ProjectPhases(hellobatonStream): + """Define custom stream.""" + name = 'project_phases' + path = '/project_phases/' + primary_keys = ['id'] + schema_filepath = SCHEMAS_DIR / 'project_phases.json' \ No newline at end of file diff --git a/tap_hellobaton/tap.py b/tap_hellobaton/tap.py index c8b1feb..363c66c 100644 --- a/tap_hellobaton/tap.py +++ b/tap_hellobaton/tap.py @@ -1,10 +1,7 @@ """hellobaton tap class.""" - from typing import List - from singer_sdk import Tap, Stream -from singer_sdk import typing as th # JSON schema typing helpers - +from singer_sdk import typing as th from tap_hellobaton.streams import ( ProjectAttachementsStream, ProjectUsersStream, @@ -21,7 +18,11 @@ CommentsStream, CustomFieldsStream, CustomFieldValuesStream, - CustomFieldOptionsStream + CustomFieldOptionsStream, + ExternalTasks, + MilestoneFeedback, + TaskDeliverables, + ProjectPhases, ) STREAM_TYPES = [ @@ -40,32 +41,34 @@ CommentsStream, CustomFieldsStream, CustomFieldValuesStream, - CustomFieldOptionsStream + CustomFieldOptionsStream, + ExternalTasks, + MilestoneFeedback, + TaskDeliverables, + ProjectPhases, ] + class Taphellobaton(Tap): """hellobaton tap class.""" - name = "tap-hellobaton" + name = "tap-hellobaton" config_jsonschema = th.PropertiesList( th.Property( "api_key", th.StringType, required=True, - description="The token to authenticate against the API service" + description="The token to authenticate against the API service", ), th.Property( "company", th.StringType, required=True, - description="Company instance to add to the base url" + description="Company instance to add to the base url", ), th.Property( - "user_agent", - th.StringType, - required=False, - description="User agent string" - ) + "user_agent", th.StringType, required=False, description="User agent string" + ), ).to_dict() def discover_streams(self) -> List[Stream]: