From eb7382dc4ff79f8bc405d1efd267a1b7f08c15de Mon Sep 17 00:00:00 2001 From: "cristina.mariscal" Date: Tue, 25 Jun 2024 17:51:22 -0600 Subject: [PATCH 01/37] Accounts and Addresses streams migration to low code --- .../connectors/source-twilio/metadata.yaml | 2 +- .../connectors/source-twilio/poetry.lock | 330 +++++++++++++++- .../connectors/source-twilio/pyproject.toml | 4 +- .../source-twilio/source_twilio/manifest.yaml | 365 ++++++++++++++++++ .../source_twilio/schemas/accounts.json | 111 ------ .../source_twilio/schemas/addresses.json | 72 ---- .../source-twilio/source_twilio/source.py | 13 +- .../source-twilio/unit_tests/test_source.py | 8 +- docs/integrations/sources/twilio.md | 1 + 9 files changed, 689 insertions(+), 217 deletions(-) create mode 100644 airbyte-integrations/connectors/source-twilio/source_twilio/manifest.yaml delete mode 100644 airbyte-integrations/connectors/source-twilio/source_twilio/schemas/accounts.json delete mode 100644 airbyte-integrations/connectors/source-twilio/source_twilio/schemas/addresses.json diff --git a/airbyte-integrations/connectors/source-twilio/metadata.yaml b/airbyte-integrations/connectors/source-twilio/metadata.yaml index 0f381a99d167..d4cde745bbba 100644 --- a/airbyte-integrations/connectors/source-twilio/metadata.yaml +++ b/airbyte-integrations/connectors/source-twilio/metadata.yaml @@ -13,7 +13,7 @@ data: connectorSubtype: api connectorType: source definitionId: b9dc6155-672e-42ea-b10d-9f1f1fb95ab1 - dockerImageTag: 0.11.6 + dockerImageTag: 0.11.7 dockerRepository: airbyte/source-twilio documentationUrl: https://docs.airbyte.com/integrations/sources/twilio githubIssueLabel: source-twilio diff --git a/airbyte-integrations/connectors/source-twilio/poetry.lock b/airbyte-integrations/connectors/source-twilio/poetry.lock index 249a824931e9..e1b607036a0f 100644 --- a/airbyte-integrations/connectors/source-twilio/poetry.lock +++ b/airbyte-integrations/connectors/source-twilio/poetry.lock @@ -1,31 +1,35 @@ -# This file is automatically @generated by Poetry 1.6.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.8.2 and should not be changed by hand. [[package]] name = "airbyte-cdk" -version = "0.80.0" +version = "1.8.0" description = "A framework for writing Airbyte Connectors." optional = false python-versions = "<4.0,>=3.9" files = [ - {file = "airbyte_cdk-0.80.0-py3-none-any.whl", hash = "sha256:060e92323a73674fa4e9e2e4a1eb312b9b9d072c9bbe5fa28f54ef21cb4974f3"}, - {file = "airbyte_cdk-0.80.0.tar.gz", hash = "sha256:1383512a83917fecca5b24cea4c72aa5c561cf96dd464485fbcefda48fe574c5"}, + {file = "airbyte_cdk-1.8.0-py3-none-any.whl", hash = "sha256:ca23d7877005fe87ffc4a3a3de29ee55eed625d874eb59b49664b156f9ae9ee2"}, + {file = "airbyte_cdk-1.8.0.tar.gz", hash = "sha256:ac82fbfd6b650b7ed015900748e30fdd2a4c574caa54d1bcc03cb584a17f1533"}, ] [package.dependencies] -airbyte-protocol-models = "0.5.1" +airbyte-protocol-models = ">=0.9.0,<1.0" backoff = "*" cachetools = "*" +cryptography = ">=42.0.5,<43.0.0" Deprecated = ">=1.2,<1.3" -dpath = ">=2.0.1,<2.1.0" +dpath = ">=2.1.6,<3.0.0" genson = "1.2.2" isodate = ">=0.6.1,<0.7.0" Jinja2 = ">=3.1.2,<3.2.0" jsonref = ">=0.2,<0.3" jsonschema = ">=3.2.0,<3.3.0" +langchain_core = "0.1.42" pendulum = "<3.0.0" pydantic = ">=1.10.8,<2.0.0" +pyjwt = ">=2.8.0,<3.0.0" pyrate-limiter = ">=3.1.0,<3.2.0" python-dateutil = "*" +pytz = "2024.1" PyYAML = ">=6.0.1,<7.0.0" requests = "*" requests_cache = "*" @@ -34,17 +38,17 @@ wcmatch = "8.4" [package.extras] file-based = ["avro (>=1.11.2,<1.12.0)", "fastavro (>=1.8.0,<1.9.0)", "markdown", "pdf2image (==1.16.3)", "pdfminer.six (==20221105)", "pyarrow (>=15.0.0,<15.1.0)", "pytesseract (==0.3.10)", "unstructured.pytesseract (>=0.3.12)", "unstructured[docx,pptx] (==0.10.27)"] sphinx-docs = ["Sphinx (>=4.2,<4.3)", "sphinx-rtd-theme (>=1.0,<1.1)"] -vector-db-based = ["cohere (==4.21)", "langchain (==0.0.271)", "openai[embeddings] (==0.27.9)", "tiktoken (==0.4.0)"] +vector-db-based = ["cohere (==4.21)", "langchain (==0.1.16)", "openai[embeddings] (==0.27.9)", "tiktoken (==0.4.0)"] [[package]] name = "airbyte-protocol-models" -version = "0.5.1" +version = "0.12.2" description = "Declares the Airbyte Protocol." optional = false python-versions = ">=3.8" files = [ - {file = "airbyte_protocol_models-0.5.1-py3-none-any.whl", hash = "sha256:dfe84e130e51ce2ae81a06d5aa36f6c5ce3152b9e36e6f0195fad6c3dab0927e"}, - {file = "airbyte_protocol_models-0.5.1.tar.gz", hash = "sha256:7c8b16c7c1c7956b1996052e40585a3a93b1e44cb509c4e97c1ee4fe507ea086"}, + {file = "airbyte_protocol_models-0.12.2-py3-none-any.whl", hash = "sha256:1780db5b26285865b858d26502933def8e11919c9436ccf7b8b9cb0170b07c2a"}, + {file = "airbyte_protocol_models-0.12.2.tar.gz", hash = "sha256:b7c4d9a7c32c0691601c2b9416af090a858e126666e2c8c880d7a1798eb519f0"}, ] [package.dependencies] @@ -148,6 +152,70 @@ files = [ {file = "certifi-2024.6.2.tar.gz", hash = "sha256:3cd43f1c6fa7dedc5899d69d3ad0398fd018ad1a17fba83ddaf78aa46c747516"}, ] +[[package]] +name = "cffi" +version = "1.16.0" +description = "Foreign Function Interface for Python calling C code." +optional = false +python-versions = ">=3.8" +files = [ + {file = "cffi-1.16.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:6b3d6606d369fc1da4fd8c357d026317fbb9c9b75d36dc16e90e84c26854b088"}, + {file = "cffi-1.16.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:ac0f5edd2360eea2f1daa9e26a41db02dd4b0451b48f7c318e217ee092a213e9"}, + {file = "cffi-1.16.0-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7e61e3e4fa664a8588aa25c883eab612a188c725755afff6289454d6362b9673"}, + {file = "cffi-1.16.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a72e8961a86d19bdb45851d8f1f08b041ea37d2bd8d4fd19903bc3083d80c896"}, + {file = "cffi-1.16.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5b50bf3f55561dac5438f8e70bfcdfd74543fd60df5fa5f62d94e5867deca684"}, + {file = "cffi-1.16.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7651c50c8c5ef7bdb41108b7b8c5a83013bfaa8a935590c5d74627c047a583c7"}, + {file = "cffi-1.16.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e4108df7fe9b707191e55f33efbcb2d81928e10cea45527879a4749cbe472614"}, + {file = "cffi-1.16.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:32c68ef735dbe5857c810328cb2481e24722a59a2003018885514d4c09af9743"}, + {file = "cffi-1.16.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:673739cb539f8cdaa07d92d02efa93c9ccf87e345b9a0b556e3ecc666718468d"}, + {file = "cffi-1.16.0-cp310-cp310-win32.whl", hash = "sha256:9f90389693731ff1f659e55c7d1640e2ec43ff725cc61b04b2f9c6d8d017df6a"}, + {file = "cffi-1.16.0-cp310-cp310-win_amd64.whl", hash = "sha256:e6024675e67af929088fda399b2094574609396b1decb609c55fa58b028a32a1"}, + {file = "cffi-1.16.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:b84834d0cf97e7d27dd5b7f3aca7b6e9263c56308ab9dc8aae9784abb774d404"}, + {file = "cffi-1.16.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:1b8ebc27c014c59692bb2664c7d13ce7a6e9a629be20e54e7271fa696ff2b417"}, + {file = "cffi-1.16.0-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ee07e47c12890ef248766a6e55bd38ebfb2bb8edd4142d56db91b21ea68b7627"}, + {file = "cffi-1.16.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d8a9d3ebe49f084ad71f9269834ceccbf398253c9fac910c4fd7053ff1386936"}, + {file = "cffi-1.16.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e70f54f1796669ef691ca07d046cd81a29cb4deb1e5f942003f401c0c4a2695d"}, + {file = "cffi-1.16.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5bf44d66cdf9e893637896c7faa22298baebcd18d1ddb6d2626a6e39793a1d56"}, + {file = "cffi-1.16.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7b78010e7b97fef4bee1e896df8a4bbb6712b7f05b7ef630f9d1da00f6444d2e"}, + {file = "cffi-1.16.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:c6a164aa47843fb1b01e941d385aab7215563bb8816d80ff3a363a9f8448a8dc"}, + {file = "cffi-1.16.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e09f3ff613345df5e8c3667da1d918f9149bd623cd9070c983c013792a9a62eb"}, + {file = "cffi-1.16.0-cp311-cp311-win32.whl", hash = "sha256:2c56b361916f390cd758a57f2e16233eb4f64bcbeee88a4881ea90fca14dc6ab"}, + {file = "cffi-1.16.0-cp311-cp311-win_amd64.whl", hash = "sha256:db8e577c19c0fda0beb7e0d4e09e0ba74b1e4c092e0e40bfa12fe05b6f6d75ba"}, + {file = "cffi-1.16.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:fa3a0128b152627161ce47201262d3140edb5a5c3da88d73a1b790a959126956"}, + {file = "cffi-1.16.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:68e7c44931cc171c54ccb702482e9fc723192e88d25a0e133edd7aff8fcd1f6e"}, + {file = "cffi-1.16.0-cp312-cp312-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:abd808f9c129ba2beda4cfc53bde801e5bcf9d6e0f22f095e45327c038bfe68e"}, + {file = "cffi-1.16.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:88e2b3c14bdb32e440be531ade29d3c50a1a59cd4e51b1dd8b0865c54ea5d2e2"}, + {file = "cffi-1.16.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:fcc8eb6d5902bb1cf6dc4f187ee3ea80a1eba0a89aba40a5cb20a5087d961357"}, + {file = "cffi-1.16.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b7be2d771cdba2942e13215c4e340bfd76398e9227ad10402a8767ab1865d2e6"}, + {file = "cffi-1.16.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e715596e683d2ce000574bae5d07bd522c781a822866c20495e52520564f0969"}, + {file = "cffi-1.16.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:2d92b25dbf6cae33f65005baf472d2c245c050b1ce709cc4588cdcdd5495b520"}, + {file = "cffi-1.16.0-cp312-cp312-win32.whl", hash = "sha256:b2ca4e77f9f47c55c194982e10f058db063937845bb2b7a86c84a6cfe0aefa8b"}, + {file = "cffi-1.16.0-cp312-cp312-win_amd64.whl", hash = "sha256:68678abf380b42ce21a5f2abde8efee05c114c2fdb2e9eef2efdb0257fba1235"}, + {file = "cffi-1.16.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:0c9ef6ff37e974b73c25eecc13952c55bceed9112be2d9d938ded8e856138bcc"}, + {file = "cffi-1.16.0-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a09582f178759ee8128d9270cd1344154fd473bb77d94ce0aeb2a93ebf0feaf0"}, + {file = "cffi-1.16.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e760191dd42581e023a68b758769e2da259b5d52e3103c6060ddc02c9edb8d7b"}, + {file = "cffi-1.16.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:80876338e19c951fdfed6198e70bc88f1c9758b94578d5a7c4c91a87af3cf31c"}, + {file = "cffi-1.16.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a6a14b17d7e17fa0d207ac08642c8820f84f25ce17a442fd15e27ea18d67c59b"}, + {file = "cffi-1.16.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6602bc8dc6f3a9e02b6c22c4fc1e47aa50f8f8e6d3f78a5e16ac33ef5fefa324"}, + {file = "cffi-1.16.0-cp38-cp38-win32.whl", hash = "sha256:131fd094d1065b19540c3d72594260f118b231090295d8c34e19a7bbcf2e860a"}, + {file = "cffi-1.16.0-cp38-cp38-win_amd64.whl", hash = "sha256:31d13b0f99e0836b7ff893d37af07366ebc90b678b6664c955b54561fc36ef36"}, + {file = "cffi-1.16.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:582215a0e9adbe0e379761260553ba11c58943e4bbe9c36430c4ca6ac74b15ed"}, + {file = "cffi-1.16.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:b29ebffcf550f9da55bec9e02ad430c992a87e5f512cd63388abb76f1036d8d2"}, + {file = "cffi-1.16.0-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:dc9b18bf40cc75f66f40a7379f6a9513244fe33c0e8aa72e2d56b0196a7ef872"}, + {file = "cffi-1.16.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9cb4a35b3642fc5c005a6755a5d17c6c8b6bcb6981baf81cea8bfbc8903e8ba8"}, + {file = "cffi-1.16.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b86851a328eedc692acf81fb05444bdf1891747c25af7529e39ddafaf68a4f3f"}, + {file = "cffi-1.16.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c0f31130ebc2d37cdd8e44605fb5fa7ad59049298b3f745c74fa74c62fbfcfc4"}, + {file = "cffi-1.16.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8f8e709127c6c77446a8c0a8c8bf3c8ee706a06cd44b1e827c3e6a2ee6b8c098"}, + {file = "cffi-1.16.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:748dcd1e3d3d7cd5443ef03ce8685043294ad6bd7c02a38d1bd367cfd968e000"}, + {file = "cffi-1.16.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:8895613bcc094d4a1b2dbe179d88d7fb4a15cee43c052e8885783fac397d91fe"}, + {file = "cffi-1.16.0-cp39-cp39-win32.whl", hash = "sha256:ed86a35631f7bfbb28e108dd96773b9d5a6ce4811cf6ea468bb6a359b256b1e4"}, + {file = "cffi-1.16.0-cp39-cp39-win_amd64.whl", hash = "sha256:3686dffb02459559c74dd3d81748269ffb0eb027c39a6fc99502de37d501faa8"}, + {file = "cffi-1.16.0.tar.gz", hash = "sha256:bcb3ef43e58665bbda2fb198698fcae6776483e0c4a631aa5647806c25e02cc0"}, +] + +[package.dependencies] +pycparser = "*" + [[package]] name = "charset-normalizer" version = "3.3.2" @@ -258,6 +326,60 @@ files = [ {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, ] +[[package]] +name = "cryptography" +version = "42.0.8" +description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." +optional = false +python-versions = ">=3.7" +files = [ + {file = "cryptography-42.0.8-cp37-abi3-macosx_10_12_universal2.whl", hash = "sha256:81d8a521705787afe7a18d5bfb47ea9d9cc068206270aad0b96a725022e18d2e"}, + {file = "cryptography-42.0.8-cp37-abi3-macosx_10_12_x86_64.whl", hash = "sha256:961e61cefdcb06e0c6d7e3a1b22ebe8b996eb2bf50614e89384be54c48c6b63d"}, + {file = "cryptography-42.0.8-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e3ec3672626e1b9e55afd0df6d774ff0e953452886e06e0f1eb7eb0c832e8902"}, + {file = "cryptography-42.0.8-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e599b53fd95357d92304510fb7bda8523ed1f79ca98dce2f43c115950aa78801"}, + {file = "cryptography-42.0.8-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:5226d5d21ab681f432a9c1cf8b658c0cb02533eece706b155e5fbd8a0cdd3949"}, + {file = "cryptography-42.0.8-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:6b7c4f03ce01afd3b76cf69a5455caa9cfa3de8c8f493e0d3ab7d20611c8dae9"}, + {file = "cryptography-42.0.8-cp37-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:2346b911eb349ab547076f47f2e035fc8ff2c02380a7cbbf8d87114fa0f1c583"}, + {file = "cryptography-42.0.8-cp37-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:ad803773e9df0b92e0a817d22fd8a3675493f690b96130a5e24f1b8fabbea9c7"}, + {file = "cryptography-42.0.8-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:2f66d9cd9147ee495a8374a45ca445819f8929a3efcd2e3df6428e46c3cbb10b"}, + {file = "cryptography-42.0.8-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:d45b940883a03e19e944456a558b67a41160e367a719833c53de6911cabba2b7"}, + {file = "cryptography-42.0.8-cp37-abi3-win32.whl", hash = "sha256:a0c5b2b0585b6af82d7e385f55a8bc568abff8923af147ee3c07bd8b42cda8b2"}, + {file = "cryptography-42.0.8-cp37-abi3-win_amd64.whl", hash = "sha256:57080dee41209e556a9a4ce60d229244f7a66ef52750f813bfbe18959770cfba"}, + {file = "cryptography-42.0.8-cp39-abi3-macosx_10_12_universal2.whl", hash = "sha256:dea567d1b0e8bc5764b9443858b673b734100c2871dc93163f58c46a97a83d28"}, + {file = "cryptography-42.0.8-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c4783183f7cb757b73b2ae9aed6599b96338eb957233c58ca8f49a49cc32fd5e"}, + {file = "cryptography-42.0.8-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a0608251135d0e03111152e41f0cc2392d1e74e35703960d4190b2e0f4ca9c70"}, + {file = "cryptography-42.0.8-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:dc0fdf6787f37b1c6b08e6dfc892d9d068b5bdb671198c72072828b80bd5fe4c"}, + {file = "cryptography-42.0.8-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:9c0c1716c8447ee7dbf08d6db2e5c41c688544c61074b54fc4564196f55c25a7"}, + {file = "cryptography-42.0.8-cp39-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:fff12c88a672ab9c9c1cf7b0c80e3ad9e2ebd9d828d955c126be4fd3e5578c9e"}, + {file = "cryptography-42.0.8-cp39-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:cafb92b2bc622cd1aa6a1dce4b93307792633f4c5fe1f46c6b97cf67073ec961"}, + {file = "cryptography-42.0.8-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:31f721658a29331f895a5a54e7e82075554ccfb8b163a18719d342f5ffe5ecb1"}, + {file = "cryptography-42.0.8-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:b297f90c5723d04bcc8265fc2a0f86d4ea2e0f7ab4b6994459548d3a6b992a14"}, + {file = "cryptography-42.0.8-cp39-abi3-win32.whl", hash = "sha256:2f88d197e66c65be5e42cd72e5c18afbfae3f741742070e3019ac8f4ac57262c"}, + {file = "cryptography-42.0.8-cp39-abi3-win_amd64.whl", hash = "sha256:fa76fbb7596cc5839320000cdd5d0955313696d9511debab7ee7278fc8b5c84a"}, + {file = "cryptography-42.0.8-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:ba4f0a211697362e89ad822e667d8d340b4d8d55fae72cdd619389fb5912eefe"}, + {file = "cryptography-42.0.8-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:81884c4d096c272f00aeb1f11cf62ccd39763581645b0812e99a91505fa48e0c"}, + {file = "cryptography-42.0.8-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:c9bb2ae11bfbab395bdd072985abde58ea9860ed84e59dbc0463a5d0159f5b71"}, + {file = "cryptography-42.0.8-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:7016f837e15b0a1c119d27ecd89b3515f01f90a8615ed5e9427e30d9cdbfed3d"}, + {file = "cryptography-42.0.8-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:5a94eccb2a81a309806027e1670a358b99b8fe8bfe9f8d329f27d72c094dde8c"}, + {file = "cryptography-42.0.8-pp39-pypy39_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:dec9b018df185f08483f294cae6ccac29e7a6e0678996587363dc352dc65c842"}, + {file = "cryptography-42.0.8-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:343728aac38decfdeecf55ecab3264b015be68fc2816ca800db649607aeee648"}, + {file = "cryptography-42.0.8-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:013629ae70b40af70c9a7a5db40abe5d9054e6f4380e50ce769947b73bf3caad"}, + {file = "cryptography-42.0.8.tar.gz", hash = "sha256:8d09d05439ce7baa8e9e95b07ec5b6c886f548deb7e0f69ef25f64b3bce842f2"}, +] + +[package.dependencies] +cffi = {version = ">=1.12", markers = "platform_python_implementation != \"PyPy\""} + +[package.extras] +docs = ["sphinx (>=5.3.0)", "sphinx-rtd-theme (>=1.1.1)"] +docstest = ["pyenchant (>=1.6.11)", "readme-renderer", "sphinxcontrib-spelling (>=4.0.1)"] +nox = ["nox"] +pep8test = ["check-sdist", "click", "mypy", "ruff"] +sdist = ["build"] +ssh = ["bcrypt (>=3.1.5)"] +test = ["certifi", "pretend", "pytest (>=6.2.0)", "pytest-benchmark", "pytest-cov", "pytest-xdist"] +test-randomorder = ["pytest-randomly"] + [[package]] name = "deprecated" version = "1.2.14" @@ -277,13 +399,13 @@ dev = ["PyTest", "PyTest-Cov", "bump2version (<1)", "sphinx (<2)", "tox"] [[package]] name = "dpath" -version = "2.0.8" +version = "2.2.0" description = "Filesystem-like pathing and searching for dictionaries" optional = false python-versions = ">=3.7" files = [ - {file = "dpath-2.0.8-py3-none-any.whl", hash = "sha256:f92f595214dd93a00558d75d4b858beee519f4cffca87f02616ad6cd013f3436"}, - {file = "dpath-2.0.8.tar.gz", hash = "sha256:a3440157ebe80d0a3ad794f1b61c571bef125214800ffdb9afc9424e8250fe9b"}, + {file = "dpath-2.2.0-py3-none-any.whl", hash = "sha256:b330a375ded0a0d2ed404440f6c6a715deae5313af40bbb01c8a41d891900576"}, + {file = "dpath-2.2.0.tar.gz", hash = "sha256:34f7e630dc55ea3f219e555726f5da4b4b25f2200319c8e6902c394258dd6a3e"}, ] [[package]] @@ -377,6 +499,31 @@ MarkupSafe = ">=2.0" [package.extras] i18n = ["Babel (>=2.7)"] +[[package]] +name = "jsonpatch" +version = "1.33" +description = "Apply JSON-Patches (RFC 6902)" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*, !=3.6.*" +files = [ + {file = "jsonpatch-1.33-py2.py3-none-any.whl", hash = "sha256:0ae28c0cd062bbd8b8ecc26d7d164fbbea9652a1a3693f3b956c1eae5145dade"}, + {file = "jsonpatch-1.33.tar.gz", hash = "sha256:9fcd4009c41e6d12348b4a0ff2563ba56a2923a7dfee731d004e212e1ee5030c"}, +] + +[package.dependencies] +jsonpointer = ">=1.9" + +[[package]] +name = "jsonpointer" +version = "3.0.0" +description = "Identify specific nodes in a JSON document (RFC 6901)" +optional = false +python-versions = ">=3.7" +files = [ + {file = "jsonpointer-3.0.0-py2.py3-none-any.whl", hash = "sha256:13e088adc14fca8b6aa8177c044e12701e6ad4b28ff10e65f2267a90109c9942"}, + {file = "jsonpointer-3.0.0.tar.gz", hash = "sha256:2b2d729f2091522d61c3b31f82e11870f60b68f43fbc705cb76bf4b832af59ef"}, +] + [[package]] name = "jsonref" version = "0.2" @@ -409,6 +556,44 @@ six = ">=1.11.0" format = ["idna", "jsonpointer (>1.13)", "rfc3987", "strict-rfc3339", "webcolors"] format-nongpl = ["idna", "jsonpointer (>1.13)", "rfc3339-validator", "rfc3986-validator (>0.1.0)", "webcolors"] +[[package]] +name = "langchain-core" +version = "0.1.42" +description = "Building applications with LLMs through composability" +optional = false +python-versions = "<4.0,>=3.8.1" +files = [ + {file = "langchain_core-0.1.42-py3-none-any.whl", hash = "sha256:c5653ffa08a44f740295c157a24c0def4a753333f6a2c41f76bf431cd00be8b5"}, + {file = "langchain_core-0.1.42.tar.gz", hash = "sha256:40751bf60ea5d8e2b2efe65290db434717ee3834870c002e40e2811f09d814e6"}, +] + +[package.dependencies] +jsonpatch = ">=1.33,<2.0" +langsmith = ">=0.1.0,<0.2.0" +packaging = ">=23.2,<24.0" +pydantic = ">=1,<3" +PyYAML = ">=5.3" +tenacity = ">=8.1.0,<9.0.0" + +[package.extras] +extended-testing = ["jinja2 (>=3,<4)"] + +[[package]] +name = "langsmith" +version = "0.1.82" +description = "Client library to connect to the LangSmith LLM Tracing and Evaluation Platform." +optional = false +python-versions = "<4.0,>=3.8.1" +files = [ + {file = "langsmith-0.1.82-py3-none-any.whl", hash = "sha256:9b3653e7d316036b0c60bf0bc3e280662d660f485a4ebd8e5c9d84f9831ae79c"}, + {file = "langsmith-0.1.82.tar.gz", hash = "sha256:c02e2bbc488c10c13b52c69d271eb40bd38da078d37b6ae7ae04a18bd48140be"}, +] + +[package.dependencies] +orjson = ">=3.9.14,<4.0.0" +pydantic = {version = ">=1,<3", markers = "python_full_version < \"3.12.4\""} +requests = ">=2,<3" + [[package]] name = "markupsafe" version = "2.1.5" @@ -478,15 +663,70 @@ files = [ {file = "MarkupSafe-2.1.5.tar.gz", hash = "sha256:d283d37a890ba4c1ae73ffadf8046435c76e7bc2247bbb63c00bd1a709c6544b"}, ] +[[package]] +name = "orjson" +version = "3.10.5" +description = "Fast, correct Python JSON library supporting dataclasses, datetimes, and numpy" +optional = false +python-versions = ">=3.8" +files = [ + {file = "orjson-3.10.5-cp310-cp310-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:545d493c1f560d5ccfc134803ceb8955a14c3fcb47bbb4b2fee0232646d0b932"}, + {file = "orjson-3.10.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f4324929c2dd917598212bfd554757feca3e5e0fa60da08be11b4aa8b90013c1"}, + {file = "orjson-3.10.5-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8c13ca5e2ddded0ce6a927ea5a9f27cae77eee4c75547b4297252cb20c4d30e6"}, + {file = "orjson-3.10.5-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b6c8e30adfa52c025f042a87f450a6b9ea29649d828e0fec4858ed5e6caecf63"}, + {file = "orjson-3.10.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:338fd4f071b242f26e9ca802f443edc588fa4ab60bfa81f38beaedf42eda226c"}, + {file = "orjson-3.10.5-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:6970ed7a3126cfed873c5d21ece1cd5d6f83ca6c9afb71bbae21a0b034588d96"}, + {file = "orjson-3.10.5-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:235dadefb793ad12f7fa11e98a480db1f7c6469ff9e3da5e73c7809c700d746b"}, + {file = "orjson-3.10.5-cp310-none-win32.whl", hash = "sha256:be79e2393679eda6a590638abda16d167754393f5d0850dcbca2d0c3735cebe2"}, + {file = "orjson-3.10.5-cp310-none-win_amd64.whl", hash = "sha256:c4a65310ccb5c9910c47b078ba78e2787cb3878cdded1702ac3d0da71ddc5228"}, + {file = "orjson-3.10.5-cp311-cp311-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:cdf7365063e80899ae3a697def1277c17a7df7ccfc979990a403dfe77bb54d40"}, + {file = "orjson-3.10.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6b68742c469745d0e6ca5724506858f75e2f1e5b59a4315861f9e2b1df77775a"}, + {file = "orjson-3.10.5-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7d10cc1b594951522e35a3463da19e899abe6ca95f3c84c69e9e901e0bd93d38"}, + {file = "orjson-3.10.5-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:dcbe82b35d1ac43b0d84072408330fd3295c2896973112d495e7234f7e3da2e1"}, + {file = "orjson-3.10.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:10c0eb7e0c75e1e486c7563fe231b40fdd658a035ae125c6ba651ca3b07936f5"}, + {file = "orjson-3.10.5-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:53ed1c879b10de56f35daf06dbc4a0d9a5db98f6ee853c2dbd3ee9d13e6f302f"}, + {file = "orjson-3.10.5-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:099e81a5975237fda3100f918839af95f42f981447ba8f47adb7b6a3cdb078fa"}, + {file = "orjson-3.10.5-cp311-none-win32.whl", hash = "sha256:1146bf85ea37ac421594107195db8bc77104f74bc83e8ee21a2e58596bfb2f04"}, + {file = "orjson-3.10.5-cp311-none-win_amd64.whl", hash = "sha256:36a10f43c5f3a55c2f680efe07aa93ef4a342d2960dd2b1b7ea2dd764fe4a37c"}, + {file = "orjson-3.10.5-cp312-cp312-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:68f85ecae7af14a585a563ac741b0547a3f291de81cd1e20903e79f25170458f"}, + {file = "orjson-3.10.5-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:28afa96f496474ce60d3340fe8d9a263aa93ea01201cd2bad844c45cd21f5268"}, + {file = "orjson-3.10.5-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9cd684927af3e11b6e754df80b9ffafd9fb6adcaa9d3e8fdd5891be5a5cad51e"}, + {file = "orjson-3.10.5-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3d21b9983da032505f7050795e98b5d9eee0df903258951566ecc358f6696969"}, + {file = "orjson-3.10.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1ad1de7fef79736dde8c3554e75361ec351158a906d747bd901a52a5c9c8d24b"}, + {file = "orjson-3.10.5-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:2d97531cdfe9bdd76d492e69800afd97e5930cb0da6a825646667b2c6c6c0211"}, + {file = "orjson-3.10.5-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:d69858c32f09c3e1ce44b617b3ebba1aba030e777000ebdf72b0d8e365d0b2b3"}, + {file = "orjson-3.10.5-cp312-none-win32.whl", hash = "sha256:64c9cc089f127e5875901ac05e5c25aa13cfa5dbbbd9602bda51e5c611d6e3e2"}, + {file = "orjson-3.10.5-cp312-none-win_amd64.whl", hash = "sha256:b2efbd67feff8c1f7728937c0d7f6ca8c25ec81373dc8db4ef394c1d93d13dc5"}, + {file = "orjson-3.10.5-cp38-cp38-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:03b565c3b93f5d6e001db48b747d31ea3819b89abf041ee10ac6988886d18e01"}, + {file = "orjson-3.10.5-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:584c902ec19ab7928fd5add1783c909094cc53f31ac7acfada817b0847975f26"}, + {file = "orjson-3.10.5-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5a35455cc0b0b3a1eaf67224035f5388591ec72b9b6136d66b49a553ce9eb1e6"}, + {file = "orjson-3.10.5-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1670fe88b116c2745a3a30b0f099b699a02bb3482c2591514baf5433819e4f4d"}, + {file = "orjson-3.10.5-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:185c394ef45b18b9a7d8e8f333606e2e8194a50c6e3c664215aae8cf42c5385e"}, + {file = "orjson-3.10.5-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:ca0b3a94ac8d3886c9581b9f9de3ce858263865fdaa383fbc31c310b9eac07c9"}, + {file = "orjson-3.10.5-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:dfc91d4720d48e2a709e9c368d5125b4b5899dced34b5400c3837dadc7d6271b"}, + {file = "orjson-3.10.5-cp38-none-win32.whl", hash = "sha256:c05f16701ab2a4ca146d0bca950af254cb7c02f3c01fca8efbbad82d23b3d9d4"}, + {file = "orjson-3.10.5-cp38-none-win_amd64.whl", hash = "sha256:8a11d459338f96a9aa7f232ba95679fc0c7cedbd1b990d736467894210205c09"}, + {file = "orjson-3.10.5-cp39-cp39-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:85c89131d7b3218db1b24c4abecea92fd6c7f9fab87441cfc342d3acc725d807"}, + {file = "orjson-3.10.5-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fb66215277a230c456f9038d5e2d84778141643207f85336ef8d2a9da26bd7ca"}, + {file = "orjson-3.10.5-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:51bbcdea96cdefa4a9b4461e690c75ad4e33796530d182bdd5c38980202c134a"}, + {file = "orjson-3.10.5-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:dbead71dbe65f959b7bd8cf91e0e11d5338033eba34c114f69078d59827ee139"}, + {file = "orjson-3.10.5-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5df58d206e78c40da118a8c14fc189207fffdcb1f21b3b4c9c0c18e839b5a214"}, + {file = "orjson-3.10.5-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:c4057c3b511bb8aef605616bd3f1f002a697c7e4da6adf095ca5b84c0fd43595"}, + {file = "orjson-3.10.5-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:b39e006b00c57125ab974362e740c14a0c6a66ff695bff44615dcf4a70ce2b86"}, + {file = "orjson-3.10.5-cp39-none-win32.whl", hash = "sha256:eded5138cc565a9d618e111c6d5c2547bbdd951114eb822f7f6309e04db0fb47"}, + {file = "orjson-3.10.5-cp39-none-win_amd64.whl", hash = "sha256:cc28e90a7cae7fcba2493953cff61da5a52950e78dc2dacfe931a317ee3d8de7"}, + {file = "orjson-3.10.5.tar.gz", hash = "sha256:7a5baef8a4284405d96c90c7c62b755e9ef1ada84c2406c24a9ebec86b89f46d"}, +] + [[package]] name = "packaging" -version = "24.1" +version = "23.2" description = "Core utilities for Python packages" optional = false -python-versions = ">=3.8" +python-versions = ">=3.7" files = [ - {file = "packaging-24.1-py3-none-any.whl", hash = "sha256:5b8f2217dbdbd2f7f384c41c628544e6d52f2d0f53c6d0c3ea61aa5d1d7ff124"}, - {file = "packaging-24.1.tar.gz", hash = "sha256:026ed72c8ed3fcce5bf8950572258698927fd1dbda10a5e981cdf0ac37f4f002"}, + {file = "packaging-23.2-py3-none-any.whl", hash = "sha256:8c491190033a9af7e1d931d0b5dacc2ef47509b34dd0de67ed209b5203fc88c7"}, + {file = "packaging-23.2.tar.gz", hash = "sha256:048fb0e9405036518eaaf48a55953c750c11e1a1b68e0dd1a9d62ed0c092cfc5"}, ] [[package]] @@ -565,6 +805,17 @@ files = [ {file = "py-1.11.0.tar.gz", hash = "sha256:51c75c4126074b472f746a24399ad32f6053d1b34b68d2fa41e558e6f4a98719"}, ] +[[package]] +name = "pycparser" +version = "2.22" +description = "C parser in Python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pycparser-2.22-py3-none-any.whl", hash = "sha256:c3702b6d3dd8c7abc1afa565d7e63d53a1d0bd86cdc24edd75470f4de499cfcc"}, + {file = "pycparser-2.22.tar.gz", hash = "sha256:491c8be9c040f5390f5bf44a5b07752bd07f56edf992381b05c701439eec10f6"}, +] + [[package]] name = "pydantic" version = "1.10.17" @@ -624,6 +875,23 @@ typing-extensions = ">=4.2.0" dotenv = ["python-dotenv (>=0.10.4)"] email = ["email-validator (>=1.0.3)"] +[[package]] +name = "pyjwt" +version = "2.8.0" +description = "JSON Web Token implementation in Python" +optional = false +python-versions = ">=3.7" +files = [ + {file = "PyJWT-2.8.0-py3-none-any.whl", hash = "sha256:59127c392cc44c2da5bb3192169a91f429924e17aff6534d70fdc02ab3e04320"}, + {file = "PyJWT-2.8.0.tar.gz", hash = "sha256:57e28d156e3d5c10088e0c68abb90bfac3df82b40a71bd0daa20c65ccd5c23de"}, +] + +[package.extras] +crypto = ["cryptography (>=3.4.0)"] +dev = ["coverage[toml] (==5.0.4)", "cryptography (>=3.4.0)", "pre-commit", "pytest (>=6.0.0,<7.0.0)", "sphinx (>=4.5.0,<5.0.0)", "sphinx-rtd-theme", "zope.interface"] +docs = ["sphinx (>=4.5.0,<5.0.0)", "sphinx-rtd-theme", "zope.interface"] +tests = ["coverage[toml] (==5.0.4)", "pytest (>=6.0.0,<7.0.0)"] + [[package]] name = "pyrate-limiter" version = "3.1.1" @@ -735,6 +1003,17 @@ files = [ [package.dependencies] six = ">=1.5" +[[package]] +name = "pytz" +version = "2024.1" +description = "World timezone definitions, modern and historical" +optional = false +python-versions = "*" +files = [ + {file = "pytz-2024.1-py2.py3-none-any.whl", hash = "sha256:328171f4e3623139da4983451950b28e95ac706e13f3f2630a879749e7a8b319"}, + {file = "pytz-2024.1.tar.gz", hash = "sha256:2a29735ea9c18baf14b448846bde5a48030ed267578472d8955cd0e7443a9812"}, +] + [[package]] name = "pytzdata" version = "2020.1" @@ -900,6 +1179,21 @@ files = [ {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, ] +[[package]] +name = "tenacity" +version = "8.4.2" +description = "Retry code until it succeeds" +optional = false +python-versions = ">=3.8" +files = [ + {file = "tenacity-8.4.2-py3-none-any.whl", hash = "sha256:9e6f7cf7da729125c7437222f8a522279751cdfbe6b67bfe64f75d3a348661b2"}, + {file = "tenacity-8.4.2.tar.gz", hash = "sha256:cd80a53a79336edba8489e767f729e4f391c896956b57140b5d7511a64bbd3ef"}, +] + +[package.extras] +doc = ["reno", "sphinx"] +test = ["pytest", "tornado (>=4.5)", "typeguard"] + [[package]] name = "toml" version = "0.10.2" @@ -1049,4 +1343,4 @@ files = [ [metadata] lock-version = "2.0" python-versions = "^3.9,<3.12" -content-hash = "5fa13fee3738931e120c8085932e07441855e97a5a48bdc15700f9ba74532ff5" +content-hash = "8e983cc17e259e1923dd43d392d98ac081169e47e2d7c23d488798fbc8ae348e" diff --git a/airbyte-integrations/connectors/source-twilio/pyproject.toml b/airbyte-integrations/connectors/source-twilio/pyproject.toml index 8563df0bccc7..2356ce7e3fd0 100644 --- a/airbyte-integrations/connectors/source-twilio/pyproject.toml +++ b/airbyte-integrations/connectors/source-twilio/pyproject.toml @@ -3,7 +3,7 @@ requires = [ "poetry-core>=1.0.0",] build-backend = "poetry.core.masonry.api" [tool.poetry] -version = "0.11.6" +version = "0.11.7" name = "source-twilio" description = "Source implementation for Twilio." authors = [ "Airbyte ",] @@ -17,7 +17,7 @@ include = "source_twilio" [tool.poetry.dependencies] python = "^3.9,<3.12" -airbyte-cdk = "0.80.0" +airbyte-cdk = "^1" pendulum = "==2.1.2" requests = "==2.31.0" diff --git a/airbyte-integrations/connectors/source-twilio/source_twilio/manifest.yaml b/airbyte-integrations/connectors/source-twilio/source_twilio/manifest.yaml new file mode 100644 index 000000000000..ff2882e3bcd0 --- /dev/null +++ b/airbyte-integrations/connectors/source-twilio/source_twilio/manifest.yaml @@ -0,0 +1,365 @@ +version: 1.3.1 + +type: DeclarativeSource + +check: + type: CheckStream + stream_names: + - Accounts + +definitions: + base_requester: + type: HttpRequester + url_base: https:// + authenticator: + type: BasicHttpAuthenticator + password: '{{ config["auth_token"] }}' + username: '{{ config["account_sid"] }}' + streams: + Accounts: + type: DeclarativeStream + name: Accounts + primary_key: + - sid + retriever: + type: SimpleRetriever + requester: + $ref: "#/definitions/base_requester" + path: https://api.twilio.com/2010-04-01/Accounts.json + http_method: GET + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - accounts + schema_loader: + type: InlineSchemaLoader + schema: + $ref: "#/schemas/Accounts" + Addresses: + type: DeclarativeStream + name: Addresses + primary_key: + - sid + retriever: + type: SimpleRetriever + requester: + $ref: "#/definitions/base_requester" + path: >- + {{ config['base_api_url'] }}/Accounts/{{ + stream_partition.account_sid }}/Addresses.json + http_method: GET + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - addresses + paginator: + type: DefaultPaginator + page_token_option: + type: RequestOption + inject_into: request_parameter + field_name: Page + page_size_option: + type: RequestOption + field_name: PageSize + inject_into: request_parameter + pagination_strategy: + type: PageIncrement + page_size: 1000 + start_from_page: 0 + partition_router: + type: SubstreamPartitionRouter + parent_stream_configs: + - type: ParentStreamConfig + parent_key: sid + partition_field: account_sid + stream: + $ref: "#/definitions/streams/Accounts" + schema_loader: + type: InlineSchemaLoader + schema: + $ref: "#/schemas/Addresses" + +streams: + - $ref: "#/definitions/streams/Accounts" + - $ref: "#/definitions/streams/Addresses" + +spec: + type: Spec + connection_specification: + type: object + $schema: http://json-schema.org/draft-07/schema# + required: + - username + - base_api_url + - studio_base_api_url + - start_date + properties: + username: + type: string + order: 0 + title: Username + password: + type: string + order: 1 + title: Password + always_show: true + airbyte_secret: true + base_api_url: + type: string + order: 2 + title: Base API URL + default: api.twilio.com/2010-04-01/ + studio_base_api_url: + type: string + order: 3 + title: Studio Base API URL + default: studio.twilio.com/v1/ + description: Studio Base API URL + start_date: + type: string + title: Start date + format: date-time + pattern: ^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$ + order: 4 + additionalProperties: true + +metadata: + autoImportSchema: + Accounts: true + Addresses: true + +schemas: + Accounts: + type: object + $schema: http://json-schema.org/schema# + additionalProperties: true + properties: + auth_token: + description: The authentication token for the account + type: + - 'null' + - string + date_created: + description: The timestamp when the account was created + format: date-time + type: + - 'null' + - string + date_updated: + description: The timestamp when the account was last updated + format: date-time + type: + - 'null' + - string + friendly_name: + description: A user-defined friendly name for the account + type: + - 'null' + - string + owner_account_sid: + description: The SID of the owner account + type: + - 'null' + - string + sid: + description: The unique identifier for the account + type: + - 'null' + - string + status: + description: The current status of the account + type: + - 'null' + - string + subresource_uris: + additionalProperties: true + description: URIs for accessing various subresources related to the account + properties: + addresses: + type: + - 'null' + - string + applications: + type: + - 'null' + - string + authorized_connect_apps: + type: + - 'null' + - string + available_phone_numbers: + type: + - 'null' + - string + balance: + type: + - 'null' + - string + calls: + type: + - 'null' + - string + conferences: + type: + - 'null' + - string + connect_apps: + type: + - 'null' + - string + incoming_phone_numbers: + type: + - 'null' + - string + keys: + type: + - 'null' + - string + messages: + type: + - 'null' + - string + notifications: + type: + - 'null' + - string + outgoing_caller_ids: + type: + - 'null' + - string + queues: + type: + - 'null' + - string + recordings: + type: + - 'null' + - string + short_codes: + type: + - 'null' + - string + signing_keys: + type: + - 'null' + - string + sip: + type: + - 'null' + - string + transcriptions: + type: + - 'null' + - string + usage: + type: + - 'null' + - string + type: + - 'null' + - object + type: + description: The type of the account + type: + - 'null' + - string + uri: + description: The URI for accessing the account resource + type: + - 'null' + - string + required: + - sid + Addresses: + type: object + $schema: http://json-schema.org/schema# + additionalProperties: true + properties: + account_sid: + description: The unique identifier of the account associated with this address. + type: + - 'null' + - string + city: + description: The city of the address. + type: + - 'null' + - string + customer_name: + description: The name of the customer associated with this address. + type: + - 'null' + - string + date_created: + description: The date and time when the address was created. + format: date-time + type: + - 'null' + - string + date_updated: + description: The date and time when the address was last updated. + format: date-time + type: + - 'null' + - string + emergency_enabled: + description: Indicates whether emergency services are enabled for this address. + type: + - 'null' + - boolean + friendly_name: + description: A friendly name or label for the address. + type: + - 'null' + - string + iso_country: + description: The ISO 3166-1 alpha-2 country code of the address. + type: + - 'null' + - string + postal_code: + description: The postal code of the address. + type: + - 'null' + - string + region: + description: The region or state of the address. + type: + - 'null' + - string + sid: + description: The unique identifier of the address. + type: + - 'null' + - string + street: + description: The street address. + type: + - 'null' + - string + street_secondary: + description: Additional information about the street address, such as suite number. + type: + - 'null' + - string + uri: + description: The URI of the address resource. + type: + - 'null' + - string + validated: + description: Indicates whether the address has been validated. + type: + - 'null' + - boolean + verified: + description: Indicates whether the address has been verified. + type: + - 'null' + - boolean + required: + - sid diff --git a/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/accounts.json b/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/accounts.json deleted file mode 100644 index 86277a3d9857..000000000000 --- a/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/accounts.json +++ /dev/null @@ -1,111 +0,0 @@ -{ - "properties": { - "auth_token": { - "description": "The authentication token for the account", - "type": ["null", "string"] - }, - "date_created": { - "description": "The timestamp when the account was created", - "format": "date-time", - "type": ["null", "string"] - }, - "date_updated": { - "description": "The timestamp when the account was last updated", - "format": "date-time", - "type": ["null", "string"] - }, - "friendly_name": { - "description": "A user-defined friendly name for the account", - "type": ["null", "string"] - }, - "owner_account_sid": { - "description": "The SID of the owner account", - "type": ["null", "string"] - }, - "sid": { - "description": "The unique identifier for the account", - "type": ["null", "string"] - }, - "status": { - "description": "The current status of the account", - "type": ["null", "string"] - }, - "subresource_uris": { - "description": "URIs for accessing various subresources related to the account", - "type": ["null", "object"], - "additionalProperties": true, - "properties": { - "addresses": { - "type": ["null", "string"] - }, - "conferences": { - "type": ["null", "string"] - }, - "signing_keys": { - "type": ["null", "string"] - }, - "transcriptions": { - "type": ["null", "string"] - }, - "connect_apps": { - "type": ["null", "string"] - }, - "sip": { - "type": ["null", "string"] - }, - "authorized_connect_apps": { - "type": ["null", "string"] - }, - "usage": { - "type": ["null", "string"] - }, - "keys": { - "type": ["null", "string"] - }, - "applications": { - "type": ["null", "string"] - }, - "recordings": { - "type": ["null", "string"] - }, - "short_codes": { - "type": ["null", "string"] - }, - "calls": { - "type": ["null", "string"] - }, - "notifications": { - "type": ["null", "string"] - }, - "incoming_phone_numbers": { - "type": ["null", "string"] - }, - "queues": { - "type": ["null", "string"] - }, - "messages": { - "type": ["null", "string"] - }, - "outgoing_caller_ids": { - "type": ["null", "string"] - }, - "available_phone_numbers": { - "type": ["null", "string"] - }, - "balance": { - "type": ["null", "string"] - } - } - }, - "type": { - "description": "The type of the account", - "type": ["null", "string"] - }, - "uri": { - "description": "The URI for accessing the account resource", - "type": ["null", "string"] - } - }, - "type": "object", - "additionalProperties": true -} diff --git a/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/addresses.json b/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/addresses.json deleted file mode 100644 index 488af95a4464..000000000000 --- a/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/addresses.json +++ /dev/null @@ -1,72 +0,0 @@ -{ - "properties": { - "account_sid": { - "description": "The unique identifier of the account associated with this address.", - "type": ["null", "string"] - }, - "city": { - "description": "The city of the address.", - "type": ["null", "string"] - }, - "customer_name": { - "description": "The name of the customer associated with this address.", - "type": ["null", "string"] - }, - "date_created": { - "description": "The date and time when the address was created.", - "format": "date-time", - "type": ["null", "string"] - }, - "date_updated": { - "description": "The date and time when the address was last updated.", - "format": "date-time", - "type": ["null", "string"] - }, - "emergency_enabled": { - "description": "Indicates whether emergency services are enabled for this address.", - "type": ["null", "boolean"] - }, - "friendly_name": { - "description": "A friendly name or label for the address.", - "type": ["null", "string"] - }, - "iso_country": { - "description": "The ISO 3166-1 alpha-2 country code of the address.", - "type": ["null", "string"] - }, - "postal_code": { - "description": "The postal code of the address.", - "type": ["null", "string"] - }, - "region": { - "description": "The region or state of the address.", - "type": ["null", "string"] - }, - "sid": { - "description": "The unique identifier of the address.", - "type": ["null", "string"] - }, - "street": { - "description": "The street address.", - "type": ["null", "string"] - }, - "street_secondary": { - "description": "Additional information about the street address, such as suite number.", - "type": ["null", "string"] - }, - "validated": { - "description": "Indicates whether the address has been validated.", - "type": ["null", "boolean"] - }, - "verified": { - "description": "Indicates whether the address has been verified.", - "type": ["null", "boolean"] - }, - "uri": { - "description": "The URI of the address resource.", - "type": ["null", "string"] - } - }, - "type": "object", - "additionalProperties": true -} diff --git a/airbyte-integrations/connectors/source-twilio/source_twilio/source.py b/airbyte-integrations/connectors/source-twilio/source_twilio/source.py index 26c2269b16af..7243e677a08b 100644 --- a/airbyte-integrations/connectors/source-twilio/source_twilio/source.py +++ b/airbyte-integrations/connectors/source-twilio/source_twilio/source.py @@ -8,12 +8,11 @@ import pendulum from airbyte_cdk.models import SyncMode -from airbyte_cdk.sources import AbstractSource +from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource from airbyte_cdk.sources.streams import Stream from source_twilio.auth import HttpBasicAuthenticator from source_twilio.streams import ( Accounts, - Addresses, Alerts, Applications, AvailablePhoneNumberCountries, @@ -51,7 +50,10 @@ RETENTION_WINDOW_LIMIT = 400 -class SourceTwilio(AbstractSource): +class SourceTwilio(YamlDeclarativeSource): + def __init__(self): + super().__init__(**{"path_to_yaml": "manifest.yaml"}) + def check_connection(self, logger: logging.Logger, config: Mapping[str, Any]) -> Tuple[bool, Any]: try: auth = HttpBasicAuthenticator( @@ -91,10 +93,7 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: incremental_stream_kwargs_message_stream["start_date"] = ( pendulum.now() - datetime.timedelta(days=RETENTION_WINDOW_LIMIT - 1) ).to_iso8601_string() - - streams = [ - Accounts(**full_refresh_stream_kwargs), - Addresses(**full_refresh_stream_kwargs), + streams = super().streams(config) + [ Alerts(**incremental_stream_kwargs), Applications(**full_refresh_stream_kwargs), AvailablePhoneNumberCountries(**full_refresh_stream_kwargs), diff --git a/airbyte-integrations/connectors/source-twilio/unit_tests/test_source.py b/airbyte-integrations/connectors/source-twilio/unit_tests/test_source.py index ad932927703f..566a534c2e1a 100644 --- a/airbyte-integrations/connectors/source-twilio/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-twilio/unit_tests/test_source.py @@ -8,8 +8,6 @@ import requests from source_twilio.source import SourceTwilio from source_twilio.streams import ( - Accounts, - Addresses, Alerts, Applications, AvailablePhoneNumberCountries, @@ -42,8 +40,8 @@ @pytest.fixture def config(): return { - "account_sid": "airbyte.io", - "auth_token": "secret", + "account_sid": "ACdade166c12e160e9ed0a6088226718fb", + "auth_token": "d36d25e4517633af8e17dd02dd6ba21b", "start_date": "2022-01-01T00:00:00Z", "lookback_window": 0, } @@ -80,8 +78,6 @@ def test_check_connection_handles_exceptions(mocker, config, exception, expected @pytest.mark.parametrize( "stream_cls", [ - (Accounts), - (Addresses), (Alerts), (Applications), (AvailablePhoneNumberCountries), diff --git a/docs/integrations/sources/twilio.md b/docs/integrations/sources/twilio.md index 9f9408795897..5267f6ffd0b0 100644 --- a/docs/integrations/sources/twilio.md +++ b/docs/integrations/sources/twilio.md @@ -100,6 +100,7 @@ For more information, see [the Twilio docs for rate limitations](https://support | Version | Date | Pull Request | Subject | | :------ | :--------- | :------------------------------------------------------- | :------------------------------------------------------------------------------------------------------ | +| 0.11.7 | 2024-06-24 | [40248](https://github.com/airbytehq/airbyte/pull/40248) | Twilio accounts stream migration to low code | | 0.11.6 | 2024-06-22 | [40030](https://github.com/airbytehq/airbyte/pull/40030) | Update dependencies | | 0.11.5 | 2024-06-06 | [39252](https://github.com/airbytehq/airbyte/pull/39252) | [autopull] Upgrade base image to v1.2.2 | | 0.11.4 | 2024-05-22 | [38559](https://github.com/airbytehq/airbyte/pull/38564) | Migrate authenticator to `requests_native_auth` package | From c6c54cbe2f7bba05fb27526d4ced500f372f898c Mon Sep 17 00:00:00 2001 From: "cristina.mariscal" Date: Wed, 26 Jun 2024 12:31:48 -0600 Subject: [PATCH 02/37] yaml fixes --- .../connectors/source-twilio/metadata.yaml | 2 +- .../connectors/source-twilio/source_twilio/manifest.yaml | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/airbyte-integrations/connectors/source-twilio/metadata.yaml b/airbyte-integrations/connectors/source-twilio/metadata.yaml index d4cde745bbba..a0f36ff54735 100644 --- a/airbyte-integrations/connectors/source-twilio/metadata.yaml +++ b/airbyte-integrations/connectors/source-twilio/metadata.yaml @@ -34,7 +34,7 @@ data: supportLevel: certified tags: - language:python - - cdk:python + - cdk:low-code connectorTestSuitesOptions: - suite: unitTests - suite: acceptanceTests diff --git a/airbyte-integrations/connectors/source-twilio/source_twilio/manifest.yaml b/airbyte-integrations/connectors/source-twilio/source_twilio/manifest.yaml index ff2882e3bcd0..55cbe4ce59e9 100644 --- a/airbyte-integrations/connectors/source-twilio/source_twilio/manifest.yaml +++ b/airbyte-integrations/connectors/source-twilio/source_twilio/manifest.yaml @@ -5,7 +5,7 @@ type: DeclarativeSource check: type: CheckStream stream_names: - - Accounts + - accounts definitions: base_requester: @@ -18,7 +18,7 @@ definitions: streams: Accounts: type: DeclarativeStream - name: Accounts + name: accounts primary_key: - sid retriever: @@ -39,7 +39,7 @@ definitions: $ref: "#/schemas/Accounts" Addresses: type: DeclarativeStream - name: Addresses + name: addresses primary_key: - sid retriever: @@ -93,7 +93,7 @@ spec: type: object $schema: http://json-schema.org/draft-07/schema# required: - - username + - account_sid - base_api_url - studio_base_api_url - start_date From ea9ce82193b3b0552537375231a5bf87e973e45a Mon Sep 17 00:00:00 2001 From: "cristina.mariscal" Date: Wed, 26 Jun 2024 12:43:29 -0600 Subject: [PATCH 03/37] format fixes --- .../source-twilio/source_twilio/manifest.yaml | 410 +++++++++--------- 1 file changed, 205 insertions(+), 205 deletions(-) diff --git a/airbyte-integrations/connectors/source-twilio/source_twilio/manifest.yaml b/airbyte-integrations/connectors/source-twilio/source_twilio/manifest.yaml index 55cbe4ce59e9..232fa516e37a 100644 --- a/airbyte-integrations/connectors/source-twilio/source_twilio/manifest.yaml +++ b/airbyte-integrations/connectors/source-twilio/source_twilio/manifest.yaml @@ -138,228 +138,228 @@ schemas: $schema: http://json-schema.org/schema# additionalProperties: true properties: - auth_token: - description: The authentication token for the account - type: - - 'null' - - string - date_created: - description: The timestamp when the account was created - format: date-time - type: - - 'null' - - string - date_updated: - description: The timestamp when the account was last updated - format: date-time - type: - - 'null' - - string - friendly_name: - description: A user-defined friendly name for the account - type: - - 'null' - - string - owner_account_sid: - description: The SID of the owner account - type: - - 'null' - - string - sid: - description: The unique identifier for the account - type: - - 'null' - - string - status: - description: The current status of the account - type: - - 'null' - - string - subresource_uris: - additionalProperties: true - description: URIs for accessing various subresources related to the account - properties: - addresses: + auth_token: + description: The authentication token for the account type: - - 'null' - - string - applications: + - "null" + - string + date_created: + description: The timestamp when the account was created + format: date-time type: - - 'null' - - string - authorized_connect_apps: + - "null" + - string + date_updated: + description: The timestamp when the account was last updated + format: date-time type: - - 'null' - - string - available_phone_numbers: + - "null" + - string + friendly_name: + description: A user-defined friendly name for the account type: - - 'null' - - string - balance: + - "null" + - string + owner_account_sid: + description: The SID of the owner account type: - - 'null' - - string - calls: + - "null" + - string + sid: + description: The unique identifier for the account type: - - 'null' - - string - conferences: + - "null" + - string + status: + description: The current status of the account type: - - 'null' - - string - connect_apps: + - "null" + - string + subresource_uris: + additionalProperties: true + description: URIs for accessing various subresources related to the account + properties: + addresses: + type: + - "null" + - string + applications: + type: + - "null" + - string + authorized_connect_apps: + type: + - "null" + - string + available_phone_numbers: + type: + - "null" + - string + balance: + type: + - "null" + - string + calls: + type: + - "null" + - string + conferences: + type: + - "null" + - string + connect_apps: + type: + - "null" + - string + incoming_phone_numbers: + type: + - "null" + - string + keys: + type: + - "null" + - string + messages: + type: + - "null" + - string + notifications: + type: + - "null" + - string + outgoing_caller_ids: + type: + - "null" + - string + queues: + type: + - "null" + - string + recordings: + type: + - "null" + - string + short_codes: + type: + - "null" + - string + signing_keys: + type: + - "null" + - string + sip: + type: + - "null" + - string + transcriptions: + type: + - "null" + - string + usage: + type: + - "null" + - string type: - - 'null' - - string - incoming_phone_numbers: + - "null" + - object + type: + description: The type of the account type: - - 'null' - - string - keys: + - "null" + - string + uri: + description: The URI for accessing the account resource type: - - 'null' - - string - messages: + - "null" + - string + required: + - sid + Addresses: + type: object + $schema: http://json-schema.org/schema# + additionalProperties: true + properties: + account_sid: + description: The unique identifier of the account associated with this address. type: - - 'null' - - string - notifications: + - "null" + - string + city: + description: The city of the address. type: - - 'null' - - string - outgoing_caller_ids: + - "null" + - string + customer_name: + description: The name of the customer associated with this address. type: - - 'null' - - string - queues: + - "null" + - string + date_created: + description: The date and time when the address was created. + format: date-time type: - - 'null' - - string - recordings: + - "null" + - string + date_updated: + description: The date and time when the address was last updated. + format: date-time type: - - 'null' - - string - short_codes: + - "null" + - string + emergency_enabled: + description: Indicates whether emergency services are enabled for this address. type: - - 'null' - - string - signing_keys: + - "null" + - boolean + friendly_name: + description: A friendly name or label for the address. type: - - 'null' - - string - sip: + - "null" + - string + iso_country: + description: The ISO 3166-1 alpha-2 country code of the address. type: - - 'null' - - string - transcriptions: + - "null" + - string + postal_code: + description: The postal code of the address. type: - - 'null' - - string - usage: + - "null" + - string + region: + description: The region or state of the address. type: - - 'null' - - string - type: - - 'null' - - object - type: - description: The type of the account - type: - - 'null' - - string - uri: - description: The URI for accessing the account resource - type: - - 'null' - - string - required: - - sid - Addresses: - type: object - $schema: http://json-schema.org/schema# - additionalProperties: true - properties: - account_sid: - description: The unique identifier of the account associated with this address. - type: - - 'null' - - string - city: - description: The city of the address. - type: - - 'null' - - string - customer_name: - description: The name of the customer associated with this address. - type: - - 'null' - - string - date_created: - description: The date and time when the address was created. - format: date-time - type: - - 'null' - - string - date_updated: - description: The date and time when the address was last updated. - format: date-time - type: - - 'null' - - string - emergency_enabled: - description: Indicates whether emergency services are enabled for this address. - type: - - 'null' - - boolean - friendly_name: - description: A friendly name or label for the address. - type: - - 'null' - - string - iso_country: - description: The ISO 3166-1 alpha-2 country code of the address. - type: - - 'null' - - string - postal_code: - description: The postal code of the address. - type: - - 'null' - - string - region: - description: The region or state of the address. - type: - - 'null' - - string - sid: - description: The unique identifier of the address. - type: - - 'null' - - string - street: - description: The street address. - type: - - 'null' - - string - street_secondary: - description: Additional information about the street address, such as suite number. - type: - - 'null' - - string - uri: - description: The URI of the address resource. - type: - - 'null' - - string - validated: - description: Indicates whether the address has been validated. - type: - - 'null' - - boolean - verified: - description: Indicates whether the address has been verified. - type: - - 'null' - - boolean - required: - - sid + - "null" + - string + sid: + description: The unique identifier of the address. + type: + - "null" + - string + street: + description: The street address. + type: + - "null" + - string + street_secondary: + description: Additional information about the street address, such as suite number. + type: + - "null" + - string + uri: + description: The URI of the address resource. + type: + - "null" + - string + validated: + description: Indicates whether the address has been validated. + type: + - "null" + - boolean + verified: + description: Indicates whether the address has been verified. + type: + - "null" + - boolean + required: + - sid From 531f18528d864fd1f947ff1d70941b76a78d9a64 Mon Sep 17 00:00:00 2001 From: "cristina.mariscal" Date: Wed, 26 Jun 2024 15:18:05 -0600 Subject: [PATCH 04/37] remove base_api_url and studio_base_api_url from required config fields --- .../connectors/source-twilio/source_twilio/manifest.yaml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/airbyte-integrations/connectors/source-twilio/source_twilio/manifest.yaml b/airbyte-integrations/connectors/source-twilio/source_twilio/manifest.yaml index 232fa516e37a..7a250c929799 100644 --- a/airbyte-integrations/connectors/source-twilio/source_twilio/manifest.yaml +++ b/airbyte-integrations/connectors/source-twilio/source_twilio/manifest.yaml @@ -91,11 +91,10 @@ spec: type: Spec connection_specification: type: object - $schema: http://json-schema.org/draft-07/schema# + $schema: http://json-schema.org/schema# required: - account_sid - - base_api_url - - studio_base_api_url + - auth_token - start_date properties: username: From 560e9a46727d1571c467d76432a99da5d29226df Mon Sep 17 00:00:00 2001 From: "cristina.mariscal" Date: Wed, 26 Jun 2024 18:02:14 -0600 Subject: [PATCH 05/37] adding fields to manifest spec --- .../source-twilio/source_twilio/manifest.yaml | 54 +++++++++++-------- 1 file changed, 32 insertions(+), 22 deletions(-) diff --git a/airbyte-integrations/connectors/source-twilio/source_twilio/manifest.yaml b/airbyte-integrations/connectors/source-twilio/source_twilio/manifest.yaml index 7a250c929799..e4bb2986528b 100644 --- a/airbyte-integrations/connectors/source-twilio/source_twilio/manifest.yaml +++ b/airbyte-integrations/connectors/source-twilio/source_twilio/manifest.yaml @@ -91,40 +91,50 @@ spec: type: Spec connection_specification: type: object - $schema: http://json-schema.org/schema# + $schema: http://json-schema.org/draft-07/schema# required: - account_sid - auth_token - start_date + title: Twilio Spec properties: - username: - type: string - order: 0 - title: Username - password: - type: string - order: 1 - title: Password - always_show: true + account_sid: airbyte_secret: true - base_api_url: + description: Twilio account SID + order: 1 + title: Account ID type: string + auth_token: + airbyte_secret: true + description: Twilio Auth Token. order: 2 - title: Base API URL - default: api.twilio.com/2010-04-01/ - studio_base_api_url: + title: Auth Token type: string - order: 3 - title: Studio Base API URL - default: studio.twilio.com/v1/ - description: Studio Base API URL + lookback_window: + default: 0 + description: How far into the past to look for records. (in minutes) + examples: + - 60 + maximum: 576000 + minimum: 0 + order: 4 + title: Lookback window + type: integer start_date: - type: string - title: Start date + description: UTC date and time in the format 2020-10-01T00:00:00Z. Any data + before this date will not be replicated. + examples: + - '2020-10-01T00:00:00Z' format: date-time + order: 3 pattern: ^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$ - order: 4 - additionalProperties: true + title: Replication Start Date + type: string + documentationUrl: https://docs.airbyte.com/integrations/sources/twilio + supported_destination_sync_modes: + - append + supportsIncremental: true + additionalProperties: true metadata: autoImportSchema: From 57797bff70bb0e6d4c375b429eaa297e912a0457 Mon Sep 17 00:00:00 2001 From: "cristina.mariscal" Date: Wed, 26 Jun 2024 19:27:12 -0600 Subject: [PATCH 06/37] format fixes --- .../source-twilio/source_twilio/manifest.yaml | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/airbyte-integrations/connectors/source-twilio/source_twilio/manifest.yaml b/airbyte-integrations/connectors/source-twilio/source_twilio/manifest.yaml index e4bb2986528b..031572993142 100644 --- a/airbyte-integrations/connectors/source-twilio/source_twilio/manifest.yaml +++ b/airbyte-integrations/connectors/source-twilio/source_twilio/manifest.yaml @@ -89,6 +89,11 @@ streams: spec: type: Spec + additionalProperties: true + documentationUrl: https://docs.airbyte.com/integrations/sources/twilio + supported_destination_sync_modes: + - append + supportsIncremental: true connection_specification: type: object $schema: http://json-schema.org/draft-07/schema# @@ -121,20 +126,16 @@ spec: title: Lookback window type: integer start_date: - description: UTC date and time in the format 2020-10-01T00:00:00Z. Any data + description: + UTC date and time in the format 2020-10-01T00:00:00Z. Any data before this date will not be replicated. examples: - - '2020-10-01T00:00:00Z' + - "2020-10-01T00:00:00Z" format: date-time order: 3 pattern: ^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$ title: Replication Start Date type: string - documentationUrl: https://docs.airbyte.com/integrations/sources/twilio - supported_destination_sync_modes: - - append - supportsIncremental: true - additionalProperties: true metadata: autoImportSchema: From f85dbfb0bb5846488a64b56bc0faff48b7a9ab62 Mon Sep 17 00:00:00 2001 From: "cristina.mariscal" Date: Thu, 27 Jun 2024 05:47:05 -0600 Subject: [PATCH 07/37] more fixes in spec --- .../connectors/source-twilio/source_twilio/manifest.yaml | 2 +- .../connectors/source-twilio/source_twilio/spec.json | 5 +---- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/airbyte-integrations/connectors/source-twilio/source_twilio/manifest.yaml b/airbyte-integrations/connectors/source-twilio/source_twilio/manifest.yaml index 031572993142..03935d44cf7e 100644 --- a/airbyte-integrations/connectors/source-twilio/source_twilio/manifest.yaml +++ b/airbyte-integrations/connectors/source-twilio/source_twilio/manifest.yaml @@ -90,7 +90,7 @@ streams: spec: type: Spec additionalProperties: true - documentationUrl: https://docs.airbyte.com/integrations/sources/twilio + documentation_url: https://docs.airbyte.com/integrations/sources/twilio supported_destination_sync_modes: - append supportsIncremental: true diff --git a/airbyte-integrations/connectors/source-twilio/source_twilio/spec.json b/airbyte-integrations/connectors/source-twilio/source_twilio/spec.json index 91c40846fa2a..6a9f72818545 100644 --- a/airbyte-integrations/connectors/source-twilio/source_twilio/spec.json +++ b/airbyte-integrations/connectors/source-twilio/source_twilio/spec.json @@ -5,7 +5,6 @@ "title": "Twilio Spec", "type": "object", "required": ["account_sid", "auth_token", "start_date"], - "additionalProperties": true, "properties": { "account_sid": { "title": "Account ID", @@ -41,7 +40,5 @@ "order": 4 } } - }, - "supportsIncremental": true, - "supported_destination_sync_modes": ["append"] + } } From 6f57e45ccea6fb324f9625150dac45a86baa8910 Mon Sep 17 00:00:00 2001 From: "cristina.mariscal" Date: Thu, 27 Jun 2024 10:52:04 -0600 Subject: [PATCH 08/37] all streams --- .../source-twilio/source_twilio/manifest.yaml | 5109 ++++++++++++++++- .../source_twilio/schemas/alerts.json | 69 - .../source_twilio/schemas/applications.json | 92 - .../available_phone_number_countries.json | 41 - .../available_phone_numbers_local.json | 77 - .../available_phone_numbers_mobile.json | 77 - .../available_phone_numbers_toll_free.json | 77 - .../source_twilio/schemas/calls.json | 161 - .../schemas/conference_participants.json | 64 - .../source_twilio/schemas/conferences.json | 67 - .../schemas/conversation_messages.json | 126 - .../schemas/conversation_participants.json | 71 - .../source_twilio/schemas/conversations.json | 60 - .../schemas/dependent_phone_numbers.json | 127 - .../source_twilio/schemas/executions.json | 66 - .../source_twilio/schemas/flows.json | 53 - .../schemas/incoming_phone_numbers.json | 174 - .../source_twilio/schemas/keys.json | 24 - .../source_twilio/schemas/message_media.json | 36 - .../source_twilio/schemas/messages.json | 100 - .../schemas/outgoing_caller_ids.json | 36 - .../source_twilio/schemas/queues.json | 55 - .../source_twilio/schemas/recordings.json | 115 - .../source_twilio/schemas/roles.json | 49 - .../source_twilio/schemas/services.json | 149 - .../source_twilio/schemas/step.json | 66 - .../source_twilio/schemas/transcriptions.json | 60 - .../source_twilio/schemas/trunks.json | 112 - .../source_twilio/schemas/usage_records.json | 103 - .../source_twilio/schemas/usage_triggers.json | 73 - .../schemas/user_conversations.json | 96 - .../source_twilio/schemas/users.json | 66 - .../schemas/verify_services.json | 146 - .../source-twilio/source_twilio/source.py | 122 +- .../source-twilio/source_twilio/streams.py | 670 --- .../source-twilio/unit_tests/test_source.py | 113 - .../source-twilio/unit_tests/test_streams.py | 333 -- 37 files changed, 4959 insertions(+), 4076 deletions(-) delete mode 100644 airbyte-integrations/connectors/source-twilio/source_twilio/schemas/alerts.json delete mode 100644 airbyte-integrations/connectors/source-twilio/source_twilio/schemas/applications.json delete mode 100644 airbyte-integrations/connectors/source-twilio/source_twilio/schemas/available_phone_number_countries.json delete mode 100644 airbyte-integrations/connectors/source-twilio/source_twilio/schemas/available_phone_numbers_local.json delete mode 100644 airbyte-integrations/connectors/source-twilio/source_twilio/schemas/available_phone_numbers_mobile.json delete mode 100644 airbyte-integrations/connectors/source-twilio/source_twilio/schemas/available_phone_numbers_toll_free.json delete mode 100644 airbyte-integrations/connectors/source-twilio/source_twilio/schemas/calls.json delete mode 100644 airbyte-integrations/connectors/source-twilio/source_twilio/schemas/conference_participants.json delete mode 100644 airbyte-integrations/connectors/source-twilio/source_twilio/schemas/conferences.json delete mode 100644 airbyte-integrations/connectors/source-twilio/source_twilio/schemas/conversation_messages.json delete mode 100644 airbyte-integrations/connectors/source-twilio/source_twilio/schemas/conversation_participants.json delete mode 100644 airbyte-integrations/connectors/source-twilio/source_twilio/schemas/conversations.json delete mode 100644 airbyte-integrations/connectors/source-twilio/source_twilio/schemas/dependent_phone_numbers.json delete mode 100644 airbyte-integrations/connectors/source-twilio/source_twilio/schemas/executions.json delete mode 100644 airbyte-integrations/connectors/source-twilio/source_twilio/schemas/flows.json delete mode 100644 airbyte-integrations/connectors/source-twilio/source_twilio/schemas/incoming_phone_numbers.json delete mode 100644 airbyte-integrations/connectors/source-twilio/source_twilio/schemas/keys.json delete mode 100644 airbyte-integrations/connectors/source-twilio/source_twilio/schemas/message_media.json delete mode 100644 airbyte-integrations/connectors/source-twilio/source_twilio/schemas/messages.json delete mode 100644 airbyte-integrations/connectors/source-twilio/source_twilio/schemas/outgoing_caller_ids.json delete mode 100644 airbyte-integrations/connectors/source-twilio/source_twilio/schemas/queues.json delete mode 100644 airbyte-integrations/connectors/source-twilio/source_twilio/schemas/recordings.json delete mode 100644 airbyte-integrations/connectors/source-twilio/source_twilio/schemas/roles.json delete mode 100644 airbyte-integrations/connectors/source-twilio/source_twilio/schemas/services.json delete mode 100644 airbyte-integrations/connectors/source-twilio/source_twilio/schemas/step.json delete mode 100644 airbyte-integrations/connectors/source-twilio/source_twilio/schemas/transcriptions.json delete mode 100644 airbyte-integrations/connectors/source-twilio/source_twilio/schemas/trunks.json delete mode 100644 airbyte-integrations/connectors/source-twilio/source_twilio/schemas/usage_records.json delete mode 100644 airbyte-integrations/connectors/source-twilio/source_twilio/schemas/usage_triggers.json delete mode 100644 airbyte-integrations/connectors/source-twilio/source_twilio/schemas/user_conversations.json delete mode 100644 airbyte-integrations/connectors/source-twilio/source_twilio/schemas/users.json delete mode 100644 airbyte-integrations/connectors/source-twilio/source_twilio/schemas/verify_services.json delete mode 100644 airbyte-integrations/connectors/source-twilio/source_twilio/streams.py delete mode 100644 airbyte-integrations/connectors/source-twilio/unit_tests/test_source.py delete mode 100644 airbyte-integrations/connectors/source-twilio/unit_tests/test_streams.py diff --git a/airbyte-integrations/connectors/source-twilio/source_twilio/manifest.yaml b/airbyte-integrations/connectors/source-twilio/source_twilio/manifest.yaml index 03935d44cf7e..cbafd8ae2963 100644 --- a/airbyte-integrations/connectors/source-twilio/source_twilio/manifest.yaml +++ b/airbyte-integrations/connectors/source-twilio/source_twilio/manifest.yaml @@ -47,7 +47,7 @@ definitions: requester: $ref: "#/definitions/base_requester" path: >- - {{ config['base_api_url'] }}/Accounts/{{ + https://api.twilio.com/2010-04-01/Accounts/{{ stream_partition.account_sid }}/Addresses.json http_method: GET record_selector: @@ -82,10 +82,1431 @@ definitions: type: InlineSchemaLoader schema: $ref: "#/schemas/Addresses" + Available Phone Number Countries: + type: DeclarativeStream + name: available_phone_number_countries + retriever: + type: SimpleRetriever + requester: + $ref: "#/definitions/base_requester" + path: >- + https://api.twilio.com/2010-04-01/Accounts/{{stream_partition.account_sid}}/AvailablePhoneNumbers.json + http_method: GET + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - countries + paginator: + type: DefaultPaginator + page_token_option: + type: RequestOption + inject_into: request_parameter + field_name: Page + page_size_option: + type: RequestOption + field_name: PageSize + inject_into: request_parameter + pagination_strategy: + type: PageIncrement + page_size: 1000 + start_from_page: 0 + partition_router: + type: SubstreamPartitionRouter + parent_stream_configs: + - type: ParentStreamConfig + parent_key: sid + partition_field: account_sid + stream: + $ref: "#/definitions/streams/Accounts" + schema_loader: + type: InlineSchemaLoader + schema: + $ref: "#/schemas/Available Phone Number Countries" + Available Phone Numbers Local: + type: DeclarativeStream + name: available_phone_numbers_local + retriever: + type: SimpleRetriever + requester: + $ref: "#/definitions/base_requester" + path: >- + https://api.twilio.com/2010-04-01/Accounts/{{stream_partition.account_sid}}/AvailablePhoneNumbers/{{stream_partition.country_code}}/Local.json + http_method: GET + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - available_phone_numbers + paginator: + type: DefaultPaginator + page_token_option: + type: RequestOption + inject_into: request_parameter + field_name: Page + page_size_option: + type: RequestOption + field_name: PageSize + inject_into: request_parameter + pagination_strategy: + type: PageIncrement + page_size: 1000 + start_from_page: 0 + partition_router: + - type: SubstreamPartitionRouter + parent_stream_configs: + - type: ParentStreamConfig + parent_key: sid + partition_field: account_sid + stream: + $ref: "#/definitions/streams/Accounts" + - type: SubstreamPartitionRouter + parent_stream_configs: + - type: ParentStreamConfig + parent_key: country_code + partition_field: country_code + stream: + $ref: "#/definitions/streams/Available Phone Number Countries" + schema_loader: + type: InlineSchemaLoader + schema: + $ref: "#/schemas/Available Phone Numbers Local" + Available Phone Numbers Mobile: + type: DeclarativeStream + name: available_phone_numbers_mobile + retriever: + type: SimpleRetriever + requester: + $ref: "#/definitions/base_requester" + path: >- + https://api.twilio.com/2010-04-01/Accounts/{{stream_partition.account_sid}}/AvailablePhoneNumbers/{{stream_partition.country_code}}/Mobile.json + http_method: GET + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - available_phone_numbers + paginator: + type: DefaultPaginator + page_token_option: + type: RequestOption + inject_into: request_parameter + field_name: Page + page_size_option: + type: RequestOption + field_name: PageSize + inject_into: request_parameter + pagination_strategy: + type: PageIncrement + page_size: 1000 + start_from_page: 0 + partition_router: + - type: SubstreamPartitionRouter + parent_stream_configs: + - type: ParentStreamConfig + parent_key: sid + partition_field: account_sid + stream: + $ref: "#/definitions/streams/Accounts" + - type: SubstreamPartitionRouter + parent_stream_configs: + - type: ParentStreamConfig + parent_key: country_code + partition_field: country_code + stream: + $ref: "#/definitions/streams/Available Phone Number Countries" + schema_loader: + type: InlineSchemaLoader + schema: + $ref: "#/schemas/Available Phone Numbers Mobile" + Available Phone Numbers Toll-Free: + type: DeclarativeStream + name: available_phone_numbers_toll-Free + retriever: + type: SimpleRetriever + requester: + $ref: "#/definitions/base_requester" + path: >- + https://api.twilio.com/2010-04-01/Accounts/{{stream_partition.account_sid}}/AvailablePhoneNumbers/{{stream_partition.country_code}}/TollFree.json + http_method: GET + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - available_phone_numbers + paginator: + type: DefaultPaginator + page_token_option: + type: RequestOption + inject_into: request_parameter + field_name: Page + page_size_option: + type: RequestOption + field_name: PageSize + inject_into: request_parameter + pagination_strategy: + type: PageIncrement + page_size: 1000 + start_from_page: 0 + partition_router: + - type: SubstreamPartitionRouter + parent_stream_configs: + - type: ParentStreamConfig + parent_key: sid + partition_field: account_sid + stream: + $ref: "#/definitions/streams/Accounts" + - type: SubstreamPartitionRouter + parent_stream_configs: + - type: ParentStreamConfig + parent_key: country_code + partition_field: country_code + stream: + $ref: "#/definitions/streams/Available Phone Number Countries" + schema_loader: + type: InlineSchemaLoader + schema: + $ref: "#/schemas/Available Phone Numbers Toll-Free" + Incoming Phone Numbers: + type: DeclarativeStream + name: incoming_phone_pumbers + retriever: + type: SimpleRetriever + requester: + $ref: "#/definitions/base_requester" + path: >- + https://api.twilio.com/2010-04-01/Accounts/{{stream_partition.account_sid}}/IncomingPhoneNumbers.json + http_method: GET + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - incoming_phone_numbers + paginator: + type: DefaultPaginator + page_token_option: + type: RequestOption + inject_into: request_parameter + field_name: Page + page_size_option: + type: RequestOption + field_name: PageSize + inject_into: request_parameter + pagination_strategy: + type: PageIncrement + page_size: 1000 + start_from_page: 0 + partition_router: + type: SubstreamPartitionRouter + parent_stream_configs: + - type: ParentStreamConfig + parent_key: sid + partition_field: account_sid + stream: + $ref: "#/definitions/streams/Accounts" + schema_loader: + type: InlineSchemaLoader + schema: + $ref: "#/schemas/Incoming Phone Numbers" + Keys: + type: DeclarativeStream + name: keys + retriever: + type: SimpleRetriever + requester: + $ref: "#/definitions/base_requester" + path: >- + https://api.twilio.com/2010-04-01/Accounts/{{stream_partition.account_sid}}/Keys.json + http_method: GET + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - keys + paginator: + type: DefaultPaginator + page_token_option: + type: RequestOption + inject_into: request_parameter + field_name: Page + page_size_option: + type: RequestOption + field_name: PageSize + inject_into: request_parameter + pagination_strategy: + type: PageIncrement + page_size: 1000 + start_from_page: 0 + partition_router: + type: SubstreamPartitionRouter + parent_stream_configs: + - type: ParentStreamConfig + parent_key: sid + partition_field: account_sid + stream: + $ref: "#/definitions/streams/Accounts" + schema_loader: + type: InlineSchemaLoader + schema: + $ref: "#/schemas/Keys" + Flows: + type: DeclarativeStream + name: flows + primary_key: + - sid + retriever: + type: SimpleRetriever + requester: + $ref: "#/definitions/base_requester" + path: "https://studio.twilio.com/v1/Flows" + http_method: GET + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - flows + paginator: + type: DefaultPaginator + page_token_option: + type: RequestOption + inject_into: request_parameter + field_name: Page + page_size_option: + type: RequestOption + field_name: PageSize + inject_into: request_parameter + pagination_strategy: + type: PageIncrement + page_size: 1000 + start_from_page: 0 + schema_loader: + type: InlineSchemaLoader + schema: + $ref: "#/schemas/Flows" + Steps: + type: DeclarativeStream + name: steps + primary_key: + - sid + retriever: + type: SimpleRetriever + requester: + $ref: "#/definitions/base_requester" + path: >- + https://studio.twilio.com/v1/Flows/{{stream_partition.flow_sid}}/Executions/{{stream_partition.execution_sid}}/Steps + http_method: GET + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - steps + paginator: + type: DefaultPaginator + page_token_option: + type: RequestOption + inject_into: request_parameter + field_name: Page + page_size_option: + type: RequestOption + field_name: PageSize + inject_into: request_parameter + pagination_strategy: + type: PageIncrement + page_size: 1000 + start_from_page: 0 + partition_router: + - type: SubstreamPartitionRouter + parent_stream_configs: + - type: ParentStreamConfig + parent_key: sid + partition_field: flow_sid + stream: + $ref: "#/definitions/streams/Flows" + - type: SubstreamPartitionRouter + parent_stream_configs: + - type: ParentStreamConfig + parent_key: sid + partition_field: execution_sid + stream: + $ref: "#/definitions/streams/Executions" + schema_loader: + type: InlineSchemaLoader + schema: + $ref: "#/schemas/Steps" + Alerts: + type: DeclarativeStream + name: alerts + retriever: + type: SimpleRetriever + requester: + $ref: "#/definitions/base_requester" + path: https://monitor.twilio.com/v1/Alerts + http_method: GET + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - alerts + schema_loader: + type: InlineSchemaLoader + schema: + $ref: "#/schemas/Alerts" + Applications: + type: DeclarativeStream + name: applications + primary_key: + - sid + retriever: + type: SimpleRetriever + requester: + $ref: "#/definitions/base_requester" + path: >- + https://api.twilio.com/2010-04-01/Accounts/{{ + stream_partition.account_sid }}/Applications.json + http_method: GET + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - applications + paginator: + type: DefaultPaginator + page_token_option: + type: RequestOption + inject_into: request_parameter + field_name: Page + page_size_option: + type: RequestOption + field_name: PageSize + inject_into: request_parameter + pagination_strategy: + type: PageIncrement + page_size: 1000 + start_from_page: 0 + partition_router: + type: SubstreamPartitionRouter + parent_stream_configs: + - type: ParentStreamConfig + parent_key: sid + partition_field: account_sid + stream: + $ref: "#/definitions/streams/Accounts" + schema_loader: + type: InlineSchemaLoader + schema: + $ref: "#/schemas/Applications" + Dependent Phone Numbers: + type: DeclarativeStream + name: dependent_phone_numbers + primary_key: + - sid + retriever: + type: SimpleRetriever + requester: + $ref: "#/definitions/base_requester" + path: >- + https://api.twilio.com/2010-04-01/Accounts/{{ + stream_partition.account_sid }}/Addresses/{{ + stream_partition.address_sid }}/DependentPhoneNumbers.json + http_method: GET + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - dependent_phone_numbers + paginator: + type: DefaultPaginator + page_token_option: + type: RequestOption + inject_into: request_parameter + field_name: Page + page_size_option: + type: RequestOption + field_name: PageSize + inject_into: request_parameter + pagination_strategy: + type: PageIncrement + page_size: 1000 + start_from_page: 0 + partition_router: + - type: SubstreamPartitionRouter + parent_stream_configs: + - type: ParentStreamConfig + parent_key: sid + partition_field: account_sid + stream: + $ref: "#/definitions/streams/Accounts" + - type: SubstreamPartitionRouter + parent_stream_configs: + - type: ParentStreamConfig + parent_key: sid + partition_field: address_sid + stream: + $ref: "#/definitions/streams/Addresses" + schema_loader: + type: InlineSchemaLoader + schema: + $ref: "#/schemas/Dependent Phone Numbers" + Calls: + type: DeclarativeStream + name: calls + primary_key: + - sid + retriever: + type: SimpleRetriever + requester: + $ref: "#/definitions/base_requester" + path: >- + https://api.twilio.com/2010-04-01/Accounts/{{ + stream_partition.account_sid }}/Calls.json + http_method: GET + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - calls + paginator: + type: DefaultPaginator + page_token_option: + type: RequestOption + inject_into: request_parameter + field_name: Page + page_size_option: + type: RequestOption + field_name: PageSize + inject_into: request_parameter + pagination_strategy: + type: PageIncrement + page_size: 1000 + start_from_page: 0 + partition_router: + type: SubstreamPartitionRouter + parent_stream_configs: + - type: ParentStreamConfig + parent_key: sid + partition_field: account_sid + stream: + $ref: "#/definitions/streams/Accounts" + incremental_sync: + type: DatetimeBasedCursor + cursor_field: end_time + cursor_datetime_formats: + - "%Y-%m-%d" + datetime_format: "%Y-%m-%d" + start_datetime: + type: MinMaxDatetime + datetime: '{{ config["start_date"] }}' + datetime_format: "%Y-%m-%dT%H:%M:%SZ" + start_time_option: + type: RequestOption + inject_into: request_parameter + field_name: EndTime> + end_time_option: + type: RequestOption + inject_into: request_parameter + field_name: EndTime< + end_datetime: + type: MinMaxDatetime + datetime: "{{ now_utc().strftime('%Y-%m-%dT%H:%M:%SZ') }}" + datetime_format: "%Y-%m-%dT%H:%M:%SZ" + schema_loader: + type: InlineSchemaLoader + schema: + $ref: "#/schemas/Calls" + Conferences: + type: DeclarativeStream + name: conferences + primary_key: + - sid + retriever: + type: SimpleRetriever + requester: + $ref: "#/definitions/base_requester" + path: >- + https://api.twilio.com/2010-04-01/Accounts/{{ + stream_partition.account_sid }}/Conferences.json + http_method: GET + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - conferences + paginator: + type: DefaultPaginator + page_token_option: + type: RequestOption + inject_into: request_parameter + field_name: Page + page_size_option: + type: RequestOption + field_name: PageSize + inject_into: request_parameter + pagination_strategy: + type: PageIncrement + page_size: 1000 + start_from_page: 0 + partition_router: + type: SubstreamPartitionRouter + parent_stream_configs: + - type: ParentStreamConfig + parent_key: sid + partition_field: account_sid + stream: + $ref: "#/definitions/streams/Accounts" + incremental_sync: + type: DatetimeBasedCursor + cursor_field: date_created + cursor_datetime_formats: + - "%Y-%m-%d" + datetime_format: "%Y-%m-%d" + start_datetime: + type: MinMaxDatetime + datetime: '{{ config["start_date"] }}' + datetime_format: "%Y-%m-%dT%H:%M:%SZ" + start_time_option: + type: RequestOption + inject_into: request_parameter + field_name: DateCreated> + end_time_option: + type: RequestOption + inject_into: request_parameter + field_name: DateCreated< + end_datetime: + type: MinMaxDatetime + datetime: "{{ now_utc().strftime('%Y-%m-%dT%H:%M:%SZ') }}" + datetime_format: "%Y-%m-%dT%H:%M:%SZ" + schema_loader: + type: InlineSchemaLoader + schema: + $ref: "#/schemas/Conferences" + Conference Participants: + type: DeclarativeStream + name: conference_participants + primary_key: + - sid + retriever: + type: SimpleRetriever + requester: + $ref: "#/definitions/base_requester" + path: >- + https://api.twilio.com/2010-04-01/Accounts/{{ + stream_partition.account_sid }}/Conferences/{{ + stream_partition.conference_sid }}/Participants.json + http_method: GET + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - participants + paginator: + type: DefaultPaginator + page_token_option: + type: RequestOption + inject_into: request_parameter + field_name: Page + page_size_option: + type: RequestOption + field_name: PageSize + inject_into: request_parameter + pagination_strategy: + type: PageIncrement + page_size: 1000 + start_from_page: 0 + partition_router: + - type: SubstreamPartitionRouter + parent_stream_configs: + - type: ParentStreamConfig + parent_key: sid + partition_field: account_sid + stream: + $ref: "#/definitions/streams/Accounts" + - type: SubstreamPartitionRouter + parent_stream_configs: + - type: ParentStreamConfig + parent_key: sid + partition_field: conference_sid + stream: + $ref: "#/definitions/streams/Conferences" + schema_loader: + type: InlineSchemaLoader + schema: + $ref: "#/schemas/Conference Participants" + Executions: + type: DeclarativeStream + name: executions + primary_key: + - sid + retriever: + type: SimpleRetriever + requester: + $ref: "#/definitions/base_requester" + path: >- + https://studio.twilio.com/v1/Flows/{{ stream_partition.flow_sid + }}/Executions + http_method: GET + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - executions + paginator: + type: DefaultPaginator + page_token_option: + type: RequestOption + inject_into: request_parameter + field_name: Page + page_size_option: + type: RequestOption + field_name: PageSize + inject_into: request_parameter + pagination_strategy: + type: PageIncrement + page_size: 1000 + start_from_page: 0 + partition_router: + type: SubstreamPartitionRouter + parent_stream_configs: + - type: ParentStreamConfig + parent_key: sid + partition_field: flow_sid + stream: + $ref: "#/definitions/streams/Flows" + schema_loader: + type: InlineSchemaLoader + schema: + $ref: "#/schemas/Executions" + Outgoing Caller Ids: + type: DeclarativeStream + name: outgoing_caller_ids + primary_key: + - sid + retriever: + type: SimpleRetriever + requester: + $ref: "#/definitions/base_requester" + path: >- + https://api.twilio.com/2010-04-01/Accounts/{{ + stream_partition.account_sid }}/OutgoingCallerIds + http_method: GET + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: [] + paginator: + type: DefaultPaginator + page_token_option: + type: RequestOption + inject_into: request_parameter + field_name: Page + page_size_option: + type: RequestOption + field_name: PageSize + inject_into: request_parameter + pagination_strategy: + type: PageIncrement + page_size: 1000 + start_from_page: 0 + partition_router: + type: SubstreamPartitionRouter + parent_stream_configs: + - type: ParentStreamConfig + parent_key: sid + partition_field: account_sid + stream: + $ref: "#/definitions/streams/Accounts" + schema_loader: + type: InlineSchemaLoader + schema: + $ref: "#/schemas/Outgoing Caller Ids" + Recordings: + type: DeclarativeStream + name: recordings + primary_key: + - sid + retriever: + type: SimpleRetriever + requester: + $ref: "#/definitions/base_requester" + path: >- + https://api.twilio.com/2010-04-01/Accounts/{{ + stream_partition.account_sid }}/Recordings.json + http_method: GET + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - recordings + paginator: + type: DefaultPaginator + page_token_option: + type: RequestOption + inject_into: request_parameter + field_name: Page + page_size_option: + type: RequestOption + field_name: PageSize + inject_into: request_parameter + pagination_strategy: + type: PageIncrement + page_size: 1000 + start_from_page: 0 + partition_router: + type: SubstreamPartitionRouter + parent_stream_configs: + - type: ParentStreamConfig + parent_key: sid + partition_field: account_sid + stream: + $ref: "#/definitions/streams/Accounts" + incremental_sync: + type: DatetimeBasedCursor + cursor_field: date_created + cursor_datetime_formats: + - "%Y-%m-%d" + datetime_format: "%Y-%m-%d" + start_datetime: + type: MinMaxDatetime + datetime: '{{ config["start_date"] }}' + datetime_format: "%Y-%m-%dT%H:%M:%SZ" + start_time_option: + type: RequestOption + inject_into: request_parameter + field_name: DateCreated> + end_time_option: + type: RequestOption + inject_into: request_parameter + field_name: DateCreated< + end_datetime: + type: MinMaxDatetime + datetime: "{{ now_utc().strftime('%Y-%m-%dT%H:%M:%SZ') }}" + datetime_format: "%Y-%m-%dT%H:%M:%SZ" + schema_loader: + type: InlineSchemaLoader + schema: + $ref: "#/schemas/Recordings" + Services: + type: DeclarativeStream + name: services + retriever: + type: SimpleRetriever + requester: + $ref: "#/definitions/base_requester" + path: https://chat.twilio.com/v2/Services + http_method: GET + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - services + schema_loader: + type: InlineSchemaLoader + schema: + $ref: "#/schemas/Services" + Verify Services: + type: DeclarativeStream + name: verify_services + primary_key: + - sid + retriever: + type: SimpleRetriever + requester: + $ref: "#/definitions/base_requester" + path: https://verify.twilio.com/v2/Services + http_method: GET + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - services + schema_loader: + type: InlineSchemaLoader + schema: + $ref: "#/schemas/Verify Services" + Roles: + type: DeclarativeStream + name: roles + primary_key: + - sid + retriever: + type: SimpleRetriever + requester: + $ref: "#/definitions/base_requester" + path: >- + https://chat.twilio.com/v2/Services/{{ stream_partition.service_sid + }}/Roles + http_method: GET + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - roles + partition_router: + type: SubstreamPartitionRouter + parent_stream_configs: + - type: ParentStreamConfig + parent_key: sid + partition_field: service_sid + stream: + $ref: "#/definitions/streams/Services" + schema_loader: + type: InlineSchemaLoader + schema: + $ref: "#/schemas/Roles" + Transcriptions: + type: DeclarativeStream + name: transcriptions + primary_key: + - sid + retriever: + type: SimpleRetriever + requester: + $ref: "#/definitions/base_requester" + path: >- + https://api.twilio.com/2010-04-01/Accounts/{{ + stream_partition.account_sid }}/Transcriptions.json + http_method: GET + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - transcriptions + paginator: + type: DefaultPaginator + page_token_option: + type: RequestOption + inject_into: request_parameter + field_name: Page + page_size_option: + type: RequestOption + field_name: PageSize + inject_into: request_parameter + pagination_strategy: + type: PageIncrement + page_size: 1000 + start_from_page: 0 + partition_router: + type: SubstreamPartitionRouter + parent_stream_configs: + - type: ParentStreamConfig + parent_key: sid + partition_field: account_sid + stream: + $ref: "#/definitions/streams/Accounts" + schema_loader: + type: InlineSchemaLoader + schema: + $ref: "#/schemas/Transcriptions" + Trunks: + type: DeclarativeStream + name: trunks + primary_key: + - sid + retriever: + type: SimpleRetriever + requester: + $ref: "#/definitions/base_requester" + path: https://trunking.twilio.com/v1/Trunks + http_method: GET + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - trunks + schema_loader: + type: InlineSchemaLoader + schema: + $ref: "#/schemas/Trunks" + Queues: + type: DeclarativeStream + name: queues + primary_key: + - sid + retriever: + type: SimpleRetriever + requester: + $ref: "#/definitions/base_requester" + path: >- + https://api.twilio.com/2010-04-01/Accounts/{{ + stream_partition.account_sid }}/Queues.json + http_method: GET + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - queues + paginator: + type: DefaultPaginator + page_token_option: + type: RequestOption + inject_into: request_parameter + field_name: Page + page_size_option: + type: RequestOption + field_name: PageSize + inject_into: request_parameter + pagination_strategy: + type: PageIncrement + page_size: 1000 + start_from_page: 0 + partition_router: + type: SubstreamPartitionRouter + parent_stream_configs: + - type: ParentStreamConfig + parent_key: sid + partition_field: account_sid + stream: + $ref: "#/definitions/streams/Accounts" + schema_loader: + type: InlineSchemaLoader + schema: + $ref: "#/schemas/Queues" + Messages: + type: DeclarativeStream + name: messages + primary_key: + - sid + retriever: + type: SimpleRetriever + requester: + $ref: "#/definitions/base_requester" + path: >- + https://api.twilio.com/2010-04-01/Accounts/{{ + stream_partition.account_sid }}/Messages.json + http_method: GET + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - messages + paginator: + type: DefaultPaginator + page_token_option: + type: RequestOption + inject_into: request_parameter + field_name: Page + page_size_option: + type: RequestOption + field_name: PageSize + inject_into: request_parameter + pagination_strategy: + type: PageIncrement + page_size: 1000 + start_from_page: 0 + partition_router: + type: SubstreamPartitionRouter + parent_stream_configs: + - type: ParentStreamConfig + parent_key: sid + partition_field: account_sid + stream: + $ref: "#/definitions/streams/Accounts" + incremental_sync: + type: DatetimeBasedCursor + cursor_field: date_sent + cursor_datetime_formats: + - "%Y-%m-%d" + datetime_format: "%Y-%m-%d" + start_datetime: + type: MinMaxDatetime + datetime: '{{ config["start_date"] }}' + datetime_format: "%Y-%m-%dT%H:%M:%SZ" + start_time_option: + type: RequestOption + inject_into: request_parameter + field_name: DateSent> + end_time_option: + type: RequestOption + inject_into: request_parameter + field_name: DateSent< + end_datetime: + type: MinMaxDatetime + datetime: "{{ now_utc().strftime('%Y-%m-%dT%H:%M:%SZ') }}" + datetime_format: "%Y-%m-%dT%H:%M:%SZ" + schema_loader: + type: InlineSchemaLoader + schema: + $ref: "#/schemas/Messages" + Message Media: + type: DeclarativeStream + name: message_media + primary_key: + - sid + retriever: + type: SimpleRetriever + requester: + $ref: "#/definitions/base_requester" + path: >- + https://api.twilio.com/2010-04-01/Accounts/{{ + stream_partition.account_sid }}/Messages/{{ + stream_partition.message_sid }}/Media.json + http_method: GET + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - media + paginator: + type: DefaultPaginator + page_token_option: + type: RequestOption + inject_into: request_parameter + field_name: Page + page_size_option: + type: RequestOption + field_name: PageSize + inject_into: request_parameter + pagination_strategy: + type: PageIncrement + page_size: 1000 + start_from_page: 0 + partition_router: + - type: SubstreamPartitionRouter + parent_stream_configs: + - type: ParentStreamConfig + parent_key: sid + partition_field: account_sid + stream: + $ref: "#/definitions/streams/Accounts" + - type: SubstreamPartitionRouter + parent_stream_configs: + - type: ParentStreamConfig + parent_key: sid + partition_field: message_sid + stream: + $ref: "#/definitions/streams/Messages" + incremental_sync: + type: DatetimeBasedCursor + cursor_field: date_sent + cursor_datetime_formats: + - "%Y-%m-%d" + datetime_format: "%Y-%m-%d" + start_datetime: + type: MinMaxDatetime + datetime: '{{ config["start_date"] }}' + datetime_format: "%Y-%m-%dT%H:%M:%SZ" + start_time_option: + type: RequestOption + inject_into: request_parameter + field_name: DateSent> + end_time_option: + type: RequestOption + inject_into: request_parameter + field_name: DateSent< + end_datetime: + type: MinMaxDatetime + datetime: "{{ now_utc().strftime('%Y-%m-%dT%H:%M:%SZ') }}" + datetime_format: "%Y-%m-%dT%H:%M:%SZ" + schema_loader: + type: InlineSchemaLoader + schema: + $ref: "#/schemas/Message Media" + Usage Records: + type: DeclarativeStream + name: usage_records + primary_key: + - sid + retriever: + type: SimpleRetriever + requester: + $ref: "#/definitions/base_requester" + path: >- + https://api.twilio.com/2010-04-01/Accounts/{{ + stream_partition.account_sid }}/Usage/Records.json + http_method: GET + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - usage_records + paginator: + type: DefaultPaginator + page_token_option: + type: RequestOption + inject_into: request_parameter + field_name: Page + page_size_option: + type: RequestOption + field_name: PageSize + inject_into: request_parameter + pagination_strategy: + type: PageIncrement + page_size: 1000 + start_from_page: 0 + partition_router: + type: SubstreamPartitionRouter + parent_stream_configs: + - type: ParentStreamConfig + parent_key: sid + partition_field: account_sid + stream: + $ref: "#/definitions/streams/Accounts" + incremental_sync: + type: DatetimeBasedCursor + cursor_field: start_date + cursor_datetime_formats: + - "%Y-%m-%d" + datetime_format: "%Y-%m-%d" + start_datetime: + type: MinMaxDatetime + datetime: '{{ config["start_date"] }}' + datetime_format: "%Y-%m-%dT%H:%M:%SZ" + start_time_option: + type: RequestOption + inject_into: request_parameter + field_name: StartDate + end_time_option: + type: RequestOption + inject_into: request_parameter + field_name: EndDate + end_datetime: + type: MinMaxDatetime + datetime: "{{ now_utc().strftime('%Y-%m-%dT%H:%M:%SZ') }}" + datetime_format: "%Y-%m-%dT%H:%M:%SZ" + schema_loader: + type: InlineSchemaLoader + schema: + $ref: "#/schemas/Usage Records" + Usage Triggers: + type: DeclarativeStream + name: usage_triggers + primary_key: + - sid + retriever: + type: SimpleRetriever + requester: + $ref: "#/definitions/base_requester" + path: >- + https://api.twilio.com/2010-04-01/Accounts/{{ + stream_partition.account_sid }}/Usage/Triggers.json + http_method: GET + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - usage_triggers + paginator: + type: DefaultPaginator + page_token_option: + type: RequestOption + inject_into: request_parameter + field_name: Page + page_size_option: + type: RequestOption + field_name: PageSize + inject_into: request_parameter + pagination_strategy: + type: PageIncrement + page_size: 1000 + start_from_page: 0 + partition_router: + type: SubstreamPartitionRouter + parent_stream_configs: + - type: ParentStreamConfig + parent_key: sid + partition_field: account_sid + stream: + $ref: "#/definitions/streams/Accounts" + schema_loader: + type: InlineSchemaLoader + schema: + $ref: "#/schemas/Usage Triggers" + Conversations: + type: DeclarativeStream + name: conversations + primary_key: + - sid + retriever: + type: SimpleRetriever + requester: + $ref: "#/definitions/base_requester" + path: https://conversations.twilio.com/v1/Conversations + http_method: GET + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - conversations + schema_loader: + type: InlineSchemaLoader + schema: + $ref: "#/schemas/Conversations" + Conversation Participants: + type: DeclarativeStream + name: conversation_participants + primary_key: + - sid + retriever: + type: SimpleRetriever + requester: + $ref: "#/definitions/base_requester" + path: >- + https://conversations.twilio.com/v1/Conversations/{{ + stream_partition.conversation_sid }}/Participants + http_method: GET + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - participants + partition_router: + type: SubstreamPartitionRouter + parent_stream_configs: + - type: ParentStreamConfig + parent_key: sid + partition_field: conversation_sid + stream: + $ref: "#/definitions/streams/Conversations" + schema_loader: + type: InlineSchemaLoader + schema: + $ref: "#/schemas/Conversation Participants" + Conversation Messages: + type: DeclarativeStream + name: conversation_messages + primary_key: + - sid + retriever: + type: SimpleRetriever + requester: + $ref: "#/definitions/base_requester" + path: >- + https://conversations.twilio.com/v1/Conversations/{{ + stream_partition.conversation_sid }}/Messages + http_method: GET + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - messages + partition_router: + type: SubstreamPartitionRouter + parent_stream_configs: + - type: ParentStreamConfig + parent_key: sid + partition_field: conversation_sid + stream: + $ref: "#/definitions/streams/Conversations" + schema_loader: + type: InlineSchemaLoader + schema: + $ref: "#/schemas/Conversation Messages" + Users: + type: DeclarativeStream + name: users + primary_key: + - sid + retriever: + type: SimpleRetriever + requester: + $ref: "#/definitions/base_requester" + path: https://conversations.twilio.com/v1/Users + http_method: GET + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - users + schema_loader: + type: InlineSchemaLoader + schema: + $ref: "#/schemas/Users" + User Conversation: + type: DeclarativeStream + name: user_conversation + primary_key: + - conversations + retriever: + type: SimpleRetriever + requester: + $ref: "#/definitions/base_requester" + path: >- + https://conversations.twilio.com/v1/Users/{{ + stream_partition.user_sid }}/Conversations + http_method: GET + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: [] + partition_router: + type: SubstreamPartitionRouter + parent_stream_configs: + - type: ParentStreamConfig + parent_key: sid + partition_field: user_sid + stream: + $ref: "#/definitions/streams/Users" + schema_loader: + type: InlineSchemaLoader + schema: + $ref: "#/schemas/User Conversation" streams: - $ref: "#/definitions/streams/Accounts" - $ref: "#/definitions/streams/Addresses" + - $ref: "#/definitions/streams/Available Phone Number Countries" + - $ref: "#/definitions/streams/Available Phone Numbers Local" + - $ref: "#/definitions/streams/Available Phone Numbers Mobile" + - $ref: "#/definitions/streams/Available Phone Numbers Toll-Free" + - $ref: "#/definitions/streams/Incoming Phone Numbers" + - $ref: "#/definitions/streams/Keys" + - $ref: "#/definitions/streams/Flows" + - $ref: "#/definitions/streams/Steps" + - $ref: "#/definitions/streams/Alerts" + - $ref: "#/definitions/streams/Applications" + - $ref: "#/definitions/streams/Dependent Phone Numbers" + - $ref: "#/definitions/streams/Calls" + - $ref: "#/definitions/streams/Conferences" + - $ref: "#/definitions/streams/Conference Participants" + - $ref: "#/definitions/streams/Executions" + - $ref: "#/definitions/streams/Outgoing Caller Ids" + - $ref: "#/definitions/streams/Recordings" + - $ref: "#/definitions/streams/Services" + - $ref: "#/definitions/streams/Verify Services" + - $ref: "#/definitions/streams/Roles" + - $ref: "#/definitions/streams/Transcriptions" + - $ref: "#/definitions/streams/Trunks" + - $ref: "#/definitions/streams/Queues" + - $ref: "#/definitions/streams/Messages" + - $ref: "#/definitions/streams/Message Media" + - $ref: "#/definitions/streams/Usage Records" + - $ref: "#/definitions/streams/Usage Triggers" + - $ref: "#/definitions/streams/Conversations" + - $ref: "#/definitions/streams/Conversation Participants" + - $ref: "#/definitions/streams/Conversation Messages" + - $ref: "#/definitions/streams/Users" + - $ref: "#/definitions/streams/User Conversation" spec: type: Spec @@ -96,280 +1517,3666 @@ spec: supportsIncremental: true connection_specification: type: object - $schema: http://json-schema.org/draft-07/schema# + $schema: http://json-schema.org/draft-07/schema# + required: + - account_sid + - auth_token + - start_date + title: Twilio Spec + properties: + account_sid: + airbyte_secret: true + description: Twilio account SID + order: 1 + title: Account ID + type: string + auth_token: + airbyte_secret: true + description: Twilio Auth Token. + order: 2 + title: Auth Token + type: string + lookback_window: + default: 0 + description: How far into the past to look for records. (in minutes) + examples: + - 60 + maximum: 576000 + minimum: 0 + order: 4 + title: Lookback window + type: integer + start_date: + description: + UTC date and time in the format 2020-10-01T00:00:00Z. Any data + before this date will not be replicated. + examples: + - "2020-10-01T00:00:00Z" + format: date-time + order: 3 + pattern: ^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$ + title: Replication Start Date + type: string + +metadata: + autoImportSchema: + Accounts: false + Addresses: false + Available Phone Number Countries: false + Available Phone Numbers Local: false + Available Phone Numbers Mobile: false + Available Phone Numbers Toll-Free: false + Incoming Phone Numbers: false + Keys: false + Flows: false + Steps: false + Alerts: false + Applications: false + Dependent Phone Numbers: false + Calls: false + Conferences: false + Conference Participants: false + Executions: false + Outgoing Caller Ids: false + Recordings: false + Services: false + Verify Services: false + Roles: false + Transcriptions: false + Trunks: false + Queues: false + Messages: false + Message Media: false + Usage Records: false + Usage Triggers: false + Conversations: false + Conversation Participants: false + Conversation Messages: false + Users: false + User Conversation: false + +schemas: + Accounts: + type: object + $schema: http://json-schema.org/schema# + additionalProperties: true + properties: + auth_token: + description: The authentication token for the account + type: + - "null" + - string + date_created: + description: The timestamp when the account was created + format: date-time + type: + - "null" + - string + date_updated: + description: The timestamp when the account was last updated + format: date-time + type: + - "null" + - string + friendly_name: + description: A user-defined friendly name for the account + type: + - "null" + - string + owner_account_sid: + description: The SID of the owner account + type: + - "null" + - string + sid: + description: The unique identifier for the account + type: + - "null" + - string + status: + description: The current status of the account + type: + - "null" + - string + subresource_uris: + additionalProperties: true + description: URIs for accessing various subresources related to the account + properties: + addresses: + type: + - "null" + - string + applications: + type: + - "null" + - string + authorized_connect_apps: + type: + - "null" + - string + available_phone_numbers: + type: + - "null" + - string + balance: + type: + - "null" + - string + calls: + type: + - "null" + - string + conferences: + type: + - "null" + - string + connect_apps: + type: + - "null" + - string + incoming_phone_numbers: + type: + - "null" + - string + keys: + type: + - "null" + - string + messages: + type: + - "null" + - string + notifications: + type: + - "null" + - string + outgoing_caller_ids: + type: + - "null" + - string + queues: + type: + - "null" + - string + recordings: + type: + - "null" + - string + short_codes: + type: + - "null" + - string + signing_keys: + type: + - "null" + - string + sip: + type: + - "null" + - string + transcriptions: + type: + - "null" + - string + usage: + type: + - "null" + - string + type: + - "null" + - object + type: + description: The type of the account + type: + - "null" + - string + uri: + description: The URI for accessing the account resource + type: + - "null" + - string + required: + - sid + Addresses: + type: object + $schema: http://json-schema.org/schema# + additionalProperties: true + properties: + account_sid: + description: The unique identifier of the account associated with this address. + type: + - "null" + - string + city: + description: The city of the address. + type: + - "null" + - string + customer_name: + description: The name of the customer associated with this address. + type: + - "null" + - string + date_created: + description: The date and time when the address was created. + format: date-time + type: + - "null" + - string + date_updated: + description: The date and time when the address was last updated. + format: date-time + type: + - "null" + - string + emergency_enabled: + description: Indicates whether emergency services are enabled for this address. + type: + - "null" + - boolean + friendly_name: + description: A friendly name or label for the address. + type: + - "null" + - string + iso_country: + description: The ISO 3166-1 alpha-2 country code of the address. + type: + - "null" + - string + postal_code: + description: The postal code of the address. + type: + - "null" + - string + region: + description: The region or state of the address. + type: + - "null" + - string + sid: + description: The unique identifier of the address. + type: + - "null" + - string + street: + description: The street address. + type: + - "null" + - string + street_secondary: + description: Additional information about the street address, such as suite number. + type: + - "null" + - string + uri: + description: The URI of the address resource. + type: + - "null" + - string + validated: + description: Indicates whether the address has been validated. + type: + - "null" + - boolean + verified: + description: Indicates whether the address has been verified. + type: + - "null" + - boolean + required: + - sid + Available Phone Number Countries: + type: object + $schema: http://json-schema.org/schema# + additionalProperties: true + properties: + beta: + description: Whether the phone number country data is in beta phase or not. + type: + - "null" + - boolean + country: + description: The name of the country for which phone number data is provided. + type: + - "null" + - string + country_code: + description: The country code associated with the country. + type: + - "null" + - string + subresource_uris: + additionalProperties: true + description: URIs for accessing different types of phone numbers in the country. + properties: + local: + description: URI for fetching local phone numbers in the country. + type: + - "null" + - string + mobile: + description: URI for fetching mobile phone numbers in the country. + type: + - "null" + - string + toll_free: + description: URI for fetching toll-free phone numbers in the country. + type: + - "null" + - string + type: + - "null" + - object + uri: + description: The URI to fetch more details about phone numbers in the country. + type: + - "null" + - string + Available Phone Numbers Local: + type: object + $schema: http://json-schema.org/schema# + additionalProperties: true + properties: + address_requirements: + description: + The requirements related to the address associated with the phone + number + type: + - "null" + - string + beta: + description: Indicator showing if the phone number is in beta testing + type: + - "null" + - boolean + capabilities: + additionalProperties: true + description: Object containing the capabilities of the available phone number + properties: + MMS: + description: Capability for Multimedia Messaging Service + type: + - "null" + - boolean + SMS: + description: Capability for Short Message Service (Texting) + type: + - "null" + - boolean + fax: + description: Capability for Facsimile (Faxing) + type: + - "null" + - boolean + voice: + description: Capability for Voice calls + type: + - "null" + - boolean + type: + - "null" + - object + friendly_name: + description: A user-friendly nickname for the phone number + type: + - "null" + - string + iso_country: + description: ISO country code of the phone number + type: + - "null" + - string + lata: + description: Local access and transport area code for the phone number + type: + - "null" + - string + latitude: + description: Latitude coordinate of the phone number's location + type: + - "null" + - string + locality: + description: Locality or city associated with the phone number + type: + - "null" + - string + longitude: + description: Longitude coordinate of the phone number's location + type: + - "null" + - string + phone_number: + description: The phone number itself + type: + - "null" + - string + postal_code: + description: Postal code of the phone number's location + type: + - "null" + - string + rate_center: + description: Rate center for the phone number + type: + - "null" + - string + region: + description: Region or state associated with the phone number + type: + - "null" + - string + Available Phone Numbers Mobile: + type: object + $schema: http://json-schema.org/schema# + additionalProperties: true + properties: + address_requirements: + description: Address requirements for purchasing the phone number + type: + - "null" + - string + beta: + description: Indicates if the phone number is in beta testing phase + type: + - "null" + - boolean + capabilities: + additionalProperties: true + description: + Represents the capabilities of the available mobile phone numbers + such as SMS and voice services + properties: + MMS: + description: Indicates if MMS messaging is supported + type: + - "null" + - boolean + SMS: + description: Indicates if SMS messaging is supported + type: + - "null" + - boolean + fax: + description: Indicates if fax capability is available + type: + - "null" + - boolean + voice: + description: Indicates if voice calling capability is available + type: + - "null" + - boolean + type: + - "null" + - object + friendly_name: + description: A user-friendly name for the phone number + type: + - "null" + - string + iso_country: + description: The ISO country code for the phone number + type: + - "null" + - string + lata: + description: Local access and transport area code + type: + - "null" + - string + latitude: + description: Latitude coordinate of the phone number's location + type: + - "null" + - string + locality: + description: Locality or city where the phone number is based + type: + - "null" + - string + longitude: + description: Longitude coordinate of the phone number's location + type: + - "null" + - string + phone_number: + description: The phone number in E.164 format + type: + - "null" + - string + postal_code: + description: Postal code of the phone number's area + type: + - "null" + - string + rate_center: + description: Rate center associated with the phone number + type: + - "null" + - string + region: + description: Region or state where the phone number is located + type: + - "null" + - string + Available Phone Numbers Toll-Free: + type: object + $schema: http://json-schema.org/schema# + additionalProperties: true + properties: + address_requirements: + description: + Specifies the requirements related to the address associated with + the phone number. + type: + - "null" + - string + beta: + description: Indicates if the phone number is in beta testing phase. + type: + - "null" + - boolean + capabilities: + additionalProperties: true + description: Represents the capabilities of the toll-free phone numbers available + properties: + MMS: + description: Indicates if the phone number supports Multimedia Messaging Service. + type: + - "null" + - boolean + SMS: + description: + Indicates if the phone number supports Short Message Service + (text messaging). + type: + - "null" + - boolean + fax: + description: Indicates if the phone number supports fax communication. + type: + - "null" + - boolean + voice: + description: Indicates if the phone number supports voice calls. + type: + - "null" + - boolean + type: + - "null" + - object + friendly_name: + description: A user-friendly name associated with the phone number for easy identification. + type: + - "null" + - string + iso_country: + description: The ISO country code of the phone number's location. + type: + - "null" + - string + lata: + description: The Local Access and Transport Area code of the phone number's location. + type: + - "null" + - string + latitude: + description: The latitude coordinate of the phone number's location. + type: + - "null" + - string + locality: + description: The locality (city/town) where the phone number is located. + type: + - "null" + - string + longitude: + description: The longitude coordinate of the phone number's location. + type: + - "null" + - string + phone_number: + description: The toll-free phone number that can be used for communication. + type: + - "null" + - string + postal_code: + description: The postal code of the phone number's location. + type: + - "null" + - string + rate_center: + description: The rate center associated with the phone number. + type: + - "null" + - string + region: + description: The region (state/province) where the phone number is located. + type: + - "null" + - string + Incoming Phone Numbers: + type: object + $schema: http://json-schema.org/schema# + additionalProperties: true + properties: + account_sid: + description: The SID of the account that owns this phone number. + type: + - "null" + - string + address_requirements: + description: Indicates the address requirements for this phone number. + type: + - "null" + - string + address_sid: + description: The SID of the address associated with this phone number. + type: + - "null" + - string + api_version: + description: The version of the Twilio API being used for this phone number. + type: + - "null" + - string + beta: + description: Flag indicating beta status of this phone number. + type: + - "null" + - boolean + bundle_sid: + description: The SID of the bundle associated with this phone number. + type: + - "null" + - string + capabilities: + additionalProperties: true + description: Capabilities of this phone number. + properties: + fax: + description: Capability for faxing. + type: + - "null" + - boolean + mms: + description: Capability for sending and receiving MMS messages. + type: + - "null" + - boolean + sms: + description: Capability for sending and receiving SMS messages. + type: + - "null" + - boolean + voice: + description: Capability for making and receiving voice calls. + type: + - "null" + - boolean + type: + - "null" + - object + date_created: + description: The date and time when this phone number was created. + format: date-time + type: + - "null" + - string + date_updated: + description: The date and time when this phone number was last updated. + format: date-time + type: + - "null" + - string + emergency_address_sid: + description: The SID of the emergency address associated with this phone number. + type: + - "null" + - string + emergency_address_status: + description: Status of the emergency address associated with this phone number. + type: + - "null" + - string + emergency_status: + description: Emergency status of this phone number. + type: + - "null" + - string + friendly_name: + description: A user-assigned friendly name for this phone number. + type: + - "null" + - string + identity_sid: + description: The SID of the identity associated with this phone number. + type: + - "null" + - string + origin: + description: Indicates the origin of this phone number. + type: + - "null" + - string + phone_number: + description: The phone number. + type: + - "null" + - string + sid: + description: The SID of this phone number. + type: + - "null" + - string + sms_application_sid: + description: + The SID of the application to handle incoming SMS messages for this + phone number. + type: + - "null" + - string + sms_fallback_method: + description: HTTP method to use when making requests to the sms_fallback_url. + type: + - "null" + - string + sms_fallback_url: + description: + URL to make requests to when an error occurs while processing an + incoming SMS message. + type: + - "null" + - string + sms_method: + description: HTTP method to use when making requests to the sms_url. + type: + - "null" + - string + sms_url: + description: URL to make requests to when an incoming SMS message is received. + type: + - "null" + - string + status: + description: Status of this phone number. + type: + - "null" + - string + status_callback: + description: + URL to make requests to when certain events happen related to this + phone number. + type: + - "null" + - string + status_callback_method: + description: HTTP method to use when making requests to the status_callback URL. + type: + - "null" + - string + subresource_uris: + additionalProperties: true + description: URIs for related sub-resources. + properties: + assigned_add_ons: + description: URI for accessing assigned add-ons for this phone number. + type: + - "null" + - string + type: + - "null" + - object + trunk_sid: + description: The SID of the trunk associated with this phone number. + type: + - "null" + - string + uri: + description: The URI of this phone number. + type: + - "null" + - string + voice_application_sid: + description: + The SID of the application to handle incoming voice calls for this + phone number. + type: + - "null" + - string + voice_caller_id_lookup: + description: Caller ID lookup setting for voice calls made from this phone number. + type: + - "null" + - boolean + voice_fallback_method: + description: HTTP method to use when making requests to the voice_fallback_url. + type: + - "null" + - string + voice_fallback_url: + description: + URL to make requests to when an error occurs while processing an + incoming voice call. + type: + - "null" + - string + voice_method: + description: HTTP method to use when making requests to the voice_url. + type: + - "null" + - string + voice_receive_mode: + description: Receive mode setting for this phone number. + type: + - "null" + - string + voice_url: + description: URL to make requests to when an incoming voice call is received. + type: + - "null" + - string + Keys: + type: object + $schema: http://json-schema.org/schema# + additionalProperties: true + properties: + date_created: + description: The date and time when the key was created. + format: date-time + type: + - "null" + - string + date_updated: + description: The date and time when the key was last updated. + format: date-time + type: + - "null" + - string + friendly_name: + description: The user-friendly name associated with the key. + type: + - "null" + - string + sid: + description: The unique identifier for the key (System ID). + type: + - "null" + - string + Flows: + type: object + $schema: http://json-schema.org/schema# + additionalProperties: true + properties: + account_sid: + description: Unique identifier for the account associated with the flow. + type: + - "null" + - string + date_created: + description: Date and time when the flow was created. + format: date-time + type: + - "null" + - string + date_updated: + description: Date and time when the flow was last updated. + format: date-time + type: + - "null" + - string + friendly_name: + description: A user-friendly name given to the flow. + type: + - "null" + - string + links: + additionalProperties: true + description: Represents links related to the flow data. + properties: + steps: + description: Link to the steps included in the flow. + type: + - "null" + - string + type: + - "null" + - object + sid: + description: Unique identifier for the flow. + type: + - "null" + - string + status: + description: Status of the flow (e.g., active, inactive). + enum: + - published + - draft + type: + - string + - "null" + url: + description: URL endpoint for the flow. + type: + - "null" + - string + version: + description: Version number of the flow. + type: + - "null" + - integer + Steps: + type: object + $schema: http://json-schema.org/schema# + additionalProperties: true + properties: + account_sid: + description: The SID of the account associated with the step data. + type: + - "null" + - string + context: + description: Additional context or metadata related to the step. + type: + - "null" + - string + date_created: + description: The date and time when the step was created. + type: + - "null" + - string + date_updated: + description: The date and time when the step was last updated. + type: + - "null" + - string + execution_sid: + description: The SID of the execution this step belongs to. + type: + - "null" + - string + flow_sid: + description: The SID of the flow to which this step belongs. + type: + - "null" + - string + links: + description: Contains related hypermedia links to navigate through the step data. + properties: + step_context: + description: Link to additional contextual information related to the step. + type: + - "null" + - string + type: + - "null" + - object + name: + description: The name or identifier of the step. + type: + - "null" + - string + parent_step_sid: + description: + The SID of the parent step if this step is nested within another + step. + type: + - "null" + - string + sid: + description: The unique identifier (SID) of the step. + type: + - "null" + - string + transitioned_from: + description: The previous state from which the step transitioned. + type: + - "null" + - string + transitioned_to: + description: The state to which the step transitioned. + type: + - "null" + - string + url: + description: The URL for accessing detailed information about the step. + type: + - "null" + - string + Alerts: + type: object + $schema: http://json-schema.org/schema# + additionalProperties: true + properties: + account_sid: + description: The unique identifier for the account associated with the alert + type: + - "null" + - string + alert_text: + description: The actual content or text of the alert message + type: + - "null" + - string + api_version: + description: The version of the Twilio API used for the alert + type: + - "null" + - string + date_created: + description: The date and time when the alert was created + format: date-time + type: + - "null" + - string + date_generated: + description: The date and time when the alert data was generated + format: date-time + type: + - "null" + - string + date_updated: + description: The date and time when the alert was last updated + format: date-time + type: + - "null" + - string + error_code: + description: The code representing the error, if an error occurred + type: + - "null" + - string + log_level: + description: The severity level of the alert (info, warning, error) + type: + - "null" + - string + more_info: + description: Additional information or details related to the alert + type: + - "null" + - string + request_method: + description: The method used in the HTTP request that triggered the alert + type: + - "null" + - string + request_url: + description: The URL that was requested and triggered the alert + type: + - "null" + - string + resource_sid: + description: The SID of the specific resource associated with the alert + type: + - "null" + - string + service_sid: + description: The SID of the Twilio service related to the alert + type: + - "null" + - string + sid: + description: The unique identifier (SID) for the alert entry + type: + - "null" + - string + url: + description: The URL or endpoint related to the alert + type: + - "null" + - string + incremental_sync: + type: DatetimeBasedCursor + cursor_field: start_date + cursor_datetime_formats: + - "%a, %d %b %Y %H:%M:%S %z" + datetime_format: "%a, %d %b %Y %H:%M:%S %z" + start_datetime: + type: MinMaxDatetime + datetime: '{{ config["start_date"] }}' + datetime_format: "%Y-%m-%dT%H:%M:%SZ" + start_time_option: + type: RequestOption + field_name: StartDate + inject_into: request_parameter + end_time_option: + type: RequestOption + field_name: EndDate + inject_into: request_parameter + end_datetime: + type: MinMaxDatetime + datetime: "{{ now_utc().strftime('%Y-%m-%dT%H:%M:%SZ') }}" + datetime_format: "%Y-%m-%dT%H:%M:%SZ" + Applications: + type: object + $schema: http://json-schema.org/schema# + additionalProperties: true + properties: + account_sid: + description: The unique identifier of the Twilio account associated with the application. + type: + - "null" + - string + api_version: + description: The version of the Twilio API being used for the application. + type: + - "null" + - string + date_created: + description: The date and time when the application was created. + format: date-time + type: + - "null" + - string + date_updated: + description: The date and time when the application was last updated. + format: date-time + type: + - "null" + - string + friendly_name: + description: A user-friendly name assigned to the application for identification. + type: + - "null" + - string + message_status_callback: + description: The URL where status updates for outgoing messages will be sent. + type: + - "null" + - string + public_application_connect_enabled: + description: Indicates whether public application connect is enabled for the application. + type: + - "null" + - boolean + sid: + description: A unique identifier for the application. + type: + - "null" + - string + sms_fallback_method: + description: + The HTTP method used to send a fallback URL if an error occurs during + SMS delivery. + type: + - "null" + - string + sms_fallback_url: + description: The URL that Twilio will request if an error occurs during SMS delivery. + type: + - "null" + - string + sms_method: + description: The HTTP method used to send SMS messages for the application. + type: + - "null" + - string + sms_status_callback: + description: The URL where status updates for incoming SMS messages will be sent. + type: + - "null" + - string + sms_url: + description: The URL that Twilio will request when receiving an incoming SMS message. + type: + - "null" + - string + status_callback: + description: The URL where status updates for calls will be sent. + type: + - "null" + - string + status_callback_method: + description: The HTTP method used to send status updates for calls. + type: + - "null" + - string + uri: + description: The URI of the application resource. + type: + - "null" + - string + voice_caller_id_lookup: + description: + Indicates whether or not caller ID information will be looked up + for incoming calls. + type: + - "null" + - boolean + voice_fallback_method: + description: + The HTTP method used to send a fallback URL if an error occurs during + voice call handling. + type: + - "null" + - string + voice_fallback_url: + description: + The URL that Twilio will request if an error occurs during voice + call handling. + type: + - "null" + - string + voice_method: + description: The HTTP method used for voice call handling. + type: + - "null" + - string + voice_url: + description: The URL that Twilio will request for voice call handling. + type: + - "null" + - string + required: + - sid + Dependent Phone Numbers: + type: object + $schema: http://json-schema.org/schema# + additionalProperties: true + properties: + account_sid: + description: The unique identifier for the account this phone number belongs to. + type: + - "null" + - string + address_requirements: + description: Requirements for the address associated with the phone number. + type: + - "null" + - string + api_version: + description: The version of the Twilio API being used. + type: + - "null" + - string + capabilities: + additionalProperties: true + description: List of capabilities supported by the phone number. + properties: + MMS: + description: Support for multimedia messaging (MMS). + type: + - "null" + - boolean + SMS: + description: Support for text messaging (SMS). + type: + - "null" + - boolean + Voice: + description: Support for voice calling. + type: + - "null" + - boolean + type: + - "null" + - object + date_created: + description: The date and time when the phone number was created. + format: date-time + type: + - "null" + - string + date_updated: + description: The date and time when the phone number was last updated. + format: date-time + type: + - "null" + - string + emergency_address_sid: + description: The SID of the emergency address associated with the phone number. + type: + - "null" + - string + emergency_status: + description: The emergency status of the phone number. + type: + - "null" + - string + friendly_name: + description: A user-friendly name for the phone number. + type: + - "null" + - string + phone_number: + description: The phone number itself. + type: + - "null" + - string + sid: + description: The unique identifier (SID) for the phone number resource. + type: + - "null" + - string + sms_application_sid: + description: The SID of the SMS application to handle incoming messages. + type: + - "null" + - string + sms_fallback_method: + description: The HTTP method to use for the SMS fallback URL. + type: + - "null" + - string + sms_fallback_url: + description: The URL to call if an error occurs while handling incoming SMS messages. + type: + - "null" + - string + sms_method: + description: The HTTP method to use for the SMS URL. + type: + - "null" + - string + sms_url: + description: The URL to call when a message comes in to the phone number. + type: + - "null" + - string + status_callback: + description: + The URL to send status callbacks to for events related to the phone + number. + type: + - "null" + - string + status_callback_method: + description: The HTTP method to use for status callback requests. + type: + - "null" + - string + trunk_sid: + description: The SID of the Trunk containing this phone number. + type: + - "null" + - string + uri: + description: The URI of the phone number resource. + type: + - "null" + - string + voice_application_sid: + description: The SID of the voice application to handle incoming calls. + type: + - "null" + - string + voice_caller_id_lookup: + description: + Indicates whether the phone number performs caller ID lookup on incoming + calls. + type: + - "null" + - boolean + voice_fallback_method: + description: The HTTP method to use for the voice fallback URL. + type: + - "null" + - string + voice_fallback_url: + description: + The URL to call if an error occurs while handling incoming voice + calls. + type: + - "null" + - string + voice_method: + description: The HTTP method to use for the voice URL. + type: + - "null" + - string + voice_url: + description: The URL to call when a call comes in to the phone number. + type: + - "null" + - string + required: + - sid + Calls: + type: object + $schema: http://json-schema.org/schema# + additionalProperties: true + properties: + account_sid: + description: The unique identifier for the account associated with the call. + type: + - "null" + - string + annotation: + description: Any additional notes or context added to the call. + type: + - "null" + - string + answered_by: + description: The name of the entity or system that answered the call. + type: + - "null" + - string + api_version: + description: The version of the Twilio API used for this call. + type: + - "null" + - string + caller_name: + description: The name of the caller as supplied by the caller ID service. + type: + - "null" + - string + date_created: + description: The date and time when the call record was created. + format: date-time + type: + - "null" + - string + date_updated: + description: The date and time when the call record was last updated. + format: date-time + type: + - "null" + - string + direction: + description: The direction of the call, either 'inbound' or 'outbound'. + type: + - "null" + - string + duration: + description: The duration of the call in seconds. + type: + - "null" + - integer + end_time: + description: The date and time when the call ended. + format: date-time + type: + - "null" + - string + forwarded_from: + description: The phone number that initiated the call forwarding. + type: + - "null" + - string + from: + description: The phone number that made the call. + type: + - "null" + - string + from_formatted: + description: The formatted version of the 'from' phone number. + type: + - "null" + - string + group_sid: + description: The unique identifier of the call group to which this call belongs. + type: + - "null" + - string + parent_call_sid: + description: The SID of the parent call if this call is part of a conference. + type: + - "null" + - string + phone_number_sid: + description: The SID of the phone number used for the call. + type: + - "null" + - string + price: + description: The cost of the call. + type: + - "null" + - number + price_unit: + description: The currency unit of the call cost. + type: + - "null" + - string + queue_time: + description: The time the call spent in a queue before being answered. + type: + - "null" + - integer + sid: + description: The unique identifier for the call. + type: + - "null" + - string + start_time: + description: The date and time when the call started. + format: date-time + type: + - "null" + - string + status: + description: The current status of the call, such as 'completed' or 'in-progress'. + type: + - "null" + - string + subresource_uris: + additionalProperties: true + description: Contains URIs for related subresources such as recordings or notifications. + properties: + events: + description: URI for call events related to this call. + type: + - "null" + - string + feedback: + description: URI for call feedback related to this call. + type: + - "null" + - string + feedback_summaries: + description: URI for call feedback summaries related to this call. + type: + - "null" + - string + notifications: + description: URI for call notifications related to this call. + type: + - "null" + - string + payments: + description: URI for call payments related to this call. + type: + - "null" + - string + recordings: + description: URI for call recordings related to this call. + type: + - "null" + - string + siprec: + description: URI for call SIP recording related to this call. + type: + - "null" + - string + streams: + description: URI for call streams related to this call. + type: + - "null" + - string + user_defined_message_subscriptions: + description: URI for user-defined message subscriptions related to this call. + type: + - "null" + - string + user_defined_messages: + description: URI for user-defined messages related to this call. + type: + - "null" + - string + type: + - "null" + - object + to: + description: The phone number that received the call. + type: + - "null" + - string + to_formatted: + description: The formatted version of the 'to' phone number. + type: + - "null" + - string + trunk_sid: + description: The unique identifier of the trunk used for the call. + type: + - "null" + - string + uri: + description: The URI for this call record. + type: + - "null" + - string + required: + - sid + Conferences: + type: object + $schema: http://json-schema.org/schema# + additionalProperties: true + properties: + account_sid: + description: The unique identifier of the account associated with the conference. + type: + - "null" + - string + api_version: + description: The version of the Twilio API used for the conference. + type: + - "null" + - string + call_sid_ending_conference: + description: The SID of the call that ended the conference, if applicable. + type: + - "null" + - string + date_created: + description: The date and time when the conference was created. + format: date-time + type: + - "null" + - string + date_updated: + description: The date and time when the conference was last updated. + format: date-time + type: + - "null" + - string + friendly_name: + description: A user-defined friendly name for the conference. + type: + - "null" + - string + reason_conference_ended: + description: The reason for the conference ending, if available. + type: + - "null" + - string + region: + description: The region where the conference is hosted. + type: + - "null" + - string + sid: + description: The unique identifier (SID) of the conference. + type: + - "null" + - string + status: + description: The current status of the conference. + type: + - "null" + - string + subresource_uris: + additionalProperties: true + description: Contains URLs for accessing subresources related to conferences + properties: + participants: + description: URI to access the participants associated with the conference. + type: + - "null" + - string + recordings: + description: URI to access the recordings of the conference. + type: + - "null" + - string + type: + - "null" + - object + uri: + description: The URI to access the conference resource. + type: + - "null" + - string + required: + - sid + Conference Participants: + type: object + $schema: http://json-schema.org/schema# + additionalProperties: true + properties: + account_sid: + description: "The unique identifier for the Twilio account associated with the conference participant." + type: + - "null" + - string + call_sid: + description: "The unique identifier for the call made by the participant." + type: + - "null" + - string + label: + description: "A custom label assigned to the participant for identification." + type: + - "null" + - string + conference_sid: + description: "The unique identifier for the conference the participant is a part of." + type: + - "null" + - string + date_created: + description: "The date and time when the participant was added to the conference." + format: "date-time" + type: + - "null" + - string + date_updated: + description: "The date and time when the participant's information was last updated." + format: "date-time" + type: + - "null" + - string + end_conference_on_exit: + description: "Indicates whether the conference will end when the participant exits." + type: + - "null" + - boolean + muted: + description: "Indicates whether the participant's audio is muted." + type: + - "null" + - boolean + hold: + description: "Indicates whether the participant is on hold." + type: + - "null" + - boolean + status: + description: "The current status of the participant in the conference." + type: + - "null" + - string + start_conference_on_enter: + description: "Indicates whether the conference will start when the participant enters." + type: + - "null" + - boolean + coaching: + description: "Indicates whether the participant is in coaching mode." + type: + - "null" + - boolean + call_sid_to_coach: + description: "The unique identifier for the call made to the coach." + type: + - "null" + - string + uri: + description: "The URI for accessing the details of the conference participant." + type: + - "null" + - string + Executions: + type: object + $schema: http://json-schema.org/draft-07/schema# + additionalProperties: true + properties: + account_sid: + description: The unique identifier for the account associated with the execution. + type: + - "null" + - string + contact_channel_address: + description: The contact channel address where the execution is targeted. + type: + - "null" + - string + contact_sid: + description: The unique identifier for the contact associated with the execution. + type: + - "null" + - string + context: + additionalProperties: true + description: Additional context or data related to the execution. + type: + - "null" + - object + date_created: + description: The date and time when the execution was created. + format: date-time + type: + - "null" + - string + date_updated: + description: The date and time when the execution was last updated. + format: date-time + type: + - "null" + - string + flow_sid: + description: The unique identifier for the flow associated with the execution. + type: + - "null" + - string + links: + additionalProperties: true + description: + Contains related hypermedia links associated with the executions + data. + properties: + execution_context: + description: Link to the execution context data. + type: + - "null" + - string + steps: + description: Link to the steps involved in the execution. + type: + - "null" + - string + type: + - "null" + - object + sid: + description: The unique identifier for the execution. + type: + - "null" + - string + status: + description: The status of the execution process. + type: + - "null" + - string + url: + description: The URL endpoint for accessing detailed information about the execution. + type: + - "null" + - string + Outgoing Caller Ids: + type: object + $schema: http://json-schema.org/draft-07/schema# + additionalProperties: true + properties: + account_sid: + description: + The unique identifier for the Twilio account associated with this + outgoing caller ID. + type: + - "null" + - string + date_created: + description: The date and time when the outgoing caller ID was created. + format: date-time + type: + - "null" + - string + date_updated: + description: The date and time when the outgoing caller ID was last updated. + format: date-time + type: + - "null" + - string + friendly_name: + description: A user-defined friendly name for the outgoing caller ID. + type: + - "null" + - string + phone_number: + description: The phone number associated with the outgoing caller ID. + type: + - "null" + - string + sid: + description: The unique identifier for the outgoing caller ID. + type: + - "null" + - string + uri: + description: + The URI for accessing detailed information about the outgoing caller + ID. + type: + - "null" + - string + Recordings: + type: object + $schema: http://json-schema.org/schema# + additionalProperties: true + properties: + account_sid: + description: The unique identifier of the account that owns the recording. + type: + - "null" + - string + api_version: + description: The version of the Twilio API used when the recording was created. + type: + - "null" + - string + call_sid: + description: The unique identifier of the call associated with the recording. + type: + - "null" + - string + channels: + description: The number of audio channels in the recording. + type: + - "null" + - integer + conference_sid: + description: + The unique identifier of the conference, if the recording is part + of a conference. + type: + - "null" + - string + date_created: + description: The date and time when the recording was created. + format: date-time + type: + - "null" + - string + date_updated: + description: The date and time when the recording was last updated. + format: date-time + type: + - "null" + - string + duration: + description: The duration of the recording in seconds. + type: + - "null" + - integer + encryption_details: + additionalProperties: true + description: Details about the encryption used for the recording. + properties: + encryption_cek: + description: The Content Encryption Key used for encryption. + type: + - "null" + - string + encryption_public_key_sid: + description: The SID of the public key used for encryption. + type: + - "null" + - string + iv: + description: The Initialization Vector used for encryption. + type: + - "null" + - string + type: + description: The type of encryption used. + type: + - "null" + - string + type: + - "null" + - object + error_code: + description: The error code, if any, associated with the recording. + type: + - "null" + - integer + media_url: + description: The URL where the recording audio file can be accessed. + type: string + price: + description: The cost of storing the recording. + type: + - "null" + - number + price_unit: + description: The unit in which the cost is measured (e.g., USD). + type: + - "null" + - string + sid: + description: The unique identifier of the recording. + type: + - "null" + - string + source: + description: The source of the recording (e.g., twilio). + type: + - "null" + - string + start_time: + description: The date and time when the recording started. + format: date-time + type: + - "null" + - string + status: + description: The status of the recording (e.g., completed, in-progress). + type: + - "null" + - string + subresource_uris: + additionalProperties: true + description: URIs for subresources related to the recording. + properties: + add_on_results: + description: The URI to fetch add-on results related to the recording. + type: + - "null" + - string + transcriptions: + description: The URI to fetch transcriptions of the recording. + type: + - "null" + - string + type: + - "null" + - object + uri: + description: The URI of the recording resource. + type: + - "null" + - string + required: + - sid + Services: + type: object + $schema: http://json-schema.org/schema# + additionalProperties: true + properties: + account_sid: + description: The unique identifier of the account related to the service. + type: + - "null" + - string + consumption_report_interval: + description: The interval at which consumption reports are generated. + type: + - "null" + - number + date_created: + description: The date and time when the service was created. + format: date-time + type: + - "null" + - string + date_updated: + description: The date and time when the service was last updated. + format: date-time + type: + - "null" + - string + default_channel_creator_role_sid: + description: The default role assigned to the creator of a channel. + type: + - "null" + - string + default_channel_role_sid: + description: The default role assigned to all users in a channel. + type: + - "null" + - string + default_service_role_sid: + description: The default role assigned to users of the service. + type: + - "null" + - string + friendly_name: + description: A user-friendly name given to the service. + type: + - "null" + - string + limits: + description: Limits applied to the service. + properties: + channel_members: + description: Maximum number of members allowed in a channel. + type: + - "null" + - number + user_channels: + description: Maximum number of channels a user can belong to. + type: + - "null" + - number + type: + - "null" + - object + links: + description: Links to different resources related to the service. + properties: + bindings: + description: Links related to bindings. + type: + - "null" + - string + channels: + description: Links related to channels. + type: + - "null" + - string + roles: + description: Links related to roles. + type: + - "null" + - string + users: + description: Links related to users. + type: + - "null" + - string + type: + - "null" + - object + media: + description: Media settings for the service. + properties: + compatibility_message: + description: Message to display for incompatible media. + type: + - "null" + - string + size_limit_mb: + description: Maximum size limit for media in megabytes. + type: + - "null" + - number + type: + - "null" + - object + notifications: + description: Notification settings for users. + properties: + users: + description: Users to receive notifications. + type: + - "null" + - string + type: + - "null" + - object + post_webhook_retry_count: + description: Number of retries for the post webhook. + type: + - "null" + - number + post_webhook_url: + description: URL for the post webhook. + type: + - "null" + - string + pre_webhook_retry_count: + description: Number of retries for the pre webhook. + type: + - "null" + - number + pre_webhook_url: + description: URL for the pre webhook. + type: + - "null" + - string + reachability_enabled: + description: Flag indicating if reachability is enabled. + type: + - "null" + - boolean + read_status_enabled: + description: Flag indicating if read status is enabled. + type: + - "null" + - boolean + sid: + description: The unique identifier of the service. + type: + - "null" + - string + typing_indicator_timeout: + description: Timeout duration for typing indicators. + type: + - "null" + - number + url: + description: URL of the service. + type: + - "null" + - string + webhook_filters: + description: Filters for webhooks. + items: + type: + - "null" + - string + type: + - "null" + - array + webhook_method: + description: HTTP method used for webhooks. + type: + - "null" + - string + Verify Services: + type: object + $schema: http://json-schema.org/schema# + additionalProperties: true + properties: + account_sid: + description: Unique identifier for the account associated with the verify service. + type: + - "null" + - string + code_length: + description: The number of digits in the verification code sent to users. + type: + - "null" + - number + custom_code_enabled: + description: Indicates whether custom verification codes are enabled. + type: + - "null" + - boolean + date_created: + description: The date and time when the verify service was created. + type: + - "null" + - string + date_updated: + description: The date and time when the verify service was last updated. + type: + - "null" + - string + default_template_sid: + description: + The unique identifier for the default template used for verification + messages. + type: + - "null" + - string + do_not_share_warning_enabled: + description: + Indicates if the 'do not share' warning is enabled for verification + codes. + type: + - "null" + - boolean + dtmf_input_required: + description: Indicates whether DTMF input is required during verification. + type: + - "null" + - boolean + friendly_name: + description: A user-friendly name for the verify service. + type: + - "null" + - string + links: + description: Object containing related hyperlinks for verify_services data. + properties: + access_tokens: + description: Links related to access tokens for the verify service. + type: + - "null" + - string + entities: + description: Links related to entities associated with the verify service. + type: + - "null" + - string + messaging_configurations: + description: Links related to messaging configurations for the verify service. + type: + - "null" + - string + rate_limits: + description: Links related to rate limits for the verify service. + type: + - "null" + - string + verification_checks: + description: Links related to verification checks for the verify service. + type: + - "null" + - string + verifications: + description: Links related to verifications performed using the verify service. + type: + - "null" + - string + webhooks: + description: Links related to webhooks for the verify service. + type: + - "null" + - string + type: + - "null" + - object + lookup_enabled: + description: Indicates if phone number lookup is enabled for the verify service. + type: + - "null" + - boolean + mailer_sid: + description: + Unique identifier for the mailer service associated with the verify + service. + type: + - "null" + - string + psd2_enabled: + description: + Indicates if PSD2 (Payment Services Directive 2) verification is + enabled. + type: + - "null" + - boolean + push: + description: Object containing configuration settings for push verification method. + properties: + apn_credential_sid: + description: + Unique identifier for the APN (Apple Push Notification) credential + associated with the verify service. + type: + - "null" + - string + fcm_credential_sid: + description: + Unique identifier for the FCM (Firebase Cloud Messaging) credential + associated with the verify service. + type: + - "null" + - string + include_date: + description: Indicates if the date should be included in push notifications. + type: + - "null" + - boolean + type: + - "null" + - object + sid: + description: Unique identifier for the verify service. + type: + - "null" + - string + skip_sms_to_landlines: + description: + Indicates whether SMS messages are skipped for landline numbers during + verification. + type: + - "null" + - boolean + totp: + description: + Object containing configuration settings for Time-based One-Time + Password (TOTP) verification method. + properties: + code_length: + description: + The number of digits in the TOTP (Time-based One-Time Password) + code. + type: + - "null" + - number + issuer: + description: The issuer name included in TOTP messages. + type: + - "null" + - string + skew: + description: The time skew allowed for TOTP code validation. + type: + - "null" + - number + time_step: + description: The time step interval for generating TOTP codes. + type: + - "null" + - number + type: + - "null" + - object + tts_name: + description: The name used for text-to-speech (TTS) in verification calls. + type: + - "null" + - string + url: + description: The URL associated with the verify service. + type: + - "null" + - string + required: + - sid + Roles: + type: object + $schema: http://json-schema.org/schema# + additionalProperties: true + properties: + account_sid: + description: The unique identifier for the account associated with the role. + type: + - "null" + - string + date_created: + description: The date and time when the role was created. + format: date-time + type: + - "null" + - string + date_updated: + description: The date and time when the role was last updated. + format: date-time + type: + - "null" + - string + friendly_name: + description: A user-friendly name assigned to the role. + type: + - "null" + - string + permissions: + description: The list of permissions assigned to the role. + items: + type: + - "null" + - string + type: + - "null" + - array + service_sid: + description: The unique identifier for the service associated with the role. + type: + - "null" + - string + sid: + description: The unique identifier for the role. + type: + - "null" + - string + type: + description: The type or category of the role. + type: + - "null" + - string + url: + description: The URL for accessing the role data. + type: + - "null" + - string + required: + - sid + Transcriptions: + type: object + $schema: http://json-schema.org/schema# + additionalProperties: true + properties: + account_sid: + description: + The unique identifier for the account associated with the transcription + data. + type: + - "null" + - string + api_version: + description: The version of the Twilio API used for the transcription. + type: + - "null" + - string + date_created: + description: The date and time when the transcription was created. + format: date-time + type: + - "null" + - string + date_updated: + description: The date and time when the transcription was last updated. + format: date-time + type: + - "null" + - string + duration: + description: + The duration of the audio recording for which the transcription was + generated. + type: + - "null" + - integer + price: + description: The cost of the transcription service. + type: + - "null" + - number + price_unit: + description: The currency unit used for pricing the transcription service. + type: + - "null" + - string + recording_sid: + description: The unique identifier for the recording associated with the transcription. + type: + - "null" + - string + sid: + description: The unique identifier for the transcription data. + type: + - "null" + - string + status: + description: The current status of the transcription (e.g., in-progress, completed). + type: + - "null" + - string + transcription_text: + description: The text content of the transcription generated from the audio recording. + type: + - "null" + - string + type: + description: The type of transcription (e.g., automatic, human-generated). + type: + - "null" + - string + uri: + description: The URI where the transcription data can be accessed. + type: + - "null" + - string + required: + - sid + Trunks: + type: object + $schema: http://json-schema.org/schema# + additionalProperties: true + properties: + account_sid: + description: The unique identifier of the account associated with the trunk. + type: + - "null" + - string + auth_type: + description: The authentication type used for this trunk. + type: + - "null" + - string + auth_type_set: + description: A set of authentication types associated with the trunk. + items: + auth_type: + description: The authentication type used for each item in the set. + type: + - "null" + - string + type: + - "null" + - array + cnam_lookup_enabled: + description: + Indicates whether Caller ID Name (CNAM) lookup is enabled for this + trunk. + type: + - "null" + - boolean + date_created: + description: The date and time the trunk was created. + format: date-time + type: + - "null" + - string + date_updated: + description: The date and time the trunk was last updated. + format: date-time + type: + - "null" + - string + disaster_recovery_method: + description: The method used for disaster recovery for this trunk. + type: + - "null" + - string + disaster_recovery_url: + description: The URL set up for disaster recovery in case of trunk failure. + type: + - "null" + - string + domain_name: + description: The domain name associated with the trunk. + type: + - "null" + - string + friendly_name: + description: A user-friendly name for the trunk. + type: + - "null" + - string + links: + description: Links to related resources associated with the trunk. + properties: + credential_lists: + description: Endpoint for managing credential lists associated with the trunk. + type: + - "null" + - string + ip_access_control_lists: + description: + Endpoint for managing IP Access Control Lists (ACLs) associated + with the trunk. + type: + - "null" + - string + origination_urls: + description: Endpoint for managing origination URLs associated with the trunk. + type: + - "null" + - string + phone_numbers: + description: Endpoint for managing phone numbers associated with the trunk. + type: + - "null" + - string + type: + - "null" + - object + recording: + description: Settings related to call recording on the trunk. + properties: + mode: + description: The mode used for recording calls on this trunk. + type: + - "null" + - string + trim: + description: Indicates whether recorded calls should be trimmed. + type: + - "null" + - string + type: + - "null" + - object + secure: + description: Indicates whether the trunk connection is secure. + type: + - "null" + - boolean + sid: + description: The unique identifier for the trunk. + type: + - "null" + - string + transfer_caller_id: + description: The Caller ID used when transferring calls through this trunk. + type: + - "null" + - string + transfer_mode: + description: The mode used for call transfer on this trunk. + type: + - "null" + - string + url: + description: The URL of the trunk. + type: + - "null" + - string + required: + - sid + Queues: + type: object + $schema: http://json-schema.org/schema# + additionalProperties: true + properties: + account_sid: + description: The unique identifier of the account that owns this queue. + type: + - "null" + - string + average_wait_time: + description: + The average time, in seconds, that callers have spent waiting in + this queue. + type: + - "null" + - integer + current_size: + description: The current number of callers waiting in this queue. + type: + - "null" + - integer + date_created: + description: The date and time when this queue was created. + format: date-time + type: + - "null" + - string + date_updated: + description: The date and time when this queue was last updated. + format: date-time + type: + - "null" + - string + friendly_name: + description: A user-defined human-readable name for this queue. + type: + - "null" + - string + max_size: + description: The maximum number of callers allowed in this queue. + type: + - "null" + - integer + sid: + description: A unique identifier for this queue. + type: + - "null" + - string + subresource_uris: + additionalProperties: true + description: Contains URIs for related subresources of this queue. + properties: + members: + description: A URI to fetch the members (callers) of this queue. + type: + - "null" + - string + type: + - "null" + - object + uri: + description: The URI for this queue resource. + type: + - "null" + - string + required: + - sid + Messages: + type: object + $schema: http://json-schema.org/schema# + additionalProperties: true + properties: + account_sid: + description: The unique identifier for the account associated with this message. + type: + - "null" + - string + api_version: + description: + The version of the Twilio API being used to send or receive this + message. + type: + - "null" + - string + body: + description: The text body of the message. + type: + - "null" + - string + date_created: + description: The date and time when the message was created. + format: date-time + type: + - "null" + - string + date_sent: + description: The date and time when the message was sent. + format: date-time + type: + - "null" + - string + date_updated: + description: The date and time when the message was last updated. + format: date-time + type: + - "null" + - string + direction: + description: The direction of the message, whether it's incoming or outgoing. + type: + - "null" + - string + error_code: + description: The error code associated with the message, if any. + type: + - "null" + - string + error_message: + description: The error message description if the message failed to send. + type: + - "null" + - string + from: + description: The phone number or sender ID that the message originated from. + type: + - "null" + - string + messaging_service_sid: + description: + The unique identifier for the messaging service associated with the + message. + type: + - "null" + - string + num_media: + description: + The number of media files (e.g., images, videos) included in the + message. + type: + - "null" + - integer + num_segments: + description: The number of message segments if the message exceeds 160 characters. + type: + - "null" + - integer + price: + description: The cost of sending the message. + type: + - "null" + - number + price_unit: + description: The currency unit used for pricing of the message. + type: + - "null" + - string + sid: + description: The unique identifier for this message. + type: + - "null" + - string + status: + description: The status of the message (e.g., sent, delivered, failed). + type: + - "null" + - string + subresource_uris: + additionalProperties: true + description: Contains links to the subresources related to the message. + properties: + feedback: + description: URI to access feedback related to the message. + type: + - "null" + - string + media: + description: URI to access media files associated with the message. + type: + - "null" + - string + type: + - "null" + - object + to: + description: The phone number or recipient ID that the message was sent to. + type: + - "null" + - string + uri: + description: The URI for accessing detailed information about this message. + type: + - "null" + - string + required: + - sid + Message Media: + type: object + $schema: http://json-schema.org/draft-07/schema# + additionalProperties: true + properties: + account_sid: + description: + The unique identifier of the account associated with this message + media. + type: + - "null" + - string + content_type: + description: The MIME type of the media content. + type: + - "null" + - string + date_created: + description: The date and time when the message media was created. + format: date-time + type: + - "null" + - string + date_updated: + description: The date and time when the message media was last updated. + format: date-time + type: + - "null" + - string + parent_sid: + description: + The unique identifier of the parent message associated with this + media. + type: + - "null" + - string + sid: + description: The unique identifier of the message media. + type: + - "null" + - string + uri: + description: The URI that can be used to access this message media resource. + type: + - "null" + - string + Usage Records: + type: object + $schema: http://json-schema.org/schema# + additionalProperties: true + properties: + account_sid: + description: The unique identifier for the account associated with the usage record. + type: + - "null" + - string + api_version: + description: + The version of the Twilio API used in the interaction that generated + the usage record. + type: + - "null" + - string + as_of: + description: + The timestamp indicating the date and time up to which the usage + data is accurate. + type: + - "null" + - string + category: + description: + The category to which the usage data belongs, such as calls, SMS, + or recordings. + type: + - "null" + - string + count: + description: The number of units consumed in the usage record. + type: + - "null" + - integer + count_unit: + description: + The unit of measurement for the count field, such as minutes, messages, + or bytes. + type: + - "null" + - string + description: + description: A brief explanation or summary of the usage record data. + type: + - "null" + - string + end_date: + description: The end date of the usage record period. + format: date + type: + - "null" + - string + price: + description: The total price incurred for the consumed units in the usage record. + type: + - "null" + - number + price_unit: + description: The currency unit in which the price is denoted, such as USD or EUR. + type: + - "null" + - string + start_date: + description: The start date and time of the usage record period. + format: date-time + type: + - "null" + - string + subresource_uris: + additionalProperties: true + description: Contains references to sub-resources related to the usage record. + properties: + all_time: + description: + The resource URI for fetching usage records spanning all time + periods. + type: + - "null" + - string + daily: + description: The resource URI for fetching daily usage records. + type: + - "null" + - string + last_month: + description: The resource URI for fetching usage records for the last month. + type: + - "null" + - string + monthly: + description: The resource URI for fetching monthly usage records. + type: + - "null" + - string + this_month: + description: The resource URI for fetching usage records for the current month. + type: + - "null" + - string + today: + description: The resource URI for fetching today's usage records. + type: + - "null" + - string + yearly: + description: The resource URI for fetching yearly usage records. + type: + - "null" + - string + yesterday: + description: The resource URI for fetching yesterday's usage records. + type: + - "null" + - string + type: + - "null" + - object + uri: + description: The unique resource identifier for the usage record. + type: + - "null" + - string + usage: + description: + The total usage value, which could be a cumulative count or a sum + of values depending on the category. + type: + - "null" + - number + usage_unit: + description: + The unit of measurement for the usage field, corresponding to the + usage count unit. + type: + - "null" + - string required: - - account_sid - - auth_token - start_date - title: Twilio Spec + Usage Triggers: + type: object + $schema: http://json-schema.org/schema# + additionalProperties: true properties: account_sid: - airbyte_secret: true - description: Twilio account SID - order: 1 - title: Account ID - type: string - auth_token: - airbyte_secret: true - description: Twilio Auth Token. - order: 2 - title: Auth Token - type: string - lookback_window: - default: 0 - description: How far into the past to look for records. (in minutes) - examples: - - 60 - maximum: 576000 - minimum: 0 - order: 4 - title: Lookback window - type: integer - start_date: + description: The unique identifier for the account associated with the usage trigger. + type: + - "null" + - string + api_version: + description: The version of the Twilio API used for the trigger. + type: + - "null" + - string + callback_method: + description: The HTTP method used for callback notifications. + type: + - "null" + - string + callback_url: + description: The URL where callback notifications are sent. + type: + - "null" + - string + current_value: + description: The current value for the trigger condition. + type: + - "null" + - number + date_created: + description: The date and time when the trigger was created. + format: date-time + type: + - "null" + - string + date_fired: + description: The date and time when the trigger was last fired. + format: date-time + type: + - "null" + - string + date_updated: + description: The date and time when the trigger was last updated. + format: date-time + type: + - "null" + - string + friendly_name: + description: A user-friendly name for the trigger. + type: + - "null" + - string + recurring: + description: Indicates if the trigger is set to recur periodically. + type: + - "null" + - string + sid: + description: The unique identifier for the trigger. + type: + - "null" + - string + trigger_by: + description: The attribute by which the trigger condition is evaluated. + type: + - "null" + - string + trigger_value: + description: The value that triggers the condition. + type: + - "null" + - number + uri: + description: The URI of the usage trigger resource. + type: + - "null" + - string + usage_category: + description: The category under which the usage falls. + type: + - "null" + - string + usage_record_uri: + description: The URI of the associated usage record. + type: + - "null" + - string + required: + - sid + Conversations: + type: object + $schema: http://json-schema.org/schema# + additionalProperties: true + properties: + account_sid: + description: The unique identifier for the account associated with the conversation. + type: + - "null" + - string + attributes: + description: Additional attributes or metadata associated with the conversation. + type: + - "null" + - string + bindings: + description: Information about the communication channels bound to the conversation. + type: + - "null" + - object + chat_service_sid: description: - UTC date and time in the format 2020-10-01T00:00:00Z. Any data - before this date will not be replicated. - examples: - - "2020-10-01T00:00:00Z" + The SID (Service Identifier) for the chat service to which the conversation + belongs. + type: + - "null" + - string + date_created: + description: The date and time when the conversation was created. format: date-time - order: 3 - pattern: ^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$ - title: Replication Start Date - type: string - -metadata: - autoImportSchema: - Accounts: true - Addresses: true - -schemas: - Accounts: + type: + - "null" + - string + date_updated: + description: The date and time when the conversation was last updated. + format: date-time + type: + - "null" + - string + friendly_name: + description: A human-readable name assigned to the conversation. + type: + - "null" + - string + messaging_service_sid: + description: The SID for the messaging service associated with the conversation. + type: + - "null" + - string + sid: + description: The unique identifier for the conversation. + type: + - "null" + - string + state: + description: The current state of the conversation (e.g., active, inactive). + type: + - "null" + - string + timers: + description: Information about timers set for the conversation. + type: + - "null" + - object + unique_name: + description: A unique name assigned to the conversation for easy identification. + type: + - "null" + - string + url: + description: + The URL endpoint for accessing or interacting with the conversation + data. + type: + - "null" + - string + required: + - sid + Conversation Participants: type: object $schema: http://json-schema.org/schema# additionalProperties: true properties: - auth_token: - description: The authentication token for the account + account_sid: + description: The Twilio account SID of the conversation participant. type: - "null" - string - date_created: - description: The timestamp when the account was created - format: date-time + attributes: + description: Additional attributes related to the conversation participant. type: - "null" - string - date_updated: - description: The timestamp when the account was last updated + conversation_sid: + description: The SID of the conversation the participant belongs to. + type: + - "null" + - string + date_created: + description: The date and time when the conversation participant was created. format: date-time type: - "null" - string - friendly_name: - description: A user-defined friendly name for the account + date_updated: + description: The date and time when the conversation participant was last updated. + format: date-time type: - "null" - string - owner_account_sid: - description: The SID of the owner account + identity: + description: The identity of the participant in the conversation. type: - "null" - string - sid: - description: The unique identifier for the account + last_read_message_index: + description: The index of the last message that the participant has read. type: - "null" - string - status: - description: The current status of the account + last_read_timestamp: + description: The timestamp when the participant last read a message. type: - "null" - string - subresource_uris: - additionalProperties: true - description: URIs for accessing various subresources related to the account + messaging_binding: + description: Details about the messaging binding for the conversation participant. properties: - addresses: - type: - - "null" - - string - applications: - type: - - "null" - - string - authorized_connect_apps: - type: - - "null" - - string - available_phone_numbers: - type: - - "null" - - string - balance: - type: - - "null" - - string - calls: - type: - - "null" - - string - conferences: - type: - - "null" - - string - connect_apps: - type: - - "null" - - string - incoming_phone_numbers: - type: - - "null" - - string - keys: + address: + description: The address associated with the participant's messaging binding. type: - "null" - string - messages: + proxy_address: + description: The proxy address for the messaging binding. type: - "null" - string - notifications: + type: + description: The type of the messaging binding (e.g., SMS, email). type: - "null" - string - outgoing_caller_ids: + type: + - "null" + - object + role_sid: + description: The SID of the role assigned to the participant in the conversation. + type: + - "null" + - string + sid: + description: The unique SID of the conversation participant. + type: + - "null" + - string + url: + description: The URL for accessing more details about the conversation participant. + type: + - "null" + - string + required: + - sid + Conversation Messages: + type: object + $schema: http://json-schema.org/schema# + additionalProperties: true + properties: + account_sid: + description: + The unique identifier of the account associated with the conversation_messages + data. + type: + - "null" + - string + attributes: + description: Additional attributes or metadata related to the conversation message. + type: + - "null" + - string + author: + description: The author or sender of the conversation message. + type: + - "null" + - string + body: + description: The textual content of the conversation message. + type: + - "null" + - string + content_sid: + description: + The unique identifier of the content associated with the conversation + message. + type: + - "null" + - string + conversation_sid: + description: The unique identifier of the conversation to which the message belongs. + type: + - "null" + - string + date_created: + description: The date and time when the message was created. + type: + - "null" + - string + date_updated: + description: The date and time when the message was last updated. + type: + - "null" + - string + delivery: + additionalProperties: true + description: Information related to message delivery + properties: + delivered: + description: Whether the message was successfully delivered. type: - "null" - string - queues: + failed: + description: Number of failed delivery attempts. type: - "null" - string - recordings: + read: + description: Whether the message has been read by the recipient. type: - "null" - string - short_codes: + sent: + description: Number of successful delivery attempts. type: - "null" - string - signing_keys: + total: + description: Total number of delivery attempts. type: - "null" - - string - sip: + - integer + undelivered: + description: Number of undelivered message attempts. type: - "null" - string - transcriptions: + type: + - "null" + - object + index: + description: The index of the message within the conversation. + type: + - "null" + - integer + links: + description: Links related to the conversation + properties: + channel_metadata: + description: Metadata specific to the communication channel. type: - "null" - string - usage: + delivery_receipts: + description: Receipts and status of message delivery. type: - "null" - string type: - "null" - object - type: - description: The type of the account + media: + description: List of media items attached to the message + items: + additionalProperties: true + properties: + content_type: + description: The type of media content attached to the message. + type: + - "null" + - string + filename: + description: The name of the media file. + type: + - "null" + - string + sid: + description: The unique identifier of the media content. + type: + - "null" + - string + size: + description: The size of the media content in bytes. + type: + - "null" + - integer + type: + - "null" + - object + type: + - "null" + - array + participant_sid: + description: The unique identifier of the participant associated with the message. type: - "null" - string - uri: - description: The URI for accessing the account resource + sid: + description: The unique identifier of the conversation_message. type: - "null" - string - required: - - sid - Addresses: + url: + description: The URL of the conversation message for accessing or viewing. + type: + - "null" + - string + required: + - sid + Users: type: object $schema: http://json-schema.org/schema# additionalProperties: true properties: account_sid: - description: The unique identifier of the account associated with this address. + description: The unique identifier for the user's account type: - "null" - string - city: - description: The city of the address. + attributes: + description: Additional attributes or metadata associated with the user type: - "null" - string - customer_name: - description: The name of the customer associated with this address. + chat_service_sid: + description: + The unique identifier for the chat service the user is associated + with type: - "null" - string date_created: - description: The date and time when the address was created. - format: date-time + description: The date and time when the user was created type: - "null" - string date_updated: - description: The date and time when the address was last updated. - format: date-time + description: The date and time when the user was last updated type: - "null" - string - emergency_enabled: - description: Indicates whether emergency services are enabled for this address. + friendly_name: + description: A user-friendly name for the user type: - "null" - - boolean - friendly_name: - description: A friendly name or label for the address. + - string + identity: + description: The identity or username of the user type: - "null" - string - iso_country: - description: The ISO 3166-1 alpha-2 country code of the address. + is_notifiable: + description: Indicates whether the user is set to receive notifications type: - "null" - string - postal_code: - description: The postal code of the address. + is_online: + description: Indicates whether the user is currently online type: - "null" - string - region: - description: The region or state of the address. + links: + description: Contains URLs to related resources for the user data. + properties: + user_conversations: + description: URL to retrieve the conversations associated with the user + type: + - "null" + - string + type: + - "null" + - object + role_sid: + description: The unique identifier for the role assigned to the user type: - "null" - string sid: - description: The unique identifier of the address. + description: The unique identifier for the user type: - "null" - string - street: - description: The street address. + url: + description: URL to access the user's detailed information type: - "null" - string - street_secondary: - description: Additional information about the street address, such as suite number. + required: + - sid + User Conversation: + type: object + $schema: http://json-schema.org/schema# + additionalProperties: true + properties: + account_sid: + description: + The unique identifier of the Twilio account the conversation belongs + to. type: - "null" - string - uri: - description: The URI of the address resource. + attributes: + description: Additional metadata or attributes associated with the conversation. type: - "null" - string - validated: - description: Indicates whether the address has been validated. + chat_service_sid: + description: The unique identifier of the chat service associated with the conversation. type: - "null" - - boolean - verified: - description: Indicates whether the address has been verified. + - string + conversation_sid: + description: The unique identifier of the conversation. type: - "null" - - boolean - required: - - sid + - string + conversation_state: + description: The current state or status of the conversation. + type: + - "null" + - string + created_by: + description: The user or entity that created the conversation. + type: + - "null" + - string + date_created: + description: The date and time when the conversation was initially created. + type: + - "null" + - string + date_updated: + description: The date and time when the conversation was last updated. + type: + - "null" + - string + friendly_name: + description: A user-friendly name or alias for the conversation. + type: + - "null" + - string + last_read_message_index: + description: + The index of the last message in the conversation that has been read + by the user. + type: + - "null" + - integer + links: + description: Contains links related to the user conversation data. + properties: + conversation: + description: Link to access detailed information about the conversation. + type: + - "null" + - string + participant: + description: Link to access participant information related to the conversation. + type: + - "null" + - string + type: + - "null" + - object + notification_level: + description: The level of notification settings for the conversation. + type: + - "null" + - string + participant_sid: + description: The unique identifier of the participant in the conversation. + type: + - "null" + - string + timers: + description: Contains timers or time-related information for the user conversation. + properties: + chat_service_sid: + description: Additional timer related to the chat service. + type: + - "null" + - string + type: + - "null" + - object + unique_name: + description: A unique name assigned to the conversation. + type: + - "null" + - string + unread_messages_count: + description: The count of unread messages in the conversation. + type: + - "null" + - integer + url: + description: The URL to access the conversation details. + type: + - "null" + - string + user_sid: + description: The unique identifier of the user associated with the conversation. + type: + - "null" + - string + required: + - conversations diff --git a/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/alerts.json b/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/alerts.json deleted file mode 100644 index 522fbc5f8cf6..000000000000 --- a/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/alerts.json +++ /dev/null @@ -1,69 +0,0 @@ -{ - "properties": { - "log_level": { - "description": "The severity level of the alert (info, warning, error)", - "type": ["null", "string"] - }, - "resource_sid": { - "description": "The SID of the specific resource associated with the alert", - "type": ["null", "string"] - }, - "date_updated": { - "description": "The date and time when the alert was last updated", - "format": "date-time", - "type": ["null", "string"] - }, - "account_sid": { - "description": "The unique identifier for the account associated with the alert", - "type": ["null", "string"] - }, - "url": { - "description": "The URL or endpoint related to the alert", - "type": ["null", "string"] - }, - "request_method": { - "description": "The method used in the HTTP request that triggered the alert", - "type": ["null", "string"] - }, - "date_generated": { - "description": "The date and time when the alert data was generated", - "format": "date-time", - "type": ["null", "string"] - }, - "alert_text": { - "description": "The actual content or text of the alert message", - "type": ["null", "string"] - }, - "sid": { - "description": "The unique identifier (SID) for the alert entry", - "type": ["null", "string"] - }, - "date_created": { - "description": "The date and time when the alert was created", - "format": "date-time", - "type": ["null", "string"] - }, - "request_url": { - "description": "The URL that was requested and triggered the alert", - "type": ["null", "string"] - }, - "service_sid": { - "description": "The SID of the Twilio service related to the alert", - "type": ["null", "string"] - }, - "error_code": { - "description": "The code representing the error, if an error occurred", - "type": ["null", "string"] - }, - "api_version": { - "description": "The version of the Twilio API used for the alert", - "type": ["null", "string"] - }, - "more_info": { - "description": "Additional information or details related to the alert", - "type": ["null", "string"] - } - }, - "type": "object", - "additionalProperties": true -} diff --git a/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/applications.json b/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/applications.json deleted file mode 100644 index cbed581ce931..000000000000 --- a/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/applications.json +++ /dev/null @@ -1,92 +0,0 @@ -{ - "properties": { - "account_sid": { - "description": "The unique identifier of the Twilio account associated with the application.", - "type": ["null", "string"] - }, - "api_version": { - "description": "The version of the Twilio API being used for the application.", - "type": ["null", "string"] - }, - "date_created": { - "description": "The date and time when the application was created.", - "format": "date-time", - "type": ["null", "string"] - }, - "date_updated": { - "description": "The date and time when the application was last updated.", - "format": "date-time", - "type": ["null", "string"] - }, - "friendly_name": { - "description": "A user-friendly name assigned to the application for identification.", - "type": ["null", "string"] - }, - "message_status_callback": { - "description": "The URL where status updates for outgoing messages will be sent.", - "type": ["null", "string"] - }, - "sid": { - "description": "A unique identifier for the application.", - "type": ["null", "string"] - }, - "sms_fallback_method": { - "description": "The HTTP method used to send a fallback URL if an error occurs during SMS delivery.", - "type": ["null", "string"] - }, - "sms_fallback_url": { - "description": "The URL that Twilio will request if an error occurs during SMS delivery.", - "type": ["null", "string"] - }, - "sms_method": { - "description": "The HTTP method used to send SMS messages for the application.", - "type": ["null", "string"] - }, - "public_application_connect_enabled": { - "description": "Indicates whether public application connect is enabled for the application.", - "type": ["null", "boolean"] - }, - "sms_status_callback": { - "description": "The URL where status updates for incoming SMS messages will be sent.", - "type": ["null", "string"] - }, - "sms_url": { - "description": "The URL that Twilio will request when receiving an incoming SMS message.", - "type": ["null", "string"] - }, - "status_callback": { - "description": "The URL where status updates for calls will be sent.", - "type": ["null", "string"] - }, - "status_callback_method": { - "description": "The HTTP method used to send status updates for calls.", - "type": ["null", "string"] - }, - "uri": { - "description": "The URI of the application resource.", - "type": ["null", "string"] - }, - "voice_caller_id_lookup": { - "description": "Indicates whether or not caller ID information will be looked up for incoming calls.", - "type": ["null", "boolean"] - }, - "voice_fallback_method": { - "description": "The HTTP method used to send a fallback URL if an error occurs during voice call handling.", - "type": ["null", "string"] - }, - "voice_fallback_url": { - "description": "The URL that Twilio will request if an error occurs during voice call handling.", - "type": ["null", "string"] - }, - "voice_method": { - "description": "The HTTP method used for voice call handling.", - "type": ["null", "string"] - }, - "voice_url": { - "description": "The URL that Twilio will request for voice call handling.", - "type": ["null", "string"] - } - }, - "type": "object", - "additionalProperties": true -} diff --git a/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/available_phone_number_countries.json b/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/available_phone_number_countries.json deleted file mode 100644 index d9c8d2bcf787..000000000000 --- a/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/available_phone_number_countries.json +++ /dev/null @@ -1,41 +0,0 @@ -{ - "properties": { - "country_code": { - "description": "The country code associated with the country.", - "type": ["null", "string"] - }, - "country": { - "description": "The name of the country for which phone number data is provided.", - "type": ["null", "string"] - }, - "uri": { - "description": "The URI to fetch more details about phone numbers in the country.", - "type": ["null", "string"] - }, - "beta": { - "description": "Whether the phone number country data is in beta phase or not.", - "type": ["null", "boolean"] - }, - "subresource_uris": { - "description": "URIs for accessing different types of phone numbers in the country.", - "type": ["null", "object"], - "additionalProperties": true, - "properties": { - "local": { - "description": "URI for fetching local phone numbers in the country.", - "type": ["null", "string"] - }, - "mobile": { - "description": "URI for fetching mobile phone numbers in the country.", - "type": ["null", "string"] - }, - "toll_free": { - "description": "URI for fetching toll-free phone numbers in the country.", - "type": ["null", "string"] - } - } - } - }, - "type": "object", - "additionalProperties": true -} diff --git a/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/available_phone_numbers_local.json b/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/available_phone_numbers_local.json deleted file mode 100644 index 2d27185690af..000000000000 --- a/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/available_phone_numbers_local.json +++ /dev/null @@ -1,77 +0,0 @@ -{ - "properties": { - "friendly_name": { - "description": "A user-friendly nickname for the phone number", - "type": ["null", "string"] - }, - "phone_number": { - "description": "The phone number itself", - "type": ["null", "string"] - }, - "lata": { - "description": "Local access and transport area code for the phone number", - "type": ["null", "string"] - }, - "rate_center": { - "description": "Rate center for the phone number", - "type": ["null", "string"] - }, - "latitude": { - "description": "Latitude coordinate of the phone number's location", - "type": ["null", "string"] - }, - "longitude": { - "description": "Longitude coordinate of the phone number's location", - "type": ["null", "string"] - }, - "locality": { - "description": "Locality or city associated with the phone number", - "type": ["null", "string"] - }, - "region": { - "description": "Region or state associated with the phone number", - "type": ["null", "string"] - }, - "postal_code": { - "description": "Postal code of the phone number's location", - "type": ["null", "string"] - }, - "iso_country": { - "description": "ISO country code of the phone number", - "type": ["null", "string"] - }, - "address_requirements": { - "description": "The requirements related to the address associated with the phone number", - "type": ["null", "string"] - }, - "beta": { - "description": "Indicator showing if the phone number is in beta testing", - "type": ["null", "boolean"] - }, - "capabilities": { - "description": "Object containing the capabilities of the available phone number", - "properties": { - "voice": { - "description": "Capability for Voice calls", - "type": ["null", "boolean"] - }, - "SMS": { - "description": "Capability for Short Message Service (Texting)", - "type": ["null", "boolean"] - }, - "MMS": { - "description": "Capability for Multimedia Messaging Service", - "type": ["null", "boolean"] - }, - "fax": { - "description": "Capability for Facsimile (Faxing)", - "type": ["null", "boolean"] - } - }, - "type": ["null", "object"], - "additionalProperties": true - } - }, - "type": "object", - "additionalProperties": true -} diff --git a/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/available_phone_numbers_mobile.json b/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/available_phone_numbers_mobile.json deleted file mode 100644 index 6cb70bf86939..000000000000 --- a/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/available_phone_numbers_mobile.json +++ /dev/null @@ -1,77 +0,0 @@ -{ - "properties": { - "friendly_name": { - "description": "A user-friendly name for the phone number", - "type": ["null", "string"] - }, - "phone_number": { - "description": "The phone number in E.164 format", - "type": ["null", "string"] - }, - "lata": { - "description": "Local access and transport area code", - "type": ["null", "string"] - }, - "rate_center": { - "description": "Rate center associated with the phone number", - "type": ["null", "string"] - }, - "latitude": { - "description": "Latitude coordinate of the phone number's location", - "type": ["null", "string"] - }, - "longitude": { - "description": "Longitude coordinate of the phone number's location", - "type": ["null", "string"] - }, - "locality": { - "description": "Locality or city where the phone number is based", - "type": ["null", "string"] - }, - "region": { - "description": "Region or state where the phone number is located", - "type": ["null", "string"] - }, - "postal_code": { - "description": "Postal code of the phone number's area", - "type": ["null", "string"] - }, - "iso_country": { - "description": "The ISO country code for the phone number", - "type": ["null", "string"] - }, - "address_requirements": { - "description": "Address requirements for purchasing the phone number", - "type": ["null", "string"] - }, - "beta": { - "description": "Indicates if the phone number is in beta testing phase", - "type": ["null", "boolean"] - }, - "capabilities": { - "description": "Represents the capabilities of the available mobile phone numbers such as SMS and voice services", - "properties": { - "voice": { - "description": "Indicates if voice calling capability is available", - "type": ["null", "boolean"] - }, - "SMS": { - "description": "Indicates if SMS messaging is supported", - "type": ["null", "boolean"] - }, - "MMS": { - "description": "Indicates if MMS messaging is supported", - "type": ["null", "boolean"] - }, - "fax": { - "description": "Indicates if fax capability is available", - "type": ["null", "boolean"] - } - }, - "type": ["null", "object"], - "additionalProperties": true - } - }, - "type": "object", - "additionalProperties": true -} diff --git a/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/available_phone_numbers_toll_free.json b/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/available_phone_numbers_toll_free.json deleted file mode 100644 index 7b4ac02ad8a2..000000000000 --- a/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/available_phone_numbers_toll_free.json +++ /dev/null @@ -1,77 +0,0 @@ -{ - "properties": { - "friendly_name": { - "description": "A user-friendly name associated with the phone number for easy identification.", - "type": ["null", "string"] - }, - "phone_number": { - "description": "The toll-free phone number that can be used for communication.", - "type": ["null", "string"] - }, - "lata": { - "description": "The Local Access and Transport Area code of the phone number's location.", - "type": ["null", "string"] - }, - "rate_center": { - "description": "The rate center associated with the phone number.", - "type": ["null", "string"] - }, - "latitude": { - "description": "The latitude coordinate of the phone number's location.", - "type": ["null", "string"] - }, - "longitude": { - "description": "The longitude coordinate of the phone number's location.", - "type": ["null", "string"] - }, - "locality": { - "description": "The locality (city/town) where the phone number is located.", - "type": ["null", "string"] - }, - "region": { - "description": "The region (state/province) where the phone number is located.", - "type": ["null", "string"] - }, - "postal_code": { - "description": "The postal code of the phone number's location.", - "type": ["null", "string"] - }, - "iso_country": { - "description": "The ISO country code of the phone number's location.", - "type": ["null", "string"] - }, - "address_requirements": { - "description": "Specifies the requirements related to the address associated with the phone number.", - "type": ["null", "string"] - }, - "beta": { - "description": "Indicates if the phone number is in beta testing phase.", - "type": ["null", "boolean"] - }, - "capabilities": { - "description": "Represents the capabilities of the toll-free phone numbers available", - "properties": { - "voice": { - "description": "Indicates if the phone number supports voice calls.", - "type": ["null", "boolean"] - }, - "SMS": { - "description": "Indicates if the phone number supports Short Message Service (text messaging).", - "type": ["null", "boolean"] - }, - "MMS": { - "description": "Indicates if the phone number supports Multimedia Messaging Service.", - "type": ["null", "boolean"] - }, - "fax": { - "description": "Indicates if the phone number supports fax communication.", - "type": ["null", "boolean"] - } - }, - "type": ["null", "object"], - "additionalProperties": true - } - }, - "type": "object", - "additionalProperties": true -} diff --git a/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/calls.json b/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/calls.json deleted file mode 100644 index 8ed1e3f0d94b..000000000000 --- a/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/calls.json +++ /dev/null @@ -1,161 +0,0 @@ -{ - "properties": { - "sid": { - "description": "The unique identifier for the call.", - "type": ["null", "string"] - }, - "date_created": { - "description": "The date and time when the call record was created.", - "format": "date-time", - "type": ["null", "string"] - }, - "date_updated": { - "description": "The date and time when the call record was last updated.", - "format": "date-time", - "type": ["null", "string"] - }, - "parent_call_sid": { - "description": "The SID of the parent call if this call is part of a conference.", - "type": ["null", "string"] - }, - "account_sid": { - "description": "The unique identifier for the account associated with the call.", - "type": ["null", "string"] - }, - "to": { - "description": "The phone number that received the call.", - "type": ["null", "string"] - }, - "to_formatted": { - "description": "The formatted version of the 'to' phone number.", - "type": ["null", "string"] - }, - "from": { - "description": "The phone number that made the call.", - "type": ["null", "string"] - }, - "from_formatted": { - "description": "The formatted version of the 'from' phone number.", - "type": ["null", "string"] - }, - "phone_number_sid": { - "description": "The SID of the phone number used for the call.", - "type": ["null", "string"] - }, - "status": { - "description": "The current status of the call, such as 'completed' or 'in-progress'.", - "type": ["null", "string"] - }, - "start_time": { - "description": "The date and time when the call started.", - "format": "date-time", - "type": ["null", "string"] - }, - "end_time": { - "description": "The date and time when the call ended.", - "format": "date-time", - "type": ["null", "string"] - }, - "duration": { - "description": "The duration of the call in seconds.", - "type": ["null", "integer"] - }, - "price": { - "description": "The cost of the call.", - "type": ["null", "number"] - }, - "price_unit": { - "description": "The currency unit of the call cost.", - "type": ["null", "string"] - }, - "direction": { - "description": "The direction of the call, either 'inbound' or 'outbound'.", - "type": ["null", "string"] - }, - "answered_by": { - "description": "The name of the entity or system that answered the call.", - "type": ["null", "string"] - }, - "annotation": { - "description": "Any additional notes or context added to the call.", - "type": ["null", "string"] - }, - "api_version": { - "description": "The version of the Twilio API used for this call.", - "type": ["null", "string"] - }, - "forwarded_from": { - "description": "The phone number that initiated the call forwarding.", - "type": ["null", "string"] - }, - "group_sid": { - "description": "The unique identifier of the call group to which this call belongs.", - "type": ["null", "string"] - }, - "caller_name": { - "description": "The name of the caller as supplied by the caller ID service.", - "type": ["null", "string"] - }, - "queue_time": { - "description": "The time the call spent in a queue before being answered.", - "type": ["null", "integer"] - }, - "trunk_sid": { - "description": "The unique identifier of the trunk used for the call.", - "type": ["null", "string"] - }, - "uri": { - "description": "The URI for this call record.", - "type": ["null", "string"] - }, - "subresource_uris": { - "description": "Contains URIs for related subresources such as recordings or notifications.", - "type": ["null", "object"], - "additionalProperties": true, - "properties": { - "recordings": { - "description": "URI for call recordings related to this call.", - "type": ["null", "string"] - }, - "notifications": { - "description": "URI for call notifications related to this call.", - "type": ["null", "string"] - }, - "feedback": { - "description": "URI for call feedback related to this call.", - "type": ["null", "string"] - }, - "payments": { - "description": "URI for call payments related to this call.", - "type": ["null", "string"] - }, - "siprec": { - "description": "URI for call SIP recording related to this call.", - "type": ["null", "string"] - }, - "events": { - "description": "URI for call events related to this call.", - "type": ["null", "string"] - }, - "feedback_summaries": { - "description": "URI for call feedback summaries related to this call.", - "type": ["null", "string"] - }, - "streams": { - "description": "URI for call streams related to this call.", - "type": ["null", "string"] - }, - "user_defined_message_subscriptions": { - "description": "URI for user-defined message subscriptions related to this call.", - "type": ["null", "string"] - }, - "user_defined_messages": { - "description": "URI for user-defined messages related to this call.", - "type": ["null", "string"] - } - } - } - }, - "type": "object", - "additionalProperties": true -} diff --git a/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/conference_participants.json b/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/conference_participants.json deleted file mode 100644 index 93994995e243..000000000000 --- a/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/conference_participants.json +++ /dev/null @@ -1,64 +0,0 @@ -{ - "properties": { - "account_sid": { - "description": "The unique identifier for the Twilio account associated with the conference participant.", - "type": ["null", "string"] - }, - "call_sid": { - "description": "The unique identifier for the call made by the participant.", - "type": ["null", "string"] - }, - "label": { - "description": "A custom label assigned to the participant for identification.", - "type": ["null", "string"] - }, - "conference_sid": { - "description": "The unique identifier for the conference the participant is a part of.", - "type": ["null", "string"] - }, - "date_created": { - "description": "The date and time when the participant was added to the conference.", - "format": "date-time", - "type": ["null", "string"] - }, - "date_updated": { - "description": "The date and time when the participant's information was last updated.", - "format": "date-time", - "type": ["null", "string"] - }, - "end_conference_on_exit": { - "description": "Indicates whether the conference will end when the participant exits.", - "type": ["null", "boolean"] - }, - "muted": { - "description": "Indicates whether the participant's audio is muted.", - "type": ["null", "boolean"] - }, - "hold": { - "description": "Indicates whether the participant is on hold.", - "type": ["null", "boolean"] - }, - "status": { - "description": "The current status of the participant in the conference.", - "type": ["null", "string"] - }, - "start_conference_on_enter": { - "description": "Indicates whether the conference will start when the participant enters.", - "type": ["null", "boolean"] - }, - "coaching": { - "description": "Indicates whether the participant is in coaching mode.", - "type": ["null", "boolean"] - }, - "call_sid_to_coach": { - "description": "The unique identifier for the call made to the coach.", - "type": ["null", "string"] - }, - "uri": { - "description": "The URI for accessing the details of the conference participant.", - "type": ["null", "string"] - } - }, - "type": "object", - "additionalProperties": true -} diff --git a/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/conferences.json b/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/conferences.json deleted file mode 100644 index 54cad9f7484e..000000000000 --- a/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/conferences.json +++ /dev/null @@ -1,67 +0,0 @@ -{ - "properties": { - "account_sid": { - "description": "The unique identifier of the account associated with the conference.", - "type": ["null", "string"] - }, - "date_created": { - "description": "The date and time when the conference was created.", - "format": "date-time", - "type": ["null", "string"] - }, - "date_updated": { - "description": "The date and time when the conference was last updated.", - "format": "date-time", - "type": ["null", "string"] - }, - "api_version": { - "description": "The version of the Twilio API used for the conference.", - "type": ["null", "string"] - }, - "friendly_name": { - "description": "A user-defined friendly name for the conference.", - "type": ["null", "string"] - }, - "region": { - "description": "The region where the conference is hosted.", - "type": ["null", "string"] - }, - "sid": { - "description": "The unique identifier (SID) of the conference.", - "type": ["null", "string"] - }, - "status": { - "description": "The current status of the conference.", - "type": ["null", "string"] - }, - "uri": { - "description": "The URI to access the conference resource.", - "type": ["null", "string"] - }, - "reason_conference_ended": { - "description": "The reason for the conference ending, if available.", - "type": ["null", "string"] - }, - "call_sid_ending_conference": { - "description": "The SID of the call that ended the conference, if applicable.", - "type": ["null", "string"] - }, - "subresource_uris": { - "description": "Contains URLs for accessing subresources related to conferences", - "type": ["null", "object"], - "additionalProperties": true, - "properties": { - "participants": { - "description": "URI to access the participants associated with the conference.", - "type": ["null", "string"] - }, - "recordings": { - "description": "URI to access the recordings of the conference.", - "type": ["null", "string"] - } - } - } - }, - "type": "object", - "additionalProperties": true -} diff --git a/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/conversation_messages.json b/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/conversation_messages.json deleted file mode 100644 index 99e4519b29ee..000000000000 --- a/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/conversation_messages.json +++ /dev/null @@ -1,126 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "additionalProperties": true, - "properties": { - "sid": { - "description": "The unique identifier of the conversation_message.", - "type": ["null", "string"] - }, - "account_sid": { - "description": "The unique identifier of the account associated with the conversation_messages data.", - "type": ["null", "string"] - }, - "conversation_sid": { - "description": "The unique identifier of the conversation to which the message belongs.", - "type": ["null", "string"] - }, - "body": { - "description": "The textual content of the conversation message.", - "type": ["null", "string"] - }, - "media": { - "description": "List of media items attached to the message", - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "additionalProperties": true, - "properties": { - "sid": { - "description": "The unique identifier of the media content.", - "type": ["null", "string"] - }, - "size": { - "description": "The size of the media content in bytes.", - "type": ["null", "integer"] - }, - "content_type": { - "description": "The type of media content attached to the message.", - "type": ["null", "string"] - }, - "filename": { - "description": "The name of the media file.", - "type": ["null", "string"] - } - } - } - }, - "author": { - "description": "The author or sender of the conversation message.", - "type": ["null", "string"] - }, - "participant_sid": { - "description": "The unique identifier of the participant associated with the message.", - "type": ["null", "string"] - }, - "attributes": { - "description": "Additional attributes or metadata related to the conversation message.", - "type": ["null", "string"] - }, - "date_created": { - "description": "The date and time when the message was created.", - "type": ["null", "string"] - }, - "date_updated": { - "description": "The date and time when the message was last updated.", - "type": ["null", "string"] - }, - "index": { - "description": "The index of the message within the conversation.", - "type": ["null", "integer"] - }, - "delivery": { - "description": "Information related to message delivery", - "type": ["null", "object"], - "additionalProperties": true, - "properties": { - "total": { - "description": "Total number of delivery attempts.", - "type": ["null", "integer"] - }, - "sent": { - "description": "Number of successful delivery attempts.", - "type": ["null", "string"] - }, - "delivered": { - "description": "Whether the message was successfully delivered.", - "type": ["null", "string"] - }, - "read": { - "description": "Whether the message has been read by the recipient.", - "type": ["null", "string"] - }, - "failed": { - "description": "Number of failed delivery attempts.", - "type": ["null", "string"] - }, - "undelivered": { - "description": "Number of undelivered message attempts.", - "type": ["null", "string"] - } - } - }, - "content_sid": { - "description": "The unique identifier of the content associated with the conversation message.", - "type": ["null", "string"] - }, - "url": { - "description": "The URL of the conversation message for accessing or viewing.", - "type": ["null", "string"] - }, - "links": { - "description": "Links related to the conversation", - "type": ["null", "object"], - "properties": { - "delivery_receipts": { - "description": "Receipts and status of message delivery.", - "type": ["null", "string"] - }, - "channel_metadata": { - "description": "Metadata specific to the communication channel.", - "type": ["null", "string"] - } - } - } - } -} diff --git a/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/conversation_participants.json b/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/conversation_participants.json deleted file mode 100644 index 38e78cca9792..000000000000 --- a/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/conversation_participants.json +++ /dev/null @@ -1,71 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "additionalProperties": true, - "properties": { - "account_sid": { - "description": "The Twilio account SID of the conversation participant.", - "type": ["null", "string"] - }, - "conversation_sid": { - "description": "The SID of the conversation the participant belongs to.", - "type": ["null", "string"] - }, - "sid": { - "description": "The unique SID of the conversation participant.", - "type": ["null", "string"] - }, - "identity": { - "description": "The identity of the participant in the conversation.", - "type": ["null", "string"] - }, - "attributes": { - "description": "Additional attributes related to the conversation participant.", - "type": ["null", "string"] - }, - "messaging_binding": { - "description": "Details about the messaging binding for the conversation participant.", - "type": ["null", "object"], - "properties": { - "type": { - "description": "The type of the messaging binding (e.g., SMS, email).", - "type": ["null", "string"] - }, - "address": { - "description": "The address associated with the participant's messaging binding.", - "type": ["null", "string"] - }, - "proxy_address": { - "description": "The proxy address for the messaging binding.", - "type": ["null", "string"] - } - } - }, - "role_sid": { - "description": "The SID of the role assigned to the participant in the conversation.", - "type": ["null", "string"] - }, - "date_created": { - "description": "The date and time when the conversation participant was created.", - "format": "date-time", - "type": ["null", "string"] - }, - "date_updated": { - "description": "The date and time when the conversation participant was last updated.", - "format": "date-time", - "type": ["null", "string"] - }, - "url": { - "description": "The URL for accessing more details about the conversation participant.", - "type": ["null", "string"] - }, - "last_read_message_index": { - "description": "The index of the last message that the participant has read.", - "type": ["null", "string"] - }, - "last_read_timestamp": { - "description": "The timestamp when the participant last read a message.", - "type": ["null", "string"] - } - } -} diff --git a/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/conversations.json b/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/conversations.json deleted file mode 100644 index f9922b1f23a5..000000000000 --- a/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/conversations.json +++ /dev/null @@ -1,60 +0,0 @@ -{ - "properties": { - "sid": { - "description": "The unique identifier for the conversation.", - "type": ["null", "string"] - }, - "account_sid": { - "description": "The unique identifier for the account associated with the conversation.", - "type": ["null", "string"] - }, - "chat_service_sid": { - "description": "The SID (Service Identifier) for the chat service to which the conversation belongs.", - "type": ["null", "string"] - }, - "messaging_service_sid": { - "description": "The SID for the messaging service associated with the conversation.", - "type": ["null", "string"] - }, - "friendly_name": { - "description": "A human-readable name assigned to the conversation.", - "type": ["null", "string"] - }, - "unique_name": { - "description": "A unique name assigned to the conversation for easy identification.", - "type": ["null", "string"] - }, - "attributes": { - "description": "Additional attributes or metadata associated with the conversation.", - "type": ["null", "string"] - }, - "date_created": { - "description": "The date and time when the conversation was created.", - "format": "date-time", - "type": ["null", "string"] - }, - "date_updated": { - "description": "The date and time when the conversation was last updated.", - "format": "date-time", - "type": ["null", "string"] - }, - "state": { - "description": "The current state of the conversation (e.g., active, inactive).", - "type": ["null", "string"] - }, - "timers": { - "description": "Information about timers set for the conversation.", - "type": ["null", "object"] - }, - "bindings": { - "description": "Information about the communication channels bound to the conversation.", - "type": ["null", "object"] - }, - "url": { - "description": "The URL endpoint for accessing or interacting with the conversation data.", - "type": ["null", "string"] - } - }, - "type": "object", - "additionalProperties": true -} diff --git a/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/dependent_phone_numbers.json b/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/dependent_phone_numbers.json deleted file mode 100644 index 98945daa1ef2..000000000000 --- a/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/dependent_phone_numbers.json +++ /dev/null @@ -1,127 +0,0 @@ -{ - "properties": { - "sid": { - "description": "The unique identifier (SID) for the phone number resource.", - "type": ["null", "string"] - }, - "account_sid": { - "description": "The unique identifier for the account this phone number belongs to.", - "type": ["null", "string"] - }, - "friendly_name": { - "description": "A user-friendly name for the phone number.", - "type": ["null", "string"] - }, - "phone_number": { - "description": "The phone number itself.", - "type": ["null", "string"] - }, - "voice_url": { - "description": "The URL to call when a call comes in to the phone number.", - "type": ["null", "string"] - }, - "voice_method": { - "description": "The HTTP method to use for the voice URL.", - "type": ["null", "string"] - }, - "voice_fallback_url": { - "description": "The URL to call if an error occurs while handling incoming voice calls.", - "type": ["null", "string"] - }, - "voice_fallback_method": { - "description": "The HTTP method to use for the voice fallback URL.", - "type": ["null", "string"] - }, - "voice_caller_id_lookup": { - "description": "Indicates whether the phone number performs caller ID lookup on incoming calls.", - "type": ["null", "boolean"] - }, - "date_created": { - "description": "The date and time when the phone number was created.", - "format": "date-time", - "type": ["null", "string"] - }, - "date_updated": { - "description": "The date and time when the phone number was last updated.", - "format": "date-time", - "type": ["null", "string"] - }, - "sms_url": { - "description": "The URL to call when a message comes in to the phone number.", - "type": ["null", "string"] - }, - "sms_method": { - "description": "The HTTP method to use for the SMS URL.", - "type": ["null", "string"] - }, - "sms_fallback_url": { - "description": "The URL to call if an error occurs while handling incoming SMS messages.", - "type": ["null", "string"] - }, - "sms_fallback_method": { - "description": "The HTTP method to use for the SMS fallback URL.", - "type": ["null", "string"] - }, - "address_requirements": { - "description": "Requirements for the address associated with the phone number.", - "type": ["null", "string"] - }, - "capabilities": { - "description": "List of capabilities supported by the phone number.", - "properties": { - "Voice": { - "description": "Support for voice calling.", - "type": ["null", "boolean"] - }, - "SMS": { - "description": "Support for text messaging (SMS).", - "type": ["null", "boolean"] - }, - "MMS": { - "description": "Support for multimedia messaging (MMS).", - "type": ["null", "boolean"] - } - }, - "type": ["null", "object"], - "additionalProperties": true - }, - "status_callback": { - "description": "The URL to send status callbacks to for events related to the phone number.", - "type": ["null", "string"] - }, - "status_callback_method": { - "description": "The HTTP method to use for status callback requests.", - "type": ["null", "string"] - }, - "api_version": { - "description": "The version of the Twilio API being used.", - "type": ["null", "string"] - }, - "voice_application_sid": { - "description": "The SID of the voice application to handle incoming calls.", - "type": ["null", "string"] - }, - "sms_application_sid": { - "description": "The SID of the SMS application to handle incoming messages.", - "type": ["null", "string"] - }, - "trunk_sid": { - "description": "The SID of the Trunk containing this phone number.", - "type": ["null", "string"] - }, - "emergency_status": { - "description": "The emergency status of the phone number.", - "type": ["null", "string"] - }, - "emergency_address_sid": { - "description": "The SID of the emergency address associated with the phone number.", - "type": ["null", "string"] - }, - "uri": { - "description": "The URI of the phone number resource.", - "type": ["null", "string"] - } - }, - "type": "object", - "additionalProperties": true -} diff --git a/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/executions.json b/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/executions.json deleted file mode 100644 index 541a69857a54..000000000000 --- a/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/executions.json +++ /dev/null @@ -1,66 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Executions", - "type": "object", - "additionalProperties": true, - "properties": { - "sid": { - "description": "The unique identifier for the execution.", - "type": ["null", "string"] - }, - "account_sid": { - "description": "The unique identifier for the account associated with the execution.", - "type": ["null", "string"] - }, - "flow_sid": { - "description": "The unique identifier for the flow associated with the execution.", - "type": ["null", "string"] - }, - "contact_sid": { - "description": "The unique identifier for the contact associated with the execution.", - "type": ["null", "string"] - }, - "contact_channel_address": { - "description": "The contact channel address where the execution is targeted.", - "type": ["null", "string"] - }, - "status": { - "description": "The status of the execution process.", - "type": ["null", "string"] - }, - "context": { - "description": "Additional context or data related to the execution.", - "type": ["null", "object"], - "additionalProperties": true - }, - "date_created": { - "description": "The date and time when the execution was created.", - "type": ["null", "string"], - "format": "date-time" - }, - "date_updated": { - "description": "The date and time when the execution was last updated.", - "type": ["null", "string"], - "format": "date-time" - }, - "url": { - "description": "The URL endpoint for accessing detailed information about the execution.", - "type": ["null", "string"] - }, - "links": { - "description": "Contains related hypermedia links associated with the executions data.", - "type": ["null", "object"], - "additionalProperties": true, - "properties": { - "execution_context": { - "description": "Link to the execution context data.", - "type": ["null", "string"] - }, - "steps": { - "description": "Link to the steps involved in the execution.", - "type": ["null", "string"] - } - } - } - } -} diff --git a/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/flows.json b/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/flows.json deleted file mode 100644 index 78a7574fef7b..000000000000 --- a/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/flows.json +++ /dev/null @@ -1,53 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "additionalProperties": true, - "properties": { - "sid": { - "description": "Unique identifier for the flow.", - "type": ["null", "string"] - }, - "account_sid": { - "description": "Unique identifier for the account associated with the flow.", - "type": ["null", "string"] - }, - "date_created": { - "description": "Date and time when the flow was created.", - "format": "date-time", - "type": ["null", "string"] - }, - "date_updated": { - "description": "Date and time when the flow was last updated.", - "format": "date-time", - "type": ["null", "string"] - }, - "friendly_name": { - "description": "A user-friendly name given to the flow.", - "type": ["null", "string"] - }, - "status": { - "description": "Status of the flow (e.g., active, inactive).", - "type": ["string", "null"], - "enum": ["published", "draft"] - }, - "version": { - "description": "Version number of the flow.", - "type": ["null", "integer"] - }, - "url": { - "description": "URL endpoint for the flow.", - "type": ["null", "string"] - }, - "links": { - "description": "Represents links related to the flow data.", - "type": ["null", "object"], - "additionalProperties": true, - "properties": { - "steps": { - "description": "Link to the steps included in the flow.", - "type": ["null", "string"] - } - } - } - } -} diff --git a/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/incoming_phone_numbers.json b/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/incoming_phone_numbers.json deleted file mode 100644 index 31ba09500b80..000000000000 --- a/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/incoming_phone_numbers.json +++ /dev/null @@ -1,174 +0,0 @@ -{ - "properties": { - "sid": { - "description": "The SID of this phone number.", - "type": ["null", "string"] - }, - "account_sid": { - "description": "The SID of the account that owns this phone number.", - "type": ["null", "string"] - }, - "friendly_name": { - "description": "A user-assigned friendly name for this phone number.", - "type": ["null", "string"] - }, - "phone_number": { - "description": "The phone number.", - "type": ["null", "string"] - }, - "voice_url": { - "description": "URL to make requests to when an incoming voice call is received.", - "type": ["null", "string"] - }, - "voice_method": { - "description": "HTTP method to use when making requests to the voice_url.", - "type": ["null", "string"] - }, - "voice_fallback_url": { - "description": "URL to make requests to when an error occurs while processing an incoming voice call.", - "type": ["null", "string"] - }, - "voice_fallback_method": { - "description": "HTTP method to use when making requests to the voice_fallback_url.", - "type": ["null", "string"] - }, - "voice_caller_id_lookup": { - "description": "Caller ID lookup setting for voice calls made from this phone number.", - "type": ["null", "boolean"] - }, - "date_created": { - "description": "The date and time when this phone number was created.", - "format": "date-time", - "type": ["null", "string"] - }, - "date_updated": { - "description": "The date and time when this phone number was last updated.", - "format": "date-time", - "type": ["null", "string"] - }, - "sms_url": { - "description": "URL to make requests to when an incoming SMS message is received.", - "type": ["null", "string"] - }, - "sms_method": { - "description": "HTTP method to use when making requests to the sms_url.", - "type": ["null", "string"] - }, - "sms_fallback_url": { - "description": "URL to make requests to when an error occurs while processing an incoming SMS message.", - "type": ["null", "string"] - }, - "sms_fallback_method": { - "description": "HTTP method to use when making requests to the sms_fallback_url.", - "type": ["null", "string"] - }, - "address_requirements": { - "description": "Indicates the address requirements for this phone number.", - "type": ["null", "string"] - }, - "beta": { - "description": "Flag indicating beta status of this phone number.", - "type": ["null", "boolean"] - }, - "capabilities": { - "description": "Capabilities of this phone number.", - "properties": { - "voice": { - "description": "Capability for making and receiving voice calls.", - "type": ["null", "boolean"] - }, - "sms": { - "description": "Capability for sending and receiving SMS messages.", - "type": ["null", "boolean"] - }, - "mms": { - "description": "Capability for sending and receiving MMS messages.", - "type": ["null", "boolean"] - }, - "fax": { - "description": "Capability for faxing.", - "type": ["null", "boolean"] - } - }, - "type": ["null", "object"], - "additionalProperties": true - }, - "voice_receive_mode": { - "description": "Receive mode setting for this phone number.", - "type": ["null", "string"] - }, - "status_callback": { - "description": "URL to make requests to when certain events happen related to this phone number.", - "type": ["null", "string"] - }, - "status_callback_method": { - "description": "HTTP method to use when making requests to the status_callback URL.", - "type": ["null", "string"] - }, - "api_version": { - "description": "The version of the Twilio API being used for this phone number.", - "type": ["null", "string"] - }, - "voice_application_sid": { - "description": "The SID of the application to handle incoming voice calls for this phone number.", - "type": ["null", "string"] - }, - "sms_application_sid": { - "description": "The SID of the application to handle incoming SMS messages for this phone number.", - "type": ["null", "string"] - }, - "origin": { - "description": "Indicates the origin of this phone number.", - "type": ["null", "string"] - }, - "trunk_sid": { - "description": "The SID of the trunk associated with this phone number.", - "type": ["null", "string"] - }, - "emergency_status": { - "description": "Emergency status of this phone number.", - "type": ["null", "string"] - }, - "emergency_address_sid": { - "description": "The SID of the emergency address associated with this phone number.", - "type": ["null", "string"] - }, - "emergency_address_status": { - "description": "Status of the emergency address associated with this phone number.", - "type": ["null", "string"] - }, - "address_sid": { - "description": "The SID of the address associated with this phone number.", - "type": ["null", "string"] - }, - "identity_sid": { - "description": "The SID of the identity associated with this phone number.", - "type": ["null", "string"] - }, - "bundle_sid": { - "description": "The SID of the bundle associated with this phone number.", - "type": ["null", "string"] - }, - "uri": { - "description": "The URI of this phone number.", - "type": ["null", "string"] - }, - "status": { - "description": "Status of this phone number.", - "type": ["null", "string"] - }, - "subresource_uris": { - "description": "URIs for related sub-resources.", - "properties": { - "assigned_add_ons": { - "description": "URI for accessing assigned add-ons for this phone number.", - "type": ["null", "string"] - } - }, - "type": ["null", "object"], - "additionalProperties": true - } - }, - "type": "object", - "additionalProperties": true -} diff --git a/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/keys.json b/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/keys.json deleted file mode 100644 index 90acbba783b6..000000000000 --- a/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/keys.json +++ /dev/null @@ -1,24 +0,0 @@ -{ - "properties": { - "date_created": { - "description": "The date and time when the key was created.", - "format": "date-time", - "type": ["null", "string"] - }, - "date_updated": { - "description": "The date and time when the key was last updated.", - "format": "date-time", - "type": ["null", "string"] - }, - "friendly_name": { - "description": "The user-friendly name associated with the key.", - "type": ["null", "string"] - }, - "sid": { - "description": "The unique identifier for the key (System ID).", - "type": ["null", "string"] - } - }, - "type": "object", - "additionalProperties": true -} diff --git a/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/message_media.json b/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/message_media.json deleted file mode 100644 index 01a6107a32e2..000000000000 --- a/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/message_media.json +++ /dev/null @@ -1,36 +0,0 @@ -{ - "properties": { - "sid": { - "description": "The unique identifier of the message media.", - "type": ["null", "string"] - }, - "account_sid": { - "description": "The unique identifier of the account associated with this message media.", - "type": ["null", "string"] - }, - "parent_sid": { - "description": "The unique identifier of the parent message associated with this media.", - "type": ["null", "string"] - }, - "content_type": { - "description": "The MIME type of the media content.", - "type": ["null", "string"] - }, - "date_created": { - "description": "The date and time when the message media was created.", - "format": "date-time", - "type": ["null", "string"] - }, - "date_updated": { - "description": "The date and time when the message media was last updated.", - "format": "date-time", - "type": ["null", "string"] - }, - "uri": { - "description": "The URI that can be used to access this message media resource.", - "type": ["null", "string"] - } - }, - "type": "object", - "additionalProperties": true -} diff --git a/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/messages.json b/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/messages.json deleted file mode 100644 index 6624d0dab560..000000000000 --- a/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/messages.json +++ /dev/null @@ -1,100 +0,0 @@ -{ - "properties": { - "account_sid": { - "description": "The unique identifier for the account associated with this message.", - "type": ["null", "string"] - }, - "api_version": { - "description": "The version of the Twilio API being used to send or receive this message.", - "type": ["null", "string"] - }, - "body": { - "description": "The text body of the message.", - "type": ["null", "string"] - }, - "date_created": { - "description": "The date and time when the message was created.", - "format": "date-time", - "type": ["null", "string"] - }, - "date_sent": { - "description": "The date and time when the message was sent.", - "format": "date-time", - "type": ["null", "string"] - }, - "date_updated": { - "description": "The date and time when the message was last updated.", - "format": "date-time", - "type": ["null", "string"] - }, - "direction": { - "description": "The direction of the message, whether it's incoming or outgoing.", - "type": ["null", "string"] - }, - "error_code": { - "description": "The error code associated with the message, if any.", - "type": ["null", "string"] - }, - "error_message": { - "description": "The error message description if the message failed to send.", - "type": ["null", "string"] - }, - "from": { - "description": "The phone number or sender ID that the message originated from.", - "type": ["null", "string"] - }, - "messaging_service_sid": { - "description": "The unique identifier for the messaging service associated with the message.", - "type": ["null", "string"] - }, - "price_unit": { - "description": "The currency unit used for pricing of the message.", - "type": ["null", "string"] - }, - "num_media": { - "description": "The number of media files (e.g., images, videos) included in the message.", - "type": ["null", "integer"] - }, - "num_segments": { - "description": "The number of message segments if the message exceeds 160 characters.", - "type": ["null", "integer"] - }, - "price": { - "description": "The cost of sending the message.", - "type": ["null", "number"] - }, - "sid": { - "description": "The unique identifier for this message.", - "type": ["null", "string"] - }, - "status": { - "description": "The status of the message (e.g., sent, delivered, failed).", - "type": ["null", "string"] - }, - "subresource_uris": { - "description": "Contains links to the subresources related to the message.", - "type": ["null", "object"], - "additionalProperties": true, - "properties": { - "feedback": { - "description": "URI to access feedback related to the message.", - "type": ["null", "string"] - }, - "media": { - "description": "URI to access media files associated with the message.", - "type": ["null", "string"] - } - } - }, - "to": { - "description": "The phone number or recipient ID that the message was sent to.", - "type": ["null", "string"] - }, - "uri": { - "description": "The URI for accessing detailed information about this message.", - "type": ["null", "string"] - } - }, - "type": "object", - "additionalProperties": true -} diff --git a/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/outgoing_caller_ids.json b/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/outgoing_caller_ids.json deleted file mode 100644 index 6f639108a0e5..000000000000 --- a/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/outgoing_caller_ids.json +++ /dev/null @@ -1,36 +0,0 @@ -{ - "properties": { - "account_sid": { - "description": "The unique identifier for the Twilio account associated with this outgoing caller ID.", - "type": ["null", "string"] - }, - "date_created": { - "description": "The date and time when the outgoing caller ID was created.", - "format": "date-time", - "type": ["null", "string"] - }, - "date_updated": { - "description": "The date and time when the outgoing caller ID was last updated.", - "format": "date-time", - "type": ["null", "string"] - }, - "friendly_name": { - "description": "A user-defined friendly name for the outgoing caller ID.", - "type": ["null", "string"] - }, - "phone_number": { - "description": "The phone number associated with the outgoing caller ID.", - "type": ["null", "string"] - }, - "sid": { - "description": "The unique identifier for the outgoing caller ID.", - "type": ["null", "string"] - }, - "uri": { - "description": "The URI for accessing detailed information about the outgoing caller ID.", - "type": ["null", "string"] - } - }, - "type": "object", - "additionalProperties": true -} diff --git a/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/queues.json b/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/queues.json deleted file mode 100644 index 6b5c62d03e61..000000000000 --- a/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/queues.json +++ /dev/null @@ -1,55 +0,0 @@ -{ - "properties": { - "account_sid": { - "description": "The unique identifier of the account that owns this queue.", - "type": ["null", "string"] - }, - "average_wait_time": { - "description": "The average time, in seconds, that callers have spent waiting in this queue.", - "type": ["null", "integer"] - }, - "current_size": { - "description": "The current number of callers waiting in this queue.", - "type": ["null", "integer"] - }, - "date_created": { - "description": "The date and time when this queue was created.", - "format": "date-time", - "type": ["null", "string"] - }, - "date_updated": { - "description": "The date and time when this queue was last updated.", - "format": "date-time", - "type": ["null", "string"] - }, - "friendly_name": { - "description": "A user-defined human-readable name for this queue.", - "type": ["null", "string"] - }, - "max_size": { - "description": "The maximum number of callers allowed in this queue.", - "type": ["null", "integer"] - }, - "sid": { - "description": "A unique identifier for this queue.", - "type": ["null", "string"] - }, - "uri": { - "description": "The URI for this queue resource.", - "type": ["null", "string"] - }, - "subresource_uris": { - "description": "Contains URIs for related subresources of this queue.", - "type": ["null", "object"], - "additionalProperties": true, - "properties": { - "members": { - "description": "A URI to fetch the members (callers) of this queue.", - "type": ["null", "string"] - } - } - } - }, - "type": "object", - "additionalProperties": true -} diff --git a/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/recordings.json b/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/recordings.json deleted file mode 100644 index d4d07676d3f5..000000000000 --- a/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/recordings.json +++ /dev/null @@ -1,115 +0,0 @@ -{ - "properties": { - "account_sid": { - "description": "The unique identifier of the account that owns the recording.", - "type": ["null", "string"] - }, - "api_version": { - "description": "The version of the Twilio API used when the recording was created.", - "type": ["null", "string"] - }, - "call_sid": { - "description": "The unique identifier of the call associated with the recording.", - "type": ["null", "string"] - }, - "conference_sid": { - "description": "The unique identifier of the conference, if the recording is part of a conference.", - "type": ["null", "string"] - }, - "date_created": { - "description": "The date and time when the recording was created.", - "format": "date-time", - "type": ["null", "string"] - }, - "date_updated": { - "description": "The date and time when the recording was last updated.", - "format": "date-time", - "type": ["null", "string"] - }, - "start_time": { - "description": "The date and time when the recording started.", - "format": "date-time", - "type": ["null", "string"] - }, - "duration": { - "description": "The duration of the recording in seconds.", - "type": ["null", "integer"] - }, - "sid": { - "description": "The unique identifier of the recording.", - "type": ["null", "string"] - }, - "price": { - "description": "The cost of storing the recording.", - "type": ["null", "number"] - }, - "price_unit": { - "description": "The unit in which the cost is measured (e.g., USD).", - "type": ["null", "string"] - }, - "status": { - "description": "The status of the recording (e.g., completed, in-progress).", - "type": ["null", "string"] - }, - "channels": { - "description": "The number of audio channels in the recording.", - "type": ["null", "integer"] - }, - "source": { - "description": "The source of the recording (e.g., twilio).", - "type": ["null", "string"] - }, - "error_code": { - "description": "The error code, if any, associated with the recording.", - "type": ["null", "integer"] - }, - "media_url": { - "description": "The URL where the recording audio file can be accessed.", - "type": "string" - }, - "uri": { - "description": "The URI of the recording resource.", - "type": ["null", "string"] - }, - "encryption_details": { - "description": "Details about the encryption used for the recording.", - "properties": { - "type": { - "description": "The type of encryption used.", - "type": ["null", "string"] - }, - "encryption_public_key_sid": { - "description": "The SID of the public key used for encryption.", - "type": ["null", "string"] - }, - "encryption_cek": { - "description": "The Content Encryption Key used for encryption.", - "type": ["null", "string"] - }, - "iv": { - "description": "The Initialization Vector used for encryption.", - "type": ["null", "string"] - } - }, - "type": ["null", "object"], - "additionalProperties": true - }, - "subresource_uris": { - "description": "URIs for subresources related to the recording.", - "type": ["null", "object"], - "additionalProperties": true, - "properties": { - "transcriptions": { - "description": "The URI to fetch transcriptions of the recording.", - "type": ["null", "string"] - }, - "add_on_results": { - "description": "The URI to fetch add-on results related to the recording.", - "type": ["null", "string"] - } - } - } - }, - "type": "object", - "additionalProperties": true -} diff --git a/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/roles.json b/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/roles.json deleted file mode 100644 index 16a7ab794005..000000000000 --- a/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/roles.json +++ /dev/null @@ -1,49 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Roles Schema", - "additionalProperties": true, - "type": ["null", "object"], - "properties": { - "sid": { - "description": "The unique identifier for the role.", - "type": ["null", "string"] - }, - "account_sid": { - "description": "The unique identifier for the account associated with the role.", - "type": ["null", "string"] - }, - "service_sid": { - "description": "The unique identifier for the service associated with the role.", - "type": ["null", "string"] - }, - "friendly_name": { - "description": "A user-friendly name assigned to the role.", - "type": ["null", "string"] - }, - "type": { - "description": "The type or category of the role.", - "type": ["null", "string"] - }, - "permissions": { - "description": "The list of permissions assigned to the role.", - "type": ["null", "array"], - "items": { - "type": ["null", "string"] - } - }, - "date_created": { - "description": "The date and time when the role was created.", - "type": ["null", "string"], - "format": "date-time" - }, - "date_updated": { - "description": "The date and time when the role was last updated.", - "type": ["null", "string"], - "format": "date-time" - }, - "url": { - "description": "The URL for accessing the role data.", - "type": ["null", "string"] - } - } -} diff --git a/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/services.json b/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/services.json deleted file mode 100644 index 3eb0e903261d..000000000000 --- a/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/services.json +++ /dev/null @@ -1,149 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Services Schema", - "additionalProperties": true, - "type": ["null", "object"], - "properties": { - "account_sid": { - "description": "The unique identifier of the account related to the service.", - "type": ["null", "string"] - }, - "consumption_report_interval": { - "description": "The interval at which consumption reports are generated.", - "type": ["null", "number"] - }, - "date_created": { - "description": "The date and time when the service was created.", - "type": ["null", "string"], - "format": "date-time" - }, - "date_updated": { - "description": "The date and time when the service was last updated.", - "type": ["null", "string"], - "format": "date-time" - }, - "default_channel_creator_role_sid": { - "description": "The default role assigned to the creator of a channel.", - "type": ["null", "string"] - }, - "default_channel_role_sid": { - "description": "The default role assigned to all users in a channel.", - "type": ["null", "string"] - }, - "default_service_role_sid": { - "description": "The default role assigned to users of the service.", - "type": ["null", "string"] - }, - "friendly_name": { - "description": "A user-friendly name given to the service.", - "type": ["null", "string"] - }, - "limits": { - "description": "Limits applied to the service.", - "type": ["null", "object"], - "properties": { - "channel_members": { - "description": "Maximum number of members allowed in a channel.", - "type": ["null", "number"] - }, - "user_channels": { - "description": "Maximum number of channels a user can belong to.", - "type": ["null", "number"] - } - } - }, - "links": { - "description": "Links to different resources related to the service.", - "type": ["null", "object"], - "properties": { - "channels": { - "description": "Links related to channels.", - "type": ["null", "string"] - }, - "users": { - "description": "Links related to users.", - "type": ["null", "string"] - }, - "roles": { - "description": "Links related to roles.", - "type": ["null", "string"] - }, - "bindings": { - "description": "Links related to bindings.", - "type": ["null", "string"] - } - } - }, - "notifications": { - "description": "Notification settings for users.", - "type": ["null", "object"], - "properties": { - "users": { - "description": "Users to receive notifications.", - "type": ["null", "string"] - } - } - }, - "post_webhook_url": { - "description": "URL for the post webhook.", - "type": ["null", "string"] - }, - "pre_webhook_url": { - "description": "URL for the pre webhook.", - "type": ["null", "string"] - }, - "pre_webhook_retry_count": { - "description": "Number of retries for the pre webhook.", - "type": ["null", "number"] - }, - "post_webhook_retry_count": { - "description": "Number of retries for the post webhook.", - "type": ["null", "number"] - }, - "reachability_enabled": { - "description": "Flag indicating if reachability is enabled.", - "type": ["null", "boolean"] - }, - "read_status_enabled": { - "description": "Flag indicating if read status is enabled.", - "type": ["null", "boolean"] - }, - "sid": { - "description": "The unique identifier of the service.", - "type": ["null", "string"] - }, - "typing_indicator_timeout": { - "description": "Timeout duration for typing indicators.", - "type": ["null", "number"] - }, - "url": { - "description": "URL of the service.", - "type": ["null", "string"] - }, - "webhook_filters": { - "description": "Filters for webhooks.", - "type": ["null", "array"], - "items": { - "type": ["null", "string"] - } - }, - "webhook_method": { - "description": "HTTP method used for webhooks.", - "type": ["null", "string"] - }, - "media": { - "description": "Media settings for the service.", - "type": ["null", "object"], - "properties": { - "size_limit_mb": { - "description": "Maximum size limit for media in megabytes.", - "type": ["null", "number"] - }, - "compatibility_message": { - "description": "Message to display for incompatible media.", - "type": ["null", "string"] - } - } - } - } -} diff --git a/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/step.json b/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/step.json deleted file mode 100644 index 00c7b8dcad87..000000000000 --- a/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/step.json +++ /dev/null @@ -1,66 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Step Schema", - "additionalProperties": true, - "type": ["null", "object"], - "properties": { - "parent_step_sid": { - "description": "The SID of the parent step if this step is nested within another step.", - "type": ["null", "string"] - }, - "name": { - "description": "The name or identifier of the step.", - "type": ["null", "string"] - }, - "date_updated": { - "description": "The date and time when the step was last updated.", - "type": ["null", "string"] - }, - "transitioned_to": { - "description": "The state to which the step transitioned.", - "type": ["null", "string"] - }, - "account_sid": { - "description": "The SID of the account associated with the step data.", - "type": ["null", "string"] - }, - "url": { - "description": "The URL for accessing detailed information about the step.", - "type": ["null", "string"] - }, - "context": { - "description": "Additional context or metadata related to the step.", - "type": ["null", "string"] - }, - "sid": { - "description": "The unique identifier (SID) of the step.", - "type": ["null", "string"] - }, - "transitioned_from": { - "description": "The previous state from which the step transitioned.", - "type": ["null", "string"] - }, - "date_created": { - "description": "The date and time when the step was created.", - "type": ["null", "string"] - }, - "execution_sid": { - "description": "The SID of the execution this step belongs to.", - "type": ["null", "string"] - }, - "flow_sid": { - "description": "The SID of the flow to which this step belongs.", - "type": ["null", "string"] - }, - "links": { - "description": "Contains related hypermedia links to navigate through the step data.", - "type": ["null", "object"], - "properties": { - "step_context": { - "description": "Link to additional contextual information related to the step.", - "type": ["null", "string"] - } - } - } - } -} diff --git a/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/transcriptions.json b/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/transcriptions.json deleted file mode 100644 index 01c40cb99f2d..000000000000 --- a/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/transcriptions.json +++ /dev/null @@ -1,60 +0,0 @@ -{ - "properties": { - "account_sid": { - "description": "The unique identifier for the account associated with the transcription data.", - "type": ["null", "string"] - }, - "api_version": { - "description": "The version of the Twilio API used for the transcription.", - "type": ["null", "string"] - }, - "date_created": { - "description": "The date and time when the transcription was created.", - "format": "date-time", - "type": ["null", "string"] - }, - "date_updated": { - "description": "The date and time when the transcription was last updated.", - "format": "date-time", - "type": ["null", "string"] - }, - "duration": { - "description": "The duration of the audio recording for which the transcription was generated.", - "type": ["null", "integer"] - }, - "price": { - "description": "The cost of the transcription service.", - "type": ["null", "number"] - }, - "price_unit": { - "description": "The currency unit used for pricing the transcription service.", - "type": ["null", "string"] - }, - "recording_sid": { - "description": "The unique identifier for the recording associated with the transcription.", - "type": ["null", "string"] - }, - "sid": { - "description": "The unique identifier for the transcription data.", - "type": ["null", "string"] - }, - "status": { - "description": "The current status of the transcription (e.g., in-progress, completed).", - "type": ["null", "string"] - }, - "transcription_text": { - "description": "The text content of the transcription generated from the audio recording.", - "type": ["null", "string"] - }, - "type": { - "description": "The type of transcription (e.g., automatic, human-generated).", - "type": ["null", "string"] - }, - "uri": { - "description": "The URI where the transcription data can be accessed.", - "type": ["null", "string"] - } - }, - "type": "object", - "additionalProperties": true -} diff --git a/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/trunks.json b/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/trunks.json deleted file mode 100644 index 99ace291c2a7..000000000000 --- a/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/trunks.json +++ /dev/null @@ -1,112 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Trunks Schema", - "additionalProperties": true, - "type": ["null", "object"], - "properties": { - "sid": { - "description": "The unique identifier for the trunk.", - "type": ["null", "string"] - }, - "account_sid": { - "description": "The unique identifier of the account associated with the trunk.", - "type": ["null", "string"] - }, - "domain_name": { - "description": "The domain name associated with the trunk.", - "type": ["null", "string"] - }, - "disaster_recovery_method": { - "description": "The method used for disaster recovery for this trunk.", - "type": ["null", "string"] - }, - "disaster_recovery_url": { - "description": "The URL set up for disaster recovery in case of trunk failure.", - "type": ["null", "string"] - }, - "friendly_name": { - "description": "A user-friendly name for the trunk.", - "type": ["null", "string"] - }, - "secure": { - "description": "Indicates whether the trunk connection is secure.", - "type": ["null", "boolean"] - }, - "cnam_lookup_enabled": { - "description": "Indicates whether Caller ID Name (CNAM) lookup is enabled for this trunk.", - "type": ["null", "boolean"] - }, - "recording": { - "description": "Settings related to call recording on the trunk.", - "type": ["null", "object"], - "properties": { - "mode": { - "description": "The mode used for recording calls on this trunk.", - "type": ["null", "string"] - }, - "trim": { - "description": "Indicates whether recorded calls should be trimmed.", - "type": ["null", "string"] - } - } - }, - "transfer_mode": { - "description": "The mode used for call transfer on this trunk.", - "type": ["null", "string"] - }, - "transfer_caller_id": { - "description": "The Caller ID used when transferring calls through this trunk.", - "type": ["null", "string"] - }, - "auth_type": { - "description": "The authentication type used for this trunk.", - "type": ["null", "string"] - }, - "auth_type_set": { - "description": "A set of authentication types associated with the trunk.", - "type": ["null", "array"], - "items": { - "auth_type": { - "description": "The authentication type used for each item in the set.", - "type": ["null", "string"] - } - } - }, - "date_created": { - "description": "The date and time the trunk was created.", - "type": ["null", "string"], - "format": "date-time" - }, - "date_updated": { - "description": "The date and time the trunk was last updated.", - "type": ["null", "string"], - "format": "date-time" - }, - "url": { - "description": "The URL of the trunk.", - "type": ["null", "string"] - }, - "links": { - "description": "Links to related resources associated with the trunk.", - "type": ["null", "object"], - "properties": { - "origination_urls": { - "description": "Endpoint for managing origination URLs associated with the trunk.", - "type": ["null", "string"] - }, - "credential_lists": { - "description": "Endpoint for managing credential lists associated with the trunk.", - "type": ["null", "string"] - }, - "ip_access_control_lists": { - "description": "Endpoint for managing IP Access Control Lists (ACLs) associated with the trunk.", - "type": ["null", "string"] - }, - "phone_numbers": { - "description": "Endpoint for managing phone numbers associated with the trunk.", - "type": ["null", "string"] - } - } - } - } -} diff --git a/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/usage_records.json b/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/usage_records.json deleted file mode 100644 index b9dc8ff080bc..000000000000 --- a/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/usage_records.json +++ /dev/null @@ -1,103 +0,0 @@ -{ - "properties": { - "category": { - "description": "The category to which the usage data belongs, such as calls, SMS, or recordings.", - "type": ["null", "string"] - }, - "count": { - "description": "The number of units consumed in the usage record.", - "type": ["null", "integer"] - }, - "price_unit": { - "description": "The currency unit in which the price is denoted, such as USD or EUR.", - "type": ["null", "string"] - }, - "subresource_uris": { - "description": "Contains references to sub-resources related to the usage record.", - "type": ["null", "object"], - "additionalProperties": true, - "properties": { - "yesterday": { - "description": "The resource URI for fetching yesterday's usage records.", - "type": ["null", "string"] - }, - "all_time": { - "description": "The resource URI for fetching usage records spanning all time periods.", - "type": ["null", "string"] - }, - "today": { - "description": "The resource URI for fetching today's usage records.", - "type": ["null", "string"] - }, - "yearly": { - "description": "The resource URI for fetching yearly usage records.", - "type": ["null", "string"] - }, - "this_month": { - "description": "The resource URI for fetching usage records for the current month.", - "type": ["null", "string"] - }, - "monthly": { - "description": "The resource URI for fetching monthly usage records.", - "type": ["null", "string"] - }, - "daily": { - "description": "The resource URI for fetching daily usage records.", - "type": ["null", "string"] - }, - "last_month": { - "description": "The resource URI for fetching usage records for the last month.", - "type": ["null", "string"] - } - } - }, - "description": { - "description": "A brief explanation or summary of the usage record data.", - "type": ["null", "string"] - }, - "end_date": { - "description": "The end date of the usage record period.", - "format": "date", - "type": ["null", "string"] - }, - "usage_unit": { - "description": "The unit of measurement for the usage field, corresponding to the usage count unit.", - "type": ["null", "string"] - }, - "price": { - "description": "The total price incurred for the consumed units in the usage record.", - "type": ["null", "number"] - }, - "uri": { - "description": "The unique resource identifier for the usage record.", - "type": ["null", "string"] - }, - "account_sid": { - "description": "The unique identifier for the account associated with the usage record.", - "type": ["null", "string"] - }, - "usage": { - "description": "The total usage value, which could be a cumulative count or a sum of values depending on the category.", - "type": ["null", "number"] - }, - "start_date": { - "description": "The start date and time of the usage record period.", - "format": "date-time", - "type": ["null", "string"] - }, - "count_unit": { - "description": "The unit of measurement for the count field, such as minutes, messages, or bytes.", - "type": ["null", "string"] - }, - "as_of": { - "description": "The timestamp indicating the date and time up to which the usage data is accurate.", - "type": ["null", "string"] - }, - "api_version": { - "description": "The version of the Twilio API used in the interaction that generated the usage record.", - "type": ["null", "string"] - } - }, - "type": "object", - "additionalProperties": true -} diff --git a/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/usage_triggers.json b/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/usage_triggers.json deleted file mode 100644 index a5a8a58c1dfb..000000000000 --- a/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/usage_triggers.json +++ /dev/null @@ -1,73 +0,0 @@ -{ - "properties": { - "usage_record_uri": { - "description": "The URI of the associated usage record.", - "type": ["null", "string"] - }, - "date_updated": { - "description": "The date and time when the trigger was last updated.", - "format": "date-time", - "type": ["null", "string"] - }, - "date_fired": { - "description": "The date and time when the trigger was last fired.", - "format": "date-time", - "type": ["null", "string"] - }, - "friendly_name": { - "description": "A user-friendly name for the trigger.", - "type": ["null", "string"] - }, - "uri": { - "description": "The URI of the usage trigger resource.", - "type": ["null", "string"] - }, - "account_sid": { - "description": "The unique identifier for the account associated with the usage trigger.", - "type": ["null", "string"] - }, - "callback_method": { - "description": "The HTTP method used for callback notifications.", - "type": ["null", "string"] - }, - "trigger_by": { - "description": "The attribute by which the trigger condition is evaluated.", - "type": ["null", "string"] - }, - "sid": { - "description": "The unique identifier for the trigger.", - "type": ["null", "string"] - }, - "current_value": { - "description": "The current value for the trigger condition.", - "type": ["null", "number"] - }, - "date_created": { - "description": "The date and time when the trigger was created.", - "format": "date-time", - "type": ["null", "string"] - }, - "callback_url": { - "description": "The URL where callback notifications are sent.", - "type": ["null", "string"] - }, - "recurring": { - "description": "Indicates if the trigger is set to recur periodically.", - "type": ["null", "string"] - }, - "usage_category": { - "description": "The category under which the usage falls.", - "type": ["null", "string"] - }, - "trigger_value": { - "description": "The value that triggers the condition.", - "type": ["null", "number"] - }, - "api_version": { - "description": "The version of the Twilio API used for the trigger.", - "type": ["null", "string"] - } - }, - "type": "object", - "additionalProperties": true -} diff --git a/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/user_conversations.json b/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/user_conversations.json deleted file mode 100644 index ba72601e9ce6..000000000000 --- a/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/user_conversations.json +++ /dev/null @@ -1,96 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "User Conversation Schema", - "additionalProperties": true, - "type": ["null", "object"], - "properties": { - "notification_level": { - "description": "The level of notification settings for the conversation.", - "type": ["null", "string"] - }, - "unique_name": { - "description": "A unique name assigned to the conversation.", - "type": ["null", "string"] - }, - "user_sid": { - "description": "The unique identifier of the user associated with the conversation.", - "type": ["null", "string"] - }, - "friendly_name": { - "description": "A user-friendly name or alias for the conversation.", - "type": ["null", "string"] - }, - "conversation_sid": { - "description": "The unique identifier of the conversation.", - "type": ["null", "string"] - }, - "unread_messages_count": { - "description": "The count of unread messages in the conversation.", - "type": ["null", "integer"] - }, - "created_by": { - "description": "The user or entity that created the conversation.", - "type": ["null", "string"] - }, - "account_sid": { - "description": "The unique identifier of the Twilio account the conversation belongs to.", - "type": ["null", "string"] - }, - "last_read_message_index": { - "description": "The index of the last message in the conversation that has been read by the user.", - "type": ["null", "integer"] - }, - "date_created": { - "description": "The date and time when the conversation was initially created.", - "type": ["null", "string"] - }, - "timers": { - "description": "Contains timers or time-related information for the user conversation.", - "type": ["null", "object"], - "properties": { - "chat_service_sid": { - "description": "Additional timer related to the chat service.", - "type": ["null", "string"] - } - } - }, - "url": { - "description": "The URL to access the conversation details.", - "type": ["null", "string"] - }, - "date_updated": { - "description": "The date and time when the conversation was last updated.", - "type": ["null", "string"] - }, - "attributes": { - "description": "Additional metadata or attributes associated with the conversation.", - "type": ["null", "string"] - }, - "participant_sid": { - "description": "The unique identifier of the participant in the conversation.", - "type": ["null", "string"] - }, - "conversation_state": { - "description": "The current state or status of the conversation.", - "type": ["null", "string"] - }, - "chat_service_sid": { - "description": "The unique identifier of the chat service associated with the conversation.", - "type": ["null", "string"] - }, - "links": { - "description": "Contains links related to the user conversation data.", - "type": ["null", "object"], - "properties": { - "conversation": { - "description": "Link to access detailed information about the conversation.", - "type": ["null", "string"] - }, - "participant": { - "description": "Link to access participant information related to the conversation.", - "type": ["null", "string"] - } - } - } - } -} diff --git a/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/users.json b/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/users.json deleted file mode 100644 index c0d90e396288..000000000000 --- a/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/users.json +++ /dev/null @@ -1,66 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Users Schema", - "additionalProperties": true, - "type": ["null", "object"], - "properties": { - "is_notifiable": { - "description": "Indicates whether the user is set to receive notifications", - "type": ["null", "string"] - }, - "date_updated": { - "description": "The date and time when the user was last updated", - "type": ["null", "string"] - }, - "is_online": { - "description": "Indicates whether the user is currently online", - "type": ["null", "string"] - }, - "friendly_name": { - "description": "A user-friendly name for the user", - "type": ["null", "string"] - }, - "account_sid": { - "description": "The unique identifier for the user's account", - "type": ["null", "string"] - }, - "url": { - "description": "URL to access the user's detailed information", - "type": ["null", "string"] - }, - "date_created": { - "description": "The date and time when the user was created", - "type": ["null", "string"] - }, - "role_sid": { - "description": "The unique identifier for the role assigned to the user", - "type": ["null", "string"] - }, - "sid": { - "description": "The unique identifier for the user", - "type": ["null", "string"] - }, - "attributes": { - "description": "Additional attributes or metadata associated with the user", - "type": ["null", "string"] - }, - "identity": { - "description": "The identity or username of the user", - "type": ["null", "string"] - }, - "chat_service_sid": { - "description": "The unique identifier for the chat service the user is associated with", - "type": ["null", "string"] - }, - "links": { - "description": "Contains URLs to related resources for the user data.", - "type": ["null", "object"], - "properties": { - "user_conversations": { - "description": "URL to retrieve the conversations associated with the user", - "type": ["null", "string"] - } - } - } - } -} diff --git a/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/verify_services.json b/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/verify_services.json deleted file mode 100644 index f80b47ff1ba8..000000000000 --- a/airbyte-integrations/connectors/source-twilio/source_twilio/schemas/verify_services.json +++ /dev/null @@ -1,146 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Verify - Services Schema", - "additionalProperties": true, - "type": ["null", "object"], - "properties": { - "default_template_sid": { - "description": "The unique identifier for the default template used for verification messages.", - "type": ["null", "string"] - }, - "tts_name": { - "description": "The name used for text-to-speech (TTS) in verification calls.", - "type": ["null", "string"] - }, - "psd2_enabled": { - "description": "Indicates if PSD2 (Payment Services Directive 2) verification is enabled.", - "type": ["null", "boolean"] - }, - "do_not_share_warning_enabled": { - "description": "Indicates if the 'do not share' warning is enabled for verification codes.", - "type": ["null", "boolean"] - }, - "mailer_sid": { - "description": "Unique identifier for the mailer service associated with the verify service.", - "type": ["null", "string"] - }, - "friendly_name": { - "description": "A user-friendly name for the verify service.", - "type": ["null", "string"] - }, - "url": { - "description": "The URL associated with the verify service.", - "type": ["null", "string"] - }, - "account_sid": { - "description": "Unique identifier for the account associated with the verify service.", - "type": ["null", "string"] - }, - "date_updated": { - "description": "The date and time when the verify service was last updated.", - "type": ["null", "string"] - }, - "totp": { - "description": "Object containing configuration settings for Time-based One-Time Password (TOTP) verification method.", - "type": ["null", "object"], - "properties": { - "time_step": { - "description": "The time step interval for generating TOTP codes.", - "type": ["null", "number"] - }, - "skew": { - "description": "The time skew allowed for TOTP code validation.", - "type": ["null", "number"] - }, - "code_length": { - "description": "The number of digits in the TOTP (Time-based One-Time Password) code.", - "type": ["null", "number"] - }, - "issuer": { - "description": "The issuer name included in TOTP messages.", - "type": ["null", "string"] - } - } - }, - "code_length": { - "description": "The number of digits in the verification code sent to users.", - "type": ["null", "number"] - }, - "custom_code_enabled": { - "description": "Indicates whether custom verification codes are enabled.", - "type": ["null", "boolean"] - }, - "sid": { - "description": "Unique identifier for the verify service.", - "type": ["null", "string"] - }, - "push": { - "description": "Object containing configuration settings for push verification method.", - "type": ["null", "object"], - "properties": { - "apn_credential_sid": { - "description": "Unique identifier for the APN (Apple Push Notification) credential associated with the verify service.", - "type": ["null", "string"] - }, - "include_date": { - "description": "Indicates if the date should be included in push notifications.", - "type": ["null", "boolean"] - }, - "fcm_credential_sid": { - "description": "Unique identifier for the FCM (Firebase Cloud Messaging) credential associated with the verify service.", - "type": ["null", "string"] - } - } - }, - "date_created": { - "description": "The date and time when the verify service was created.", - "type": ["null", "string"] - }, - "dtmf_input_required": { - "description": "Indicates whether DTMF input is required during verification.", - "type": ["null", "boolean"] - }, - "skip_sms_to_landlines": { - "description": "Indicates whether SMS messages are skipped for landline numbers during verification.", - "type": ["null", "boolean"] - }, - "lookup_enabled": { - "description": "Indicates if phone number lookup is enabled for the verify service.", - "type": ["null", "boolean"] - }, - "links": { - "description": "Object containing related hyperlinks for verify_services data.", - "type": ["null", "object"], - "properties": { - "verification_checks": { - "description": "Links related to verification checks for the verify service.", - "type": ["null", "string"] - }, - "rate_limits": { - "description": "Links related to rate limits for the verify service.", - "type": ["null", "string"] - }, - "entities": { - "description": "Links related to entities associated with the verify service.", - "type": ["null", "string"] - }, - "access_tokens": { - "description": "Links related to access tokens for the verify service.", - "type": ["null", "string"] - }, - "verifications": { - "description": "Links related to verifications performed using the verify service.", - "type": ["null", "string"] - }, - "webhooks": { - "description": "Links related to webhooks for the verify service.", - "type": ["null", "string"] - }, - "messaging_configurations": { - "description": "Links related to messaging configurations for the verify service.", - "type": ["null", "string"] - } - } - } - } -} diff --git a/airbyte-integrations/connectors/source-twilio/source_twilio/source.py b/airbyte-integrations/connectors/source-twilio/source_twilio/source.py index 7243e677a08b..9e922246b55e 100644 --- a/airbyte-integrations/connectors/source-twilio/source_twilio/source.py +++ b/airbyte-integrations/connectors/source-twilio/source_twilio/source.py @@ -1,130 +1,10 @@ # -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# Copyright (c) 2024 Airbyte, Inc., all rights reserved. # -import datetime -import logging -from typing import Any, List, Mapping, Tuple - -import pendulum -from airbyte_cdk.models import SyncMode from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource -from airbyte_cdk.sources.streams import Stream -from source_twilio.auth import HttpBasicAuthenticator -from source_twilio.streams import ( - Accounts, - Alerts, - Applications, - AvailablePhoneNumberCountries, - AvailablePhoneNumbersLocal, - AvailablePhoneNumbersMobile, - AvailablePhoneNumbersTollFree, - Calls, - ConferenceParticipants, - Conferences, - ConversationMessages, - ConversationParticipants, - Conversations, - DependentPhoneNumbers, - Executions, - Flows, - IncomingPhoneNumbers, - Keys, - MessageMedia, - Messages, - OutgoingCallerIds, - Queues, - Recordings, - Roles, - Services, - Step, - Transcriptions, - Trunks, - UsageRecords, - UsageTriggers, - UserConversations, - Users, - VerifyServices, -) - -RETENTION_WINDOW_LIMIT = 400 class SourceTwilio(YamlDeclarativeSource): def __init__(self): super().__init__(**{"path_to_yaml": "manifest.yaml"}) - - def check_connection(self, logger: logging.Logger, config: Mapping[str, Any]) -> Tuple[bool, Any]: - try: - auth = HttpBasicAuthenticator( - ( - config["account_sid"], - config["auth_token"], - ), - ) - accounts_gen = Accounts(authenticator=auth).read_records(sync_mode=SyncMode.full_refresh) - next(accounts_gen) - return True, None - except Exception as error: - return False, f"Unable to connect to Twilio API with the provided credentials - {repr(error)}" - - def streams(self, config: Mapping[str, Any]) -> List[Stream]: - """ - :param config: A Mapping of the user input configuration as defined in the connector spec. - """ - auth = HttpBasicAuthenticator( - ( - config["account_sid"], - config["auth_token"], - ), - ) - full_refresh_stream_kwargs = {"authenticator": auth} - incremental_stream_kwargs = { - "authenticator": auth, - "start_date": config["start_date"], - "lookback_window": config.get("lookback_window", 0), - "slice_step_map": config.get("slice_step_map", {}), - } - - # Fix for `Date range specified in query is partially or entirely outside of retention window of 400 days` - # See: https://app.zenhub.com/workspaces/python-connectors-6262f8b593bb82001df56c65/issues/airbytehq/airbyte/10418 - incremental_stream_kwargs_message_stream = dict(**incremental_stream_kwargs) - if pendulum.now().diff(pendulum.parse(config["start_date"])).days >= RETENTION_WINDOW_LIMIT: - incremental_stream_kwargs_message_stream["start_date"] = ( - pendulum.now() - datetime.timedelta(days=RETENTION_WINDOW_LIMIT - 1) - ).to_iso8601_string() - streams = super().streams(config) + [ - Alerts(**incremental_stream_kwargs), - Applications(**full_refresh_stream_kwargs), - AvailablePhoneNumberCountries(**full_refresh_stream_kwargs), - AvailablePhoneNumbersLocal(**full_refresh_stream_kwargs), - AvailablePhoneNumbersMobile(**full_refresh_stream_kwargs), - AvailablePhoneNumbersTollFree(**full_refresh_stream_kwargs), - Calls(**incremental_stream_kwargs), - ConferenceParticipants(**full_refresh_stream_kwargs), - Conferences(**incremental_stream_kwargs), - Conversations(**full_refresh_stream_kwargs), - ConversationMessages(**full_refresh_stream_kwargs), - ConversationParticipants(**full_refresh_stream_kwargs), - DependentPhoneNumbers(**full_refresh_stream_kwargs), - Flows(**full_refresh_stream_kwargs), - Executions(**full_refresh_stream_kwargs), - IncomingPhoneNumbers(**full_refresh_stream_kwargs), - Keys(**full_refresh_stream_kwargs), - MessageMedia(**incremental_stream_kwargs_message_stream), - Messages(**incremental_stream_kwargs_message_stream), - OutgoingCallerIds(**full_refresh_stream_kwargs), - Queues(**full_refresh_stream_kwargs), - Recordings(**incremental_stream_kwargs), - Roles(**full_refresh_stream_kwargs), - Services(**full_refresh_stream_kwargs), - Step(**full_refresh_stream_kwargs), - Transcriptions(**full_refresh_stream_kwargs), - Trunks(**full_refresh_stream_kwargs), - UsageRecords(**incremental_stream_kwargs), - UsageTriggers(**full_refresh_stream_kwargs), - Users(**full_refresh_stream_kwargs), - UserConversations(**full_refresh_stream_kwargs), - VerifyServices(**full_refresh_stream_kwargs), - ] - return streams diff --git a/airbyte-integrations/connectors/source-twilio/source_twilio/streams.py b/airbyte-integrations/connectors/source-twilio/source_twilio/streams.py deleted file mode 100644 index 2e82d6639452..000000000000 --- a/airbyte-integrations/connectors/source-twilio/source_twilio/streams.py +++ /dev/null @@ -1,670 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -import copy -from abc import ABC, abstractmethod -from functools import cached_property -from typing import Any, Iterable, List, Mapping, MutableMapping, Optional -from urllib.parse import parse_qsl, urlparse - -import pendulum -import requests -from airbyte_cdk.models import SyncMode -from airbyte_cdk.sources.streams import IncrementalMixin -from airbyte_cdk.sources.streams.availability_strategy import AvailabilityStrategy -from airbyte_cdk.sources.streams.http import HttpStream -from airbyte_cdk.sources.utils.transform import TransformConfig, TypeTransformer -from pendulum.datetime import DateTime -from requests.auth import AuthBase - -TWILIO_CHAT_BASE = "https://chat.twilio.com/v2/" -TWILIO_CONVERSATION_BASE = "https://conversations.twilio.com/v1/" -TWILIO_API_URL_BASE = "https://api.twilio.com" -TWILIO_API_URL_BASE_VERSIONED = f"{TWILIO_API_URL_BASE}/2010-04-01/" -TWILIO_MONITOR_URL_BASE = "https://monitor.twilio.com/v1/" -TWILIO_STUDIO_API_BASE = "https://studio.twilio.com/v1/" -TWILIO_CONVERSATIONS_URL_BASE = "https://conversations.twilio.com/v1/" -TWILIO_TRUNKING_URL_BASE = "https://trunking.twilio.com/v1/" -TWILIO_VERIFY_BASE_V2 = "https://verify.twilio.com/v2/" - - -class TwilioStream(HttpStream, ABC): - url_base = TWILIO_API_URL_BASE - primary_key = "sid" - page_size = 1000 - transformer: TypeTransformer = TypeTransformer(TransformConfig.DefaultSchemaNormalization | TransformConfig.CustomSchemaNormalization) - - @property - def data_field(self): - return self.name - - @property - def changeable_fields(self): - """ - :return list of changeable fields that should be removed from the records - """ - return [] - - @property - def availability_strategy(self) -> Optional["AvailabilityStrategy"]: - return None - - def path(self, **kwargs): - return f"{self.name.title()}.json" - - def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: - stream_data = response.json() - next_page_uri = stream_data.get("next_page_uri") - if next_page_uri: - next_url = urlparse(next_page_uri) - next_page_params = dict(parse_qsl(next_url.query)) - return next_page_params - - def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: - """ - :return an iterable containing each record in the response - """ - records = response.json().get(self.data_field, []) - if self.changeable_fields: - for record in records: - for field in self.changeable_fields: - record.pop(field, None) - yield record - else: - yield from records - - def backoff_time(self, response: requests.Response) -> Optional[float]: - """This method is called if we run into the rate limit. - Twilio puts the retry time in the `Retry-After` response header so we - we return that value. If the response is anything other than a 429 (e.g: 5XX) - fall back on default retry behavior. - Rate Limits Docs: https://support.twilio.com/hc/en-us/articles/360032845014-Verify-V2-Rate-Limiting""" - - backoff_time = response.headers.get("Retry-After") - if backoff_time is not None: - return float(backoff_time) - - def request_params( - self, - stream_state: Mapping[str, Any], - stream_slice: Mapping[str, Any] = None, - next_page_token: Mapping[str, Any] = None, - ) -> MutableMapping[str, Any]: - params = super().request_params(stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token) - params["PageSize"] = self.page_size - if next_page_token: - params.update(**next_page_token) - return params - - @transformer.registerCustomTransform - def custom_transform_function(original_value: Any, field_schema: Mapping[str, Any]) -> Any: - if original_value and field_schema.get("format") == "date-time": - try: - return pendulum.from_format(original_value, "ddd, D MMM YYYY HH:mm:ss ZZ").in_timezone("UTC").to_iso8601_string() - except ValueError: - """Twilio API returns datetime in two formats: - - RFC2822, like "Fri, 11 Dec 2020 04:28:40 +0000"; - - ISO8601, like "2020-12-11T04:29:09Z". - If `ValueError` exception was raised this means that datetime was already in ISO8601 format and there - is no need in transforming anything.""" - pass - return original_value - - -class IncrementalTwilioStream(TwilioStream, IncrementalMixin): - time_filter_template = "YYYY-MM-DD HH:mm:ss[Z]" - # This attribute allows balancing between sync speed and memory consumption. - # The greater a slice is - the bigger memory consumption and the faster syncs are since fewer requests are made. - slice_step_default = pendulum.duration(years=1) - # time gap between when previous slice ends and current slice begins - slice_granularity = pendulum.duration(microseconds=1) - state_checkpoint_interval = 1000 - - def __init__( - self, - authenticator: AuthBase, - start_date: str = None, - lookback_window: int = 0, - slice_step_map: Mapping[str, int] = None, - ): - super().__init__(authenticator) - slice_step = (slice_step_map or {}).get(self.name) - self._slice_step = slice_step and pendulum.duration(days=slice_step) - self._start_date = start_date if start_date is not None else "1970-01-01T00:00:00Z" - self._lookback_window = lookback_window - self._cursor_value = None - - @property - def slice_step(self): - return self._slice_step or self.slice_step_default - - @property - @abstractmethod - def lower_boundary_filter_field(self) -> str: - """ - return: date filter query parameter name - """ - - @property - @abstractmethod - def upper_boundary_filter_field(self) -> str: - """ - return: date filter query parameter name - """ - - @property - def state(self) -> Mapping[str, Any]: - if self._cursor_value: - return { - self.cursor_field: self._cursor_value, - } - - return {} - - @state.setter - def state(self, value: MutableMapping[str, Any]): - if self._lookback_window and value.get(self.cursor_field): - new_start_date = ( - pendulum.parse(value[self.cursor_field]) - pendulum.duration(minutes=self._lookback_window) - ).to_iso8601_string() - if new_start_date > self._start_date: - value[self.cursor_field] = new_start_date - self._cursor_value = value.get(self.cursor_field) - - def generate_date_ranges(self) -> Iterable[Optional[MutableMapping[str, Any]]]: - def align_to_dt_format(dt: DateTime) -> DateTime: - return pendulum.parse(dt.format(self.time_filter_template)) - - end_datetime = pendulum.now("utc") - start_datetime = min(end_datetime, pendulum.parse(self.state.get(self.cursor_field, self._start_date))) - current_start = start_datetime - current_end = start_datetime - # Aligning to a datetime format is done to avoid the following scenario: - # start_dt = 2021-11-14T00:00:00, end_dt (now) = 2022-11-14T12:03:01, time_filter_template = "YYYY-MM-DD" - # First slice: (2021-11-14, 2022-11-14) - # (!) Second slice: (2022-11-15, 2022-11-14) - because 2022-11-14T00:00:00 (prev end) < 2022-11-14T12:03:01, - # so we have to compare dates, not date-times to avoid yielding that last slice - while align_to_dt_format(current_end) < align_to_dt_format(end_datetime): - current_end = min(end_datetime, current_start + self.slice_step) - slice_ = { - self.lower_boundary_filter_field: current_start.format(self.time_filter_template), - self.upper_boundary_filter_field: current_end.format(self.time_filter_template), - } - yield slice_ - current_start = current_end + self.slice_granularity - - def stream_slices( - self, sync_mode: SyncMode, cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None - ) -> Iterable[Optional[Mapping[str, Any]]]: - for super_slice in super().stream_slices(sync_mode=sync_mode, cursor_field=cursor_field, stream_state=stream_state): - for dt_range in self.generate_date_ranges(): - slice_ = copy.deepcopy(super_slice) if super_slice else {} - slice_.update(dt_range) - yield slice_ - - def request_params( - self, - stream_state: Mapping[str, Any], - stream_slice: Mapping[str, Any] = None, - next_page_token: Mapping[str, Any] = None, - ) -> MutableMapping[str, Any]: - params = super().request_params(stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token) - lower_bound = stream_slice and stream_slice.get(self.lower_boundary_filter_field) - upper_bound = stream_slice and stream_slice.get(self.upper_boundary_filter_field) - if lower_bound: - params[self.lower_boundary_filter_field] = lower_bound - if upper_bound: - params[self.upper_boundary_filter_field] = upper_bound - return params - - def read_records( - self, - sync_mode: SyncMode, - cursor_field: List[str] = None, - stream_slice: Mapping[str, Any] = None, - stream_state: Mapping[str, Any] = None, - ) -> Iterable[Mapping[str, Any]]: - unsorted_records = [] - for record in super().read_records(sync_mode, cursor_field, stream_slice, stream_state): - record[self.cursor_field] = pendulum.parse(record[self.cursor_field], strict=False).to_iso8601_string() - unsorted_records.append(record) - sorted_records = sorted(unsorted_records, key=lambda x: x[self.cursor_field]) - for record in sorted_records: - if record[self.cursor_field] >= self.state.get(self.cursor_field, self._start_date): - self._cursor_value = record[self.cursor_field] - yield record - - -class TwilioNestedStream(TwilioStream): - """ - Basic class for the streams that are dependant on the results of another stream output (parent-child relations). - Parent class read is always full refresh, even if it supports incremental read. - """ - - media_exist_validation = {} - uri_from_subresource = True - - def path(self, stream_slice: Mapping[str, Any], **kwargs): - return stream_slice["subresource_uri"] - - @property - def subresource_uri_key(self): - return self.data_field - - @property - @abstractmethod - def parent_stream(self) -> TwilioStream: - """ - :return: parent stream class - """ - - @cached_property - def parent_stream_instance(self): - return self.parent_stream(authenticator=self._session.auth) - - def parent_record_to_stream_slice(self, record: Mapping[str, Any]) -> Mapping[str, Any]: - return {"subresource_uri": record["subresource_uris"][self.subresource_uri_key]} - - def stream_slices(self, **kwargs) -> Iterable[Optional[Mapping[str, any]]]: - stream_instance = self.parent_stream_instance - stream_slices = stream_instance.stream_slices(sync_mode=SyncMode.full_refresh, cursor_field=stream_instance.cursor_field) - for stream_slice in stream_slices: - for item in stream_instance.read_records( - sync_mode=SyncMode.full_refresh, stream_slice=stream_slice, cursor_field=stream_instance.cursor_field - ): - if not self.uri_from_subresource: - yield self.parent_record_to_stream_slice(item) - elif item.get("subresource_uris", {}).get(self.subresource_uri_key): - validated = True - for key, value in self.media_exist_validation.items(): - validated = item.get(key) and item.get(key) != value - if not validated: - break - if validated: - yield self.parent_record_to_stream_slice(item) - - -class Accounts(TwilioStream): - """https://www.twilio.com/docs/usage/api/account#read-multiple-account-resources""" - - url_base = TWILIO_API_URL_BASE_VERSIONED - - -class Addresses(TwilioNestedStream): - """https://www.twilio.com/docs/usage/api/address#read-multiple-address-resources""" - - parent_stream = Accounts - - -class DependentPhoneNumbers(TwilioNestedStream): - """https://www.twilio.com/docs/usage/api/address?code-sample=code-list-dependent-pns-subresources&code-language=curl&code-sdk-version=json#instance-subresources""" - - parent_stream = Addresses - url_base = TWILIO_API_URL_BASE_VERSIONED - uri_from_subresource = False - - def path(self, stream_slice: Mapping[str, Any], **kwargs): - return f"Accounts/{stream_slice['account_sid']}/Addresses/{stream_slice['sid']}/DependentPhoneNumbers.json" - - def parent_record_to_stream_slice(self, record: Mapping[str, Any]) -> Mapping[str, Any]: - return {"sid": record["sid"], "account_sid": record["account_sid"]} - - -class Applications(TwilioNestedStream): - """https://www.twilio.com/docs/usage/api/applications#read-multiple-application-resources""" - - parent_stream = Accounts - - -class AvailablePhoneNumberCountries(TwilioNestedStream): - """ - https://www.twilio.com/docs/phone-numbers/api/availablephonenumber-resource#read-a-list-of-countries - - List of available phone number countries, as well as local, mobile and toll free numbers - may be different on each request, so could not pass the full refresh tests. - """ - - parent_stream = Accounts - data_field = "countries" - subresource_uri_key = "available_phone_numbers" - primary_key = None - - -class AvailablePhoneNumbersLocal(TwilioNestedStream): - """https://www.twilio.com/docs/phone-numbers/api/availablephonenumberlocal-resource#read-multiple-availablephonenumberlocal-resources""" - - parent_stream = AvailablePhoneNumberCountries - data_field = "available_phone_numbers" - subresource_uri_key = "local" - primary_key = None - - -class AvailablePhoneNumbersMobile(TwilioNestedStream): - """https://www.twilio.com/docs/phone-numbers/api/availablephonenumber-mobile-resource#read-multiple-availablephonenumbermobile-resources""" - - parent_stream = AvailablePhoneNumberCountries - data_field = "available_phone_numbers" - subresource_uri_key = "mobile" - primary_key = None - - -class AvailablePhoneNumbersTollFree(TwilioNestedStream): - """https://www.twilio.com/docs/phone-numbers/api/availablephonenumber-tollfree-resource#read-multiple-availablephonenumbertollfree-resources""" - - parent_stream = AvailablePhoneNumberCountries - data_field = "available_phone_numbers" - subresource_uri_key = "toll_free" - primary_key = None - - -class IncomingPhoneNumbers(TwilioNestedStream): - """https://www.twilio.com/docs/phone-numbers/api/incomingphonenumber-resource#read-multiple-incomingphonenumber-resources""" - - parent_stream = Accounts - - -class Keys(TwilioNestedStream): - """https://www.twilio.com/docs/usage/api/keys#read-a-key-resource""" - - parent_stream = Accounts - - -class Calls(IncrementalTwilioStream, TwilioNestedStream): - """https://www.twilio.com/docs/voice/api/call-resource#create-a-call-resource""" - - parent_stream = Accounts - lower_boundary_filter_field = "EndTime>" - upper_boundary_filter_field = "EndTime<" - cursor_field = "end_time" - time_filter_template = "YYYY-MM-DD" - slice_granularity = pendulum.duration(days=1) - - -class Conferences(IncrementalTwilioStream, TwilioNestedStream): - """https://www.twilio.com/docs/voice/api/conference-resource#read-multiple-conference-resources""" - - parent_stream = Accounts - lower_boundary_filter_field = "DateCreated>" - upper_boundary_filter_field = "DateCreated<" - cursor_field = "date_created" - time_filter_template = "YYYY-MM-DD" - slice_granularity = pendulum.duration(days=1) - - -class ConferenceParticipants(TwilioNestedStream): - """ - https://www.twilio.com/docs/voice/api/conference-participant-resource#read-multiple-participant-resources - - This streams has records only if there are active conference participants (participants, - which are on conference call at the moment request is made). - """ - - primary_key = ["account_sid", "conference_sid"] - parent_stream = Conferences - data_field = "participants" - - -class Flows(TwilioStream): - """ - https://www.twilio.com/docs/studio/rest-api/flow#read-a-list-of-flows - """ - - url_base = TWILIO_STUDIO_API_BASE - - def path(self, **kwargs): - return "Flows" - - -class Executions(TwilioNestedStream): - """ - https://www.twilio.com/docs/studio/rest-api/execution#read-a-list-of-executions - """ - - parent_stream = Flows - url_base = TWILIO_STUDIO_API_BASE - uri_from_subresource = False - - def path(self, stream_slice: Mapping[str, Any] = None, **kwargs): - return f"Flows/{ stream_slice['flow_sid'] }/Executions" - - def parent_record_to_stream_slice(self, record: Mapping[str, Any]) -> Mapping[str, Any]: - return {"flow_sid": record["sid"]} - - -class Step(TwilioNestedStream): - """ - https://www.twilio.com/docs/studio/rest-api/v2/step#read-a-list-of-step-resources - """ - - parent_stream = Executions - url_base = TWILIO_STUDIO_API_BASE - uri_from_subresource = False - data_field = "steps" - - def path(self, stream_slice: Mapping[str, Any], **kwargs): - return f"Flows/{stream_slice['flow_sid']}/Executions/{stream_slice['execution_sid']}/Steps" - - def parent_record_to_stream_slice(self, record: Mapping[str, Any]) -> Mapping[str, Any]: - return {"flow_sid": record["flow_sid"], "execution_sid": record["sid"]} - - -class OutgoingCallerIds(TwilioNestedStream): - """https://www.twilio.com/docs/voice/api/outgoing-caller-ids#outgoingcallerids-list-resource""" - - parent_stream = Accounts - - -class Recordings(IncrementalTwilioStream, TwilioNestedStream): - """https://www.twilio.com/docs/voice/api/recording#read-multiple-recording-resources""" - - parent_stream = Accounts - lower_boundary_filter_field = "DateCreated>" - upper_boundary_filter_field = "DateCreated<" - cursor_field = "date_created" - - -class Services(TwilioStream): - """ - https://www.twilio.com/docs/chat/rest/service-resource#read-multiple-service-resources - """ - - url_base = TWILIO_CHAT_BASE - - def path(self, **kwargs): - return "Services" - - -class VerifyServices(TwilioStream): - """ - https://www.twilio.com/docs/chat/rest/service-resource#read-multiple-service-resources - """ - - # Unlike other endpoints, this one won't accept requests where pageSize >100 - page_size = 100 - data_field = "services" - url_base = TWILIO_VERIFY_BASE_V2 - - def path(self, **kwargs): - return "Services" - - -class Roles(TwilioNestedStream): - """ - https://www.twilio.com/docs/chat/rest/role-resource#read-multiple-role-resources - """ - - parent_stream = Services - url_base = TWILIO_CHAT_BASE - uri_from_subresource = False - - def path(self, stream_slice: Mapping[str, Any] = None, **kwargs): - return f"Services/{ stream_slice['service_sid'] }/Roles" - - def parent_record_to_stream_slice(self, record: Mapping[str, Any]) -> Mapping[str, Any]: - return {"service_sid": record["sid"]} - - -class Transcriptions(TwilioNestedStream): - """https://www.twilio.com/docs/voice/api/recording-transcription?code-sample=code-read-list-all-transcriptions&code-language=curl&code-sdk-version=json#read-multiple-transcription-resources""" - - parent_stream = Accounts - - -class Trunks(TwilioStream): - """ - https://www.twilio.com/docs/sip-trunking/api/trunk-resource#trunk-properties - """ - - url_base = TWILIO_TRUNKING_URL_BASE - - def path(self, **kwargs): - return "Trunks" - - -class Queues(TwilioNestedStream): - """https://www.twilio.com/docs/voice/api/queue-resource#read-multiple-queue-resources""" - - parent_stream = Accounts - - -class Messages(IncrementalTwilioStream, TwilioNestedStream): - """https://www.twilio.com/docs/sms/api/message-resource#read-multiple-message-resources""" - - parent_stream = Accounts - slice_step_default = pendulum.duration(days=1) - lower_boundary_filter_field = "DateSent>" - upper_boundary_filter_field = "DateSent<" - cursor_field = "date_sent" - - -class MessageMedia(IncrementalTwilioStream, TwilioNestedStream): - """https://www.twilio.com/docs/sms/api/media-resource#read-multiple-media-resources""" - - parent_stream = Messages - data_field = "media_list" - subresource_uri_key = "media" - media_exist_validation = {"num_media": "0"} - lower_boundary_filter_field = "DateCreated>" - upper_boundary_filter_field = "DateCreated<" - cursor_field = "date_created" - - @cached_property - def parent_stream_instance(self): - return self.parent_stream(authenticator=self._session.auth, start_date=self._start_date, lookback_window=self._lookback_window) - - -class UsageNestedStream(TwilioNestedStream): - url_base = TWILIO_API_URL_BASE_VERSIONED - uri_from_subresource = False - - @property - @abstractmethod - def path_name(self) -> str: - """ - return: name of the end of the usage paths - """ - - def path(self, stream_slice: Mapping[str, Any], **kwargs): - return f"Accounts/{stream_slice['account_sid']}/Usage/{self.path_name}.json" - - def parent_record_to_stream_slice(self, record: Mapping[str, Any]) -> Mapping[str, Any]: - return {"account_sid": record["sid"]} - - -class UsageRecords(IncrementalTwilioStream, UsageNestedStream): - """https://www.twilio.com/docs/usage/api/usage-record#read-multiple-usagerecord-resources""" - - parent_stream = Accounts - lower_boundary_filter_field = "StartDate" - upper_boundary_filter_field = "EndDate" - cursor_field = "start_date" - time_filter_template = "YYYY-MM-DD" - slice_granularity = pendulum.duration(days=1) - path_name = "Records" - primary_key = [["account_sid"], ["category"]] - changeable_fields = ["as_of"] - - -class UsageTriggers(UsageNestedStream): - """https://www.twilio.com/docs/usage/api/usage-trigger#read-multiple-usagetrigger-resources""" - - parent_stream = Accounts - subresource_uri_key = "triggers" - path_name = "Triggers" - - -class Alerts(IncrementalTwilioStream): - """https://www.twilio.com/docs/usage/monitor-alert#read-multiple-alert-resources""" - - url_base = TWILIO_MONITOR_URL_BASE - lower_boundary_filter_field = "StartDate=" - upper_boundary_filter_field = "EndDate=" - cursor_field = "date_generated" - - def path(self, **kwargs): - return self.name.title() - - -class Conversations(TwilioStream): - """https://www.twilio.com/docs/conversations/api/conversation-resource#read-multiple-conversation-resources""" - - url_base = TWILIO_CONVERSATIONS_URL_BASE - - def path(self, **kwargs): - return self.name.title() - - -class ConversationParticipants(TwilioNestedStream): - """https://www.twilio.com/docs/conversations/api/conversation-participant-resource""" - - parent_stream = Conversations - url_base = TWILIO_CONVERSATIONS_URL_BASE - data_field = "participants" - uri_from_subresource = False - - def path(self, stream_slice: Mapping[str, Any], **kwargs): - return f"Conversations/{stream_slice['conversation_sid']}/Participants" - - def parent_record_to_stream_slice(self, record: Mapping[str, Any]) -> Mapping[str, Any]: - return {"conversation_sid": record["sid"]} - - -class ConversationMessages(TwilioNestedStream): - """https://www.twilio.com/docs/conversations/api/conversation-message-resource#list-all-conversation-messages""" - - parent_stream = Conversations - url_base = TWILIO_CONVERSATIONS_URL_BASE - uri_from_subresource = False - data_field = "messages" - - def path(self, stream_slice: Mapping[str, Any], **kwargs): - return f"Conversations/{stream_slice['conversation_sid']}/Messages" - - def parent_record_to_stream_slice(self, record: Mapping[str, Any]) -> Mapping[str, Any]: - return {"conversation_sid": record["sid"]} - - -class Users(TwilioStream): - """https://www.twilio.com/docs/conversations/api/user-resource""" - - url_base = TWILIO_CONVERSATIONS_URL_BASE - - def path(self, **kwargs): - return self.name.title() - - -class UserConversations(TwilioNestedStream): - """https://www.twilio.com/docs/conversations/api/user-conversation-resource#list-all-of-a-users-conversations""" - - parent_stream = Users - url_base = TWILIO_CONVERSATIONS_URL_BASE - uri_from_subresource = False - data_field = "conversations" - primary_key = ["account_sid"] - - def path(self, stream_slice: Mapping[str, Any], **kwargs): - return f"Users/{stream_slice['user_sid']}/Conversations" - - def parent_record_to_stream_slice(self, record: Mapping[str, Any]) -> Mapping[str, Any]: - return {"user_sid": record["sid"]} diff --git a/airbyte-integrations/connectors/source-twilio/unit_tests/test_source.py b/airbyte-integrations/connectors/source-twilio/unit_tests/test_source.py deleted file mode 100644 index 566a534c2e1a..000000000000 --- a/airbyte-integrations/connectors/source-twilio/unit_tests/test_source.py +++ /dev/null @@ -1,113 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from unittest.mock import Mock - -import pytest -import requests -from source_twilio.source import SourceTwilio -from source_twilio.streams import ( - Alerts, - Applications, - AvailablePhoneNumberCountries, - AvailablePhoneNumbersLocal, - AvailablePhoneNumbersMobile, - AvailablePhoneNumbersTollFree, - Calls, - ConferenceParticipants, - Conferences, - ConversationParticipants, - Conversations, - DependentPhoneNumbers, - IncomingPhoneNumbers, - Keys, - MessageMedia, - Messages, - OutgoingCallerIds, - Queues, - Recordings, - Step, - Transcriptions, - UsageRecords, - UsageTriggers, - UserConversations, - Users, - VerifyServices, -) - - -@pytest.fixture -def config(): - return { - "account_sid": "ACdade166c12e160e9ed0a6088226718fb", - "auth_token": "d36d25e4517633af8e17dd02dd6ba21b", - "start_date": "2022-01-01T00:00:00Z", - "lookback_window": 0, - } - - -TEST_INSTANCE = SourceTwilio() - - -@pytest.mark.parametrize( - "exception, expected_error_msg", - ( - ( - ConnectionError("Connection aborted"), - "Unable to connect to Twilio API with the provided credentials - ConnectionError('Connection aborted')", - ), - ( - TimeoutError("Socket timed out"), - "Unable to connect to Twilio API with the provided credentials - TimeoutError('Socket timed out')", - ), - ( - requests.exceptions.HTTPError("401 Client Error: Unauthorized for url: https://api.twilio.com/"), - "Unable to connect to Twilio API with the provided credentials - " - "HTTPError('401 Client Error: Unauthorized for url: https://api.twilio.com/')", - ), - ), -) -def test_check_connection_handles_exceptions(mocker, config, exception, expected_error_msg): - mocker.patch.object(requests.Session, "send", Mock(side_effect=exception)) - status_ok, error = TEST_INSTANCE.check_connection(logger=None, config=config) - assert not status_ok - assert error == expected_error_msg - - -@pytest.mark.parametrize( - "stream_cls", - [ - (Alerts), - (Applications), - (AvailablePhoneNumberCountries), - (AvailablePhoneNumbersLocal), - (AvailablePhoneNumbersMobile), - (AvailablePhoneNumbersTollFree), - (Calls), - (ConferenceParticipants), - (Conferences), - (DependentPhoneNumbers), - (IncomingPhoneNumbers), - (Keys), - (MessageMedia), - (Messages), - (OutgoingCallerIds), - (Queues), - (Recordings), - (Step), - (Transcriptions), - (UsageRecords), - (UsageTriggers), - (Conversations), - (ConversationParticipants), - (Users), - (UserConversations), - (VerifyServices), - ], -) -def test_streams(stream_cls, config): - streams = TEST_INSTANCE.streams(config) - for stream in streams: - if stream_cls in streams: - assert isinstance(stream, stream_cls) diff --git a/airbyte-integrations/connectors/source-twilio/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-twilio/unit_tests/test_streams.py deleted file mode 100644 index 309355222b86..000000000000 --- a/airbyte-integrations/connectors/source-twilio/unit_tests/test_streams.py +++ /dev/null @@ -1,333 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from contextlib import nullcontext -from unittest.mock import patch - -import pendulum -import pytest -import requests -from airbyte_cdk.sources.streams.http import HttpStream -from freezegun import freeze_time -from source_twilio.auth import HttpBasicAuthenticator -from source_twilio.source import SourceTwilio -from source_twilio.streams import ( - Accounts, - Addresses, - Alerts, - Calls, - DependentPhoneNumbers, - MessageMedia, - Messages, - Recordings, - TwilioNestedStream, - TwilioStream, - UsageRecords, - UsageTriggers, -) - -TEST_CONFIG = { - "account_sid": "airbyte.io", - "auth_token": "secret", - "start_date": "2022-01-01T00:00:00Z", - "lookback_window": 0, -} -TEST_CONFIG.update( - **{ - "authenticator": HttpBasicAuthenticator((TEST_CONFIG["account_sid"], TEST_CONFIG["auth_token"])), - } -) - -TEST_INSTANCE = SourceTwilio() - - -class TestTwilioStream: - - CONFIG = {"authenticator": TEST_CONFIG.get("authenticator")} - - @pytest.mark.parametrize( - "stream_cls, expected", - [ - (Accounts, "accounts"), - ], - ) - def test_data_field(self, stream_cls, expected): - stream = stream_cls(**self.CONFIG) - result = stream.data_field - assert result == expected - - @pytest.mark.parametrize( - "stream_cls, expected", - [ - (Accounts, ['name']), - ], - ) - def test_changeable_fields(self, stream_cls, expected): - with patch.object(Accounts, "changeable_fields", ['name']): - stream = stream_cls(**self.CONFIG) - result = stream.changeable_fields - assert result == expected - - @pytest.mark.parametrize( - "stream_cls, expected", - [ - (Accounts, "Accounts.json"), - ], - ) - def test_path(self, stream_cls, expected): - stream = stream_cls(**self.CONFIG) - result = stream.path() - assert result == expected - - @pytest.mark.parametrize( - "stream_cls, test_response, expected", - [ - ( - Accounts, - { - "next_page_uri": "/2010-04-01/Accounts/ACdad/Addresses.json?PageSize=1000&Page=2&PageToken=PAAD42931b949c0dedce94b2f93847fdcf95" - }, - {"Page": "2", "PageSize": "1000", "PageToken": "PAAD42931b949c0dedce94b2f93847fdcf95"}, - ), - ], - ) - def test_next_page_token(self, requests_mock, stream_cls, test_response, expected): - stream = stream_cls(**self.CONFIG) - url = f"{stream.url_base}{stream.path()}" - requests_mock.get(url, json=test_response) - response = requests.get(url) - result = stream.next_page_token(response) - assert result == expected - - @pytest.mark.parametrize( - "stream_cls, test_response, expected", - [ - (Accounts, {"accounts": [{"id": "123", "name": "test"}]}, [{"id": "123"}]), - ], - ) - def test_parse_response(self, requests_mock, stream_cls, test_response, expected): - with patch.object(TwilioStream, "changeable_fields", ["name"]): - stream = stream_cls(**self.CONFIG) - url = f"{stream.url_base}{stream.path()}" - requests_mock.get(url, json=test_response) - response = requests.get(url) - result = list(stream.parse_response(response)) - assert result[0]['id'] == expected[0]['id'] - - @pytest.mark.parametrize( - "stream_cls, expected", - [ - (Accounts, "5.5"), - ], - ) - def test_backoff_time(self, requests_mock, stream_cls, expected): - stream = stream_cls(**self.CONFIG) - url = f"{stream.url_base}{stream.path()}" - test_headers = {"Retry-After": expected} - requests_mock.get(url, headers=test_headers) - response = requests.get(url) - result = stream.backoff_time(response) - assert result == float(expected) - - @pytest.mark.parametrize( - "stream_cls, next_page_token, expected", - [ - ( - Accounts, - {"Page": "2", "PageSize": "1000", "PageToken": "PAAD42931b949c0dedce94b2f93847fdcf95"}, - {"Page": "2", "PageSize": "1000", "PageToken": "PAAD42931b949c0dedce94b2f93847fdcf95"}, - ), - ], - ) - def test_request_params(self, stream_cls, next_page_token, expected): - stream = stream_cls(**self.CONFIG) - result = stream.request_params(stream_state=None, next_page_token=next_page_token) - assert result == expected - - @pytest.mark.parametrize( - "original_value, field_schema, expected_value", - [ - ("Fri, 11 Dec 2020 04:28:40 +0000", {"format": "date-time"}, "2020-12-11T04:28:40Z"), - ("2020-12-11T04:28:40Z", {"format": "date-time"}, "2020-12-11T04:28:40Z"), - ("some_string", {}, "some_string"), - ] - ) - def test_transform_function(self, original_value, field_schema, expected_value): - assert Accounts.custom_transform_function(original_value, field_schema) == expected_value - - -class TestIncrementalTwilioStream: - - CONFIG = TEST_CONFIG - CONFIG.pop("account_sid") - CONFIG.pop("auth_token") - - @pytest.mark.parametrize( - "stream_cls, stream_slice, next_page_token, expected", - [ - ( - Calls, - {"EndTime>": "2022-01-01", "EndTime<": "2022-01-02"}, - {"Page": "2", "PageSize": "1000", "PageToken": "PAAD42931b949c0dedce94b2f93847fdcf95"}, - { - "EndTime>": "2022-01-01", - "EndTime<": "2022-01-02", - "Page": "2", - "PageSize": "1000", - "PageToken": "PAAD42931b949c0dedce94b2f93847fdcf95", - }, - ), - ], - ) - def test_request_params(self, stream_cls, stream_slice, next_page_token, expected): - stream = stream_cls(**self.CONFIG) - result = stream.request_params(stream_state=None, stream_slice=stream_slice, next_page_token=next_page_token) - assert result == expected - - @pytest.mark.parametrize( - "stream_cls, record, expected", - [ - (Calls, [{"end_time": "2022-02-01T00:00:00Z"}], [{"end_time": "2022-02-01T00:00:00Z"}]), - ], - ) - def test_read_records(self, stream_cls, record, expected): - stream = stream_cls(**self.CONFIG) - with patch.object(HttpStream, "read_records", return_value=record): - result = stream.read_records(sync_mode=None) - assert list(result) == expected - - @pytest.mark.parametrize( - "stream_cls, parent_cls_records, extra_slice_keywords", - [ - (Calls, [{"subresource_uris": {"calls": "123"}}, {"subresource_uris": {"calls": "124"}}], ["subresource_uri"]), - (Alerts, [{}], []), - ], - ) - def test_stream_slices(self, mocker, stream_cls, parent_cls_records, extra_slice_keywords): - stream = stream_cls( - authenticator=TEST_CONFIG.get("authenticator"), start_date=pendulum.now().subtract(months=13).to_iso8601_string() - ) - expected_slices = 2 * len(parent_cls_records) # 2 per year slices per each parent slice - if isinstance(stream, TwilioNestedStream): - slices_mock_context = mocker.patch.object(stream.parent_stream_instance, "stream_slices", return_value=[{}]) - records_mock_context = mocker.patch.object(stream.parent_stream_instance, "read_records", return_value=parent_cls_records) - else: - slices_mock_context, records_mock_context = nullcontext(), nullcontext() - with slices_mock_context: - with records_mock_context: - slices = list(stream.stream_slices(sync_mode="incremental")) - assert len(slices) == expected_slices - for slice_ in slices: - if isinstance(stream, TwilioNestedStream): - for kw in extra_slice_keywords: - assert kw in slice_ - assert slice_[stream.lower_boundary_filter_field] <= slice_[stream.upper_boundary_filter_field] - - @freeze_time("2022-11-16 12:03:11+00:00") - @pytest.mark.parametrize( - "stream_cls, state, expected_dt_ranges", - ( - ( - Messages, - {"date_sent": "2022-11-13 23:39:00"}, - [ - {"DateSent>": "2022-11-13 23:39:00Z", "DateSent<": "2022-11-14 23:39:00Z"}, - {"DateSent>": "2022-11-14 23:39:00Z", "DateSent<": "2022-11-15 23:39:00Z"}, - {"DateSent>": "2022-11-15 23:39:00Z", "DateSent<": "2022-11-16 12:03:11Z"}, - ], - ), - (UsageRecords, {"start_date": "2021-11-16 00:00:00"}, [{"StartDate": "2021-11-16", "EndDate": "2022-11-16"}]), - ( - Recordings, - {"date_created": "2021-11-16 00:00:00"}, - [ - {"DateCreated>": "2021-11-16 00:00:00Z", "DateCreated<": "2022-11-16 00:00:00Z"}, - {"DateCreated>": "2022-11-16 00:00:00Z", "DateCreated<": "2022-11-16 12:03:11Z"}, - ], - ), - ), - ) - def test_generate_dt_ranges(self, stream_cls, state, expected_dt_ranges): - stream = stream_cls(authenticator=TEST_CONFIG.get("authenticator"), start_date="2000-01-01 00:00:00") - stream.state = state - dt_ranges = list(stream.generate_date_ranges()) - assert dt_ranges == expected_dt_ranges - - -class TestTwilioNestedStream: - - CONFIG = {"authenticator": TEST_CONFIG.get("authenticator")} - - @pytest.mark.parametrize( - "stream_cls, expected", - [ - (Addresses, {}), - (DependentPhoneNumbers, {}), - (MessageMedia, {"num_media": "0"}), - ], - ) - def test_media_exist_validation(self, stream_cls, expected): - stream = stream_cls(**self.CONFIG) - result = stream.media_exist_validation - assert result == expected - - @pytest.mark.parametrize( - "stream_cls, parent_stream, record, expected", - [ - ( - Addresses, - Accounts, - [{"subresource_uris": {"addresses": "123"}}], - [{"subresource_uri": "123"}], - ), - ( - DependentPhoneNumbers, - Addresses, - [{"subresource_uris": {"addresses": "123"}, "sid": "123", "account_sid": "456"}], - [{"sid": "123", "account_sid": "456"}], - ), - ], - ) - def test_stream_slices(self, stream_cls, parent_stream, record, expected): - stream = stream_cls(**self.CONFIG) - with patch.object(Accounts, "read_records", return_value=record): - with patch.object(parent_stream, "stream_slices", return_value=record): - with patch.object(parent_stream, "read_records", return_value=record): - result = stream.stream_slices(sync_mode="full_refresh") - assert list(result) == expected - - -class TestUsageNestedStream: - - CONFIG = {"authenticator": TEST_CONFIG.get("authenticator")} - - @pytest.mark.parametrize( - "stream_cls, expected", - [ - (UsageTriggers, "Triggers"), - ], - ) - def test_path_name(self, stream_cls, expected): - stream = stream_cls(**self.CONFIG) - result = stream.path_name - assert result == expected - - @pytest.mark.parametrize( - "stream_cls, parent_stream, record, expected", - [ - ( - UsageTriggers, - Accounts, - [{"sid": "234", "account_sid": "678"}], - [{"account_sid": "234"}], - ), - ], - ) - def test_stream_slices(self, stream_cls, parent_stream, record, expected): - stream = stream_cls(**self.CONFIG) - with patch.object(Accounts, "read_records", return_value=record): - with patch.object(parent_stream, "stream_slices", return_value=record): - with patch.object(parent_stream, "read_records", return_value=record): - result = stream.stream_slices() - assert list(result) == expected From dd642a932cb81a63d1eb518f2e24ebb0c1734a6f Mon Sep 17 00:00:00 2001 From: "cristina.mariscal" Date: Thu, 27 Jun 2024 11:27:40 -0600 Subject: [PATCH 09/37] fixes on yaml --- .../source-twilio/source_twilio/manifest.yaml | 40 +++++++++---------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/airbyte-integrations/connectors/source-twilio/source_twilio/manifest.yaml b/airbyte-integrations/connectors/source-twilio/source_twilio/manifest.yaml index cbafd8ae2963..33ef2ed7cdfc 100644 --- a/airbyte-integrations/connectors/source-twilio/source_twilio/manifest.yaml +++ b/airbyte-integrations/connectors/source-twilio/source_twilio/manifest.yaml @@ -273,7 +273,7 @@ definitions: $ref: "#/schemas/Available Phone Numbers Toll-Free" Incoming Phone Numbers: type: DeclarativeStream - name: incoming_phone_pumbers + name: incoming_phone_numbers retriever: type: SimpleRetriever requester: @@ -390,9 +390,9 @@ definitions: type: InlineSchemaLoader schema: $ref: "#/schemas/Flows" - Steps: + Step: type: DeclarativeStream - name: steps + name: step primary_key: - sid retriever: @@ -407,7 +407,7 @@ definitions: extractor: type: DpathExtractor field_path: - - steps + - step paginator: type: DefaultPaginator page_token_option: @@ -440,7 +440,7 @@ definitions: schema_loader: type: InlineSchemaLoader schema: - $ref: "#/schemas/Steps" + $ref: "#/schemas/Step" Alerts: type: DeclarativeStream name: alerts @@ -603,7 +603,7 @@ definitions: type: DatetimeBasedCursor cursor_field: end_time cursor_datetime_formats: - - "%Y-%m-%d" + - "%a, %d %b %Y %H:%M:%S %z" datetime_format: "%Y-%m-%d" start_datetime: type: MinMaxDatetime @@ -670,7 +670,7 @@ definitions: type: DatetimeBasedCursor cursor_field: date_created cursor_datetime_formats: - - "%Y-%m-%d" + - "%a, %d %b %Y %H:%M:%S %z" datetime_format: "%Y-%m-%d" start_datetime: type: MinMaxDatetime @@ -879,7 +879,7 @@ definitions: type: DatetimeBasedCursor cursor_field: date_created cursor_datetime_formats: - - "%Y-%m-%d" + - "%a, %d %b %Y %H:%M:%S %z" datetime_format: "%Y-%m-%d" start_datetime: type: MinMaxDatetime @@ -1128,7 +1128,7 @@ definitions: type: DatetimeBasedCursor cursor_field: date_sent cursor_datetime_formats: - - "%Y-%m-%d" + - "%a, %d %b %Y %H:%M:%S %z" datetime_format: "%Y-%m-%d" start_datetime: type: MinMaxDatetime @@ -1203,7 +1203,7 @@ definitions: type: DatetimeBasedCursor cursor_field: date_sent cursor_datetime_formats: - - "%Y-%m-%d" + - "%a, %d %b %Y %H:%M:%S %z" datetime_format: "%Y-%m-%d" start_datetime: type: MinMaxDatetime @@ -1270,7 +1270,7 @@ definitions: type: DatetimeBasedCursor cursor_field: start_date cursor_datetime_formats: - - "%Y-%m-%d" + - "%a, %d %b %Y %H:%M:%S %z" datetime_format: "%Y-%m-%d" start_datetime: type: MinMaxDatetime @@ -1441,9 +1441,9 @@ definitions: type: InlineSchemaLoader schema: $ref: "#/schemas/Users" - User Conversation: + User Conversations: type: DeclarativeStream - name: user_conversation + name: user_conversations primary_key: - conversations retriever: @@ -1470,7 +1470,7 @@ definitions: schema_loader: type: InlineSchemaLoader schema: - $ref: "#/schemas/User Conversation" + $ref: "#/schemas/User Conversations" streams: - $ref: "#/definitions/streams/Accounts" @@ -1482,7 +1482,7 @@ streams: - $ref: "#/definitions/streams/Incoming Phone Numbers" - $ref: "#/definitions/streams/Keys" - $ref: "#/definitions/streams/Flows" - - $ref: "#/definitions/streams/Steps" + - $ref: "#/definitions/streams/Step" - $ref: "#/definitions/streams/Alerts" - $ref: "#/definitions/streams/Applications" - $ref: "#/definitions/streams/Dependent Phone Numbers" @@ -1506,7 +1506,7 @@ streams: - $ref: "#/definitions/streams/Conversation Participants" - $ref: "#/definitions/streams/Conversation Messages" - $ref: "#/definitions/streams/Users" - - $ref: "#/definitions/streams/User Conversation" + - $ref: "#/definitions/streams/User Conversations" spec: type: Spec @@ -1569,7 +1569,7 @@ metadata: Incoming Phone Numbers: false Keys: false Flows: false - Steps: false + Step: false Alerts: false Applications: false Dependent Phone Numbers: false @@ -1593,7 +1593,7 @@ metadata: Conversation Participants: false Conversation Messages: false Users: false - User Conversation: false + User Conversations: false schemas: Accounts: @@ -2467,7 +2467,7 @@ schemas: type: - "null" - integer - Steps: + Step: type: object $schema: http://json-schema.org/schema# additionalProperties: true @@ -5062,7 +5062,7 @@ schemas: - string required: - sid - User Conversation: + User Conversations: type: object $schema: http://json-schema.org/schema# additionalProperties: true From 72fa32aeeb72afd457e83308e687beb516e5bd10 Mon Sep 17 00:00:00 2001 From: "cristina.mariscal" Date: Thu, 27 Jun 2024 11:55:26 -0600 Subject: [PATCH 10/37] fixed dates --- .../source-twilio/source_twilio/manifest.yaml | 24 ++++++++++--------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/airbyte-integrations/connectors/source-twilio/source_twilio/manifest.yaml b/airbyte-integrations/connectors/source-twilio/source_twilio/manifest.yaml index 33ef2ed7cdfc..c733681fb6be 100644 --- a/airbyte-integrations/connectors/source-twilio/source_twilio/manifest.yaml +++ b/airbyte-integrations/connectors/source-twilio/source_twilio/manifest.yaml @@ -603,7 +603,7 @@ definitions: type: DatetimeBasedCursor cursor_field: end_time cursor_datetime_formats: - - "%a, %d %b %Y %H:%M:%S %z" + - "%Y-%m-%dT%H:%M:%SZ" datetime_format: "%Y-%m-%d" start_datetime: type: MinMaxDatetime @@ -670,7 +670,7 @@ definitions: type: DatetimeBasedCursor cursor_field: date_created cursor_datetime_formats: - - "%a, %d %b %Y %H:%M:%S %z" + - "%Y-%m-%dT%H:%M:%SZ" datetime_format: "%Y-%m-%d" start_datetime: type: MinMaxDatetime @@ -696,7 +696,8 @@ definitions: type: DeclarativeStream name: conference_participants primary_key: - - sid + - account_sid + - conference_sid retriever: type: SimpleRetriever requester: @@ -879,7 +880,7 @@ definitions: type: DatetimeBasedCursor cursor_field: date_created cursor_datetime_formats: - - "%a, %d %b %Y %H:%M:%S %z" + - "%Y-%m-%dT%H:%M:%SZ" datetime_format: "%Y-%m-%d" start_datetime: type: MinMaxDatetime @@ -1128,7 +1129,7 @@ definitions: type: DatetimeBasedCursor cursor_field: date_sent cursor_datetime_formats: - - "%a, %d %b %Y %H:%M:%S %z" + - "%Y-%m-%dT%H:%M:%SZ" datetime_format: "%Y-%m-%d" start_datetime: type: MinMaxDatetime @@ -1201,9 +1202,9 @@ definitions: $ref: "#/definitions/streams/Messages" incremental_sync: type: DatetimeBasedCursor - cursor_field: date_sent + cursor_field: date_created cursor_datetime_formats: - - "%a, %d %b %Y %H:%M:%S %z" + - "%Y-%m-%dT%H:%M:%SZ" datetime_format: "%Y-%m-%d" start_datetime: type: MinMaxDatetime @@ -1229,7 +1230,8 @@ definitions: type: DeclarativeStream name: usage_records primary_key: - - sid + - account_sid + - category retriever: type: SimpleRetriever requester: @@ -1270,7 +1272,7 @@ definitions: type: DatetimeBasedCursor cursor_field: start_date cursor_datetime_formats: - - "%a, %d %b %Y %H:%M:%S %z" + - "%Y-%m-%dT%H:%M:%SZ" datetime_format: "%Y-%m-%d" start_datetime: type: MinMaxDatetime @@ -1445,7 +1447,7 @@ definitions: type: DeclarativeStream name: user_conversations primary_key: - - conversations + - account_sid retriever: type: SimpleRetriever requester: @@ -2632,7 +2634,7 @@ schemas: type: DatetimeBasedCursor cursor_field: start_date cursor_datetime_formats: - - "%a, %d %b %Y %H:%M:%S %z" + - "%Y-%m-%dT%H:%M:%SZ" datetime_format: "%a, %d %b %Y %H:%M:%S %z" start_datetime: type: MinMaxDatetime From b32c4841a2b408e158f1126163c54cf24609110b Mon Sep 17 00:00:00 2001 From: "cristina.mariscal" Date: Thu, 27 Jun 2024 17:10:41 -0600 Subject: [PATCH 11/37] fixing datetimes of increment --- .../source-twilio/source_twilio/manifest.yaml | 128 +++++++++--------- 1 file changed, 64 insertions(+), 64 deletions(-) diff --git a/airbyte-integrations/connectors/source-twilio/source_twilio/manifest.yaml b/airbyte-integrations/connectors/source-twilio/source_twilio/manifest.yaml index c733681fb6be..2a6c3759c2fd 100644 --- a/airbyte-integrations/connectors/source-twilio/source_twilio/manifest.yaml +++ b/airbyte-integrations/connectors/source-twilio/source_twilio/manifest.yaml @@ -456,6 +456,28 @@ definitions: type: DpathExtractor field_path: - alerts + incremental_sync: + type: DatetimeBasedCursor + cursor_field: date_generated + cursor_datetime_formats: + - '%a, %d %b %Y %H:%M:%S %z' + datetime_format: '%Y-%m-%d' + start_datetime: + type: MinMaxDatetime + datetime: '{{ config["start_date"] }}' + datetime_format: '%Y-%m-%dT%H:%M:%SZ' + start_time_option: + type: RequestOption + field_name: StartDate= + inject_into: request_parameter + end_time_option: + type: RequestOption + field_name: EndDate= + inject_into: request_parameter + end_datetime: + type: MinMaxDatetime + datetime: '{{ now_utc().strftime(''%Y-%m-%dT%H:%M:%SZ'') }}' + datetime_format: '%Y-%m-%dT%H:%M:%SZ' schema_loader: type: InlineSchemaLoader schema: @@ -603,24 +625,24 @@ definitions: type: DatetimeBasedCursor cursor_field: end_time cursor_datetime_formats: - - "%Y-%m-%dT%H:%M:%SZ" - datetime_format: "%Y-%m-%d" + - '%a, %d %b %Y %H:%M:%S %z' + datetime_format: '%Y-%m-%d' start_datetime: type: MinMaxDatetime datetime: '{{ config["start_date"] }}' - datetime_format: "%Y-%m-%dT%H:%M:%SZ" + datetime_format: '%Y-%m-%dT%H:%M:%SZ' start_time_option: type: RequestOption - inject_into: request_parameter field_name: EndTime> + inject_into: request_parameter end_time_option: type: RequestOption - inject_into: request_parameter field_name: EndTime< + inject_into: request_parameter end_datetime: type: MinMaxDatetime - datetime: "{{ now_utc().strftime('%Y-%m-%dT%H:%M:%SZ') }}" - datetime_format: "%Y-%m-%dT%H:%M:%SZ" + datetime: '{{ now_utc().strftime(''%Y-%m-%dT%H:%M:%SZ'') }}' + datetime_format: '%Y-%m-%dT%H:%M:%SZ' schema_loader: type: InlineSchemaLoader schema: @@ -670,24 +692,24 @@ definitions: type: DatetimeBasedCursor cursor_field: date_created cursor_datetime_formats: - - "%Y-%m-%dT%H:%M:%SZ" - datetime_format: "%Y-%m-%d" + - '%a, %d %b %Y %H:%M:%S %z' + datetime_format: '%Y-%m-%d' start_datetime: type: MinMaxDatetime datetime: '{{ config["start_date"] }}' - datetime_format: "%Y-%m-%dT%H:%M:%SZ" + datetime_format: '%Y-%m-%dT%H:%M:%SZ' start_time_option: type: RequestOption - inject_into: request_parameter field_name: DateCreated> + inject_into: request_parameter end_time_option: type: RequestOption - inject_into: request_parameter field_name: DateCreated< + inject_into: request_parameter end_datetime: type: MinMaxDatetime - datetime: "{{ now_utc().strftime('%Y-%m-%dT%H:%M:%SZ') }}" - datetime_format: "%Y-%m-%dT%H:%M:%SZ" + datetime: '{{ now_utc().strftime(''%Y-%m-%dT%H:%M:%SZ'') }}' + datetime_format: '%Y-%m-%dT%H:%M:%SZ' schema_loader: type: InlineSchemaLoader schema: @@ -880,24 +902,24 @@ definitions: type: DatetimeBasedCursor cursor_field: date_created cursor_datetime_formats: - - "%Y-%m-%dT%H:%M:%SZ" - datetime_format: "%Y-%m-%d" + - '%a, %d %b %Y %H:%M:%S %z' + datetime_format: '%Y-%m-%d' start_datetime: type: MinMaxDatetime datetime: '{{ config["start_date"] }}' - datetime_format: "%Y-%m-%dT%H:%M:%SZ" + datetime_format: '%Y-%m-%dT%H:%M:%SZ' start_time_option: type: RequestOption - inject_into: request_parameter field_name: DateCreated> + inject_into: request_parameter end_time_option: type: RequestOption - inject_into: request_parameter field_name: DateCreated< + inject_into: request_parameter end_datetime: type: MinMaxDatetime - datetime: "{{ now_utc().strftime('%Y-%m-%dT%H:%M:%SZ') }}" - datetime_format: "%Y-%m-%dT%H:%M:%SZ" + datetime: '{{ now_utc().strftime(''%Y-%m-%dT%H:%M:%SZ'') }}' + datetime_format: '%Y-%m-%dT%H:%M:%SZ' schema_loader: type: InlineSchemaLoader schema: @@ -1129,24 +1151,24 @@ definitions: type: DatetimeBasedCursor cursor_field: date_sent cursor_datetime_formats: - - "%Y-%m-%dT%H:%M:%SZ" - datetime_format: "%Y-%m-%d" + - '%a, %d %b %Y %H:%M:%S %z' + datetime_format: '%Y-%m-%d' start_datetime: type: MinMaxDatetime datetime: '{{ config["start_date"] }}' - datetime_format: "%Y-%m-%dT%H:%M:%SZ" + datetime_format: '%Y-%m-%dT%H:%M:%SZ' start_time_option: type: RequestOption - inject_into: request_parameter field_name: DateSent> + inject_into: request_parameter end_time_option: type: RequestOption - inject_into: request_parameter field_name: DateSent< + inject_into: request_parameter end_datetime: type: MinMaxDatetime - datetime: "{{ now_utc().strftime('%Y-%m-%dT%H:%M:%SZ') }}" - datetime_format: "%Y-%m-%dT%H:%M:%SZ" + datetime: '{{ now_utc().strftime(''%Y-%m-%dT%H:%M:%SZ'') }}' + datetime_format: '%Y-%m-%dT%H:%M:%SZ' schema_loader: type: InlineSchemaLoader schema: @@ -1204,24 +1226,24 @@ definitions: type: DatetimeBasedCursor cursor_field: date_created cursor_datetime_formats: - - "%Y-%m-%dT%H:%M:%SZ" - datetime_format: "%Y-%m-%d" + - '%a, %d %b %Y %H:%M:%S %z' + datetime_format: '%Y-%m-%d' start_datetime: type: MinMaxDatetime datetime: '{{ config["start_date"] }}' - datetime_format: "%Y-%m-%dT%H:%M:%SZ" + datetime_format: '%Y-%m-%dT%H:%M:%SZ' start_time_option: type: RequestOption + field_name: DateCreated> inject_into: request_parameter - field_name: DateSent> end_time_option: type: RequestOption + field_name: DateCreated< inject_into: request_parameter - field_name: DateSent< end_datetime: type: MinMaxDatetime - datetime: "{{ now_utc().strftime('%Y-%m-%dT%H:%M:%SZ') }}" - datetime_format: "%Y-%m-%dT%H:%M:%SZ" + datetime: '{{ now_utc().strftime(''%Y-%m-%dT%H:%M:%SZ'') }}' + datetime_format: '%Y-%m-%dT%H:%M:%SZ' schema_loader: type: InlineSchemaLoader schema: @@ -1272,24 +1294,24 @@ definitions: type: DatetimeBasedCursor cursor_field: start_date cursor_datetime_formats: - - "%Y-%m-%dT%H:%M:%SZ" - datetime_format: "%Y-%m-%d" + - '%a, %d %b %Y %H:%M:%S %z' + datetime_format: '%Y-%m-%d' start_datetime: type: MinMaxDatetime datetime: '{{ config["start_date"] }}' - datetime_format: "%Y-%m-%dT%H:%M:%SZ" + datetime_format: '%Y-%m-%dT%H:%M:%SZ' start_time_option: type: RequestOption + field_name: StartDate> inject_into: request_parameter - field_name: StartDate end_time_option: type: RequestOption + field_name: EndDate< inject_into: request_parameter - field_name: EndDate end_datetime: type: MinMaxDatetime - datetime: "{{ now_utc().strftime('%Y-%m-%dT%H:%M:%SZ') }}" - datetime_format: "%Y-%m-%dT%H:%M:%SZ" + datetime: '{{ now_utc().strftime(''%Y-%m-%dT%H:%M:%SZ'') }}' + datetime_format: '%Y-%m-%dT%H:%M:%SZ' schema_loader: type: InlineSchemaLoader schema: @@ -2630,28 +2652,6 @@ schemas: type: - "null" - string - incremental_sync: - type: DatetimeBasedCursor - cursor_field: start_date - cursor_datetime_formats: - - "%Y-%m-%dT%H:%M:%SZ" - datetime_format: "%a, %d %b %Y %H:%M:%S %z" - start_datetime: - type: MinMaxDatetime - datetime: '{{ config["start_date"] }}' - datetime_format: "%Y-%m-%dT%H:%M:%SZ" - start_time_option: - type: RequestOption - field_name: StartDate - inject_into: request_parameter - end_time_option: - type: RequestOption - field_name: EndDate - inject_into: request_parameter - end_datetime: - type: MinMaxDatetime - datetime: "{{ now_utc().strftime('%Y-%m-%dT%H:%M:%SZ') }}" - datetime_format: "%Y-%m-%dT%H:%M:%SZ" Applications: type: object $schema: http://json-schema.org/schema# From 4ef3797ccc2111f018fe1c682156817a4036ab53 Mon Sep 17 00:00:00 2001 From: "cristina.mariscal" Date: Fri, 28 Jun 2024 16:01:35 -0600 Subject: [PATCH 12/37] WIP LOCAL DO NOT PUSH --- .../connectors/source-twilio/migrations.py | 33 +++++ .../source-twilio/unit_tests copy/conftest.py | 42 +++++++ .../unit_tests copy/integration/__init__.py | 0 .../unit_tests copy/integration/config.py | 16 +++ .../integration/test_automations.py | 119 ++++++++++++++++++ .../resource/http/response/automations.json | 68 ++++++++++ ...mponent_custom_email_activity_extractor.py | 66 ++++++++++ .../test_config_datacenter_migration.py | 37 ++++++ .../test_configs/test_config_api_key.json | 4 + .../test_configs/test_config_oauth.json | 8 ++ 10 files changed, 393 insertions(+) create mode 100644 airbyte-integrations/connectors/source-twilio/migrations.py create mode 100644 airbyte-integrations/connectors/source-twilio/unit_tests copy/conftest.py create mode 100644 airbyte-integrations/connectors/source-twilio/unit_tests copy/integration/__init__.py create mode 100644 airbyte-integrations/connectors/source-twilio/unit_tests copy/integration/config.py create mode 100644 airbyte-integrations/connectors/source-twilio/unit_tests copy/integration/test_automations.py create mode 100644 airbyte-integrations/connectors/source-twilio/unit_tests copy/resource/http/response/automations.json create mode 100644 airbyte-integrations/connectors/source-twilio/unit_tests copy/test_component_custom_email_activity_extractor.py create mode 100644 airbyte-integrations/connectors/source-twilio/unit_tests copy/test_config_datacenter_migration.py create mode 100644 airbyte-integrations/connectors/source-twilio/unit_tests copy/test_configs/test_config_api_key.json create mode 100644 airbyte-integrations/connectors/source-twilio/unit_tests copy/test_configs/test_config_oauth.json diff --git a/airbyte-integrations/connectors/source-twilio/migrations.py b/airbyte-integrations/connectors/source-twilio/migrations.py new file mode 100644 index 000000000000..df9c77db0179 --- /dev/null +++ b/airbyte-integrations/connectors/source-twilio/migrations.py @@ -0,0 +1,33 @@ +import logging +from typing import Any, List, Mapping + +import requests +from airbyte_cdk.config_observation import create_connector_config_control_message +from airbyte_cdk.entrypoint import AirbyteEntrypoint +from airbyte_cdk.sources import Source +from airbyte_cdk.utils import AirbyteTracedException +from airbyte_protocol.models import FailureType + +logger = logging.getLogger("airbyte_logger") + + +class MigrateDataCenter: + """ + This class stands for migrating the state at runtime, + Set stream state partition property in config based on credential type. + """ + + STREAMS = ["calls"] + + @classmethod + def migrate(cls, args: List[str], source: Source) -> None: + """ + Orchestrates the migration process. + Args: + - args (List[str]): List of command-line arguments. + - source (Source): The data source. + """ + config_path = AirbyteEntrypoint(source).extract_config(args) + if config_path: + config = source.read_config(config_path) + cls.emit_control_message(cls.modify_and_save(config_path, source, config)) diff --git a/airbyte-integrations/connectors/source-twilio/unit_tests copy/conftest.py b/airbyte-integrations/connectors/source-twilio/unit_tests copy/conftest.py new file mode 100644 index 000000000000..c387004a5110 --- /dev/null +++ b/airbyte-integrations/connectors/source-twilio/unit_tests copy/conftest.py @@ -0,0 +1,42 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from pytest import fixture + + +@fixture(name="data_center") +def data_center_fixture(): + return "some_dc" + + +@fixture(name="config") +def config_fixture(data_center): + return {"apikey": f"API_KEY-{data_center}", "start_date": "2022-01-01T00:00:00.000Z"} + + +@fixture(name="access_token") +def access_token_fixture(): + return "some_access_token" + + +@fixture(name="oauth_config") +def oauth_config_fixture(access_token): + return { + "credentials": { + "auth_type": "oauth2.0", + "client_id": "111111111", + "client_secret": "secret_1111111111", + "access_token": access_token, + } + } + + +@fixture(name="apikey_config") +def apikey_config_fixture(data_center): + return {"credentials": {"auth_type": "apikey", "apikey": f"some_api_key-{data_center}"}} + + +@fixture(name="wrong_config") +def wrong_config_fixture(): + return {"credentials": {"auth_type": "not auth_type"}} diff --git a/airbyte-integrations/connectors/source-twilio/unit_tests copy/integration/__init__.py b/airbyte-integrations/connectors/source-twilio/unit_tests copy/integration/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/airbyte-integrations/connectors/source-twilio/unit_tests copy/integration/config.py b/airbyte-integrations/connectors/source-twilio/unit_tests copy/integration/config.py new file mode 100644 index 000000000000..7e363fea9677 --- /dev/null +++ b/airbyte-integrations/connectors/source-twilio/unit_tests copy/integration/config.py @@ -0,0 +1,16 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +from datetime import datetime +from typing import Any, Dict + + +class ConfigBuilder: + def __init__(self) -> None: + self._config: Dict[str, Any] = {"credentials": {"auth_type": "apikey", "apikey": "Mailchimp_token-us10"}, "data_center": "us10"} + + def with_start_date(self, start_datetime: datetime) -> "ConfigBuilder": + self._config["start_date"] = start_datetime.isoformat()[:-3] + "Z" + return self + + def build(self) -> Dict[str, Any]: + return self._config diff --git a/airbyte-integrations/connectors/source-twilio/unit_tests copy/integration/test_automations.py b/airbyte-integrations/connectors/source-twilio/unit_tests copy/integration/test_automations.py new file mode 100644 index 000000000000..602562aa2db4 --- /dev/null +++ b/airbyte-integrations/connectors/source-twilio/unit_tests copy/integration/test_automations.py @@ -0,0 +1,119 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +import datetime +import json +from unittest import TestCase + +import freezegun +from airbyte_cdk.models import SyncMode +from airbyte_cdk.test.catalog_builder import CatalogBuilder +from airbyte_cdk.test.entrypoint_wrapper import read +from airbyte_cdk.test.mock_http import HttpMocker, HttpRequest, HttpResponse +from airbyte_cdk.test.mock_http.response_builder import find_template +from airbyte_cdk.test.state_builder import StateBuilder +from source_mailchimp import SourceMailchimp + +from .config import ConfigBuilder + +_CONFIG = ConfigBuilder().with_start_date(datetime.datetime(2023, 1, 1, 0, 0, 0, 1000)).build() + + +def _create_catalog(sync_mode: SyncMode = SyncMode.full_refresh): + return CatalogBuilder().with_stream(name="automations", sync_mode=sync_mode).build() + + +@freezegun.freeze_time("2023-01-31T23:59:59.001000Z") +class AutomationsTest(TestCase): + def setUp(self) -> None: + """Base setup for all tests. Enter test mocker.""" + + self.r_mock = HttpMocker() + self.r_mock.__enter__() + + def teardown(self): + """Stops and resets HttpMocker instance.""" + self.r_mock.__exit__() + + def test_read_full_refresh_no_pagination(self): + """Ensure http integration and record extraction""" + self.r_mock.get( + HttpRequest( + url="https://us10.api.mailchimp.com/3.0/automations", + query_params={ + "sort_field": "create_time", + "sort_dir": "ASC", + "exclude_fields": "automations._links", + "count": 1000, + "since_create_time": "2022-12-31T23:59:59.001000Z", + "before_create_time": "2023-01-31T23:59:59.001000Z", + }, + ), + HttpResponse(json.dumps(find_template("automations", __file__)), 200), + ) + + source = SourceMailchimp() + actual_messages = read(source, config=_CONFIG, catalog=_create_catalog()) + + assert len(actual_messages.records) == 1 + + def test_full_refresh_with_pagination(self): + """Ensure pagination""" + self.r_mock.get( + HttpRequest( + url="https://us10.api.mailchimp.com/3.0/automations", + query_params={ + "sort_field": "create_time", + "sort_dir": "ASC", + "exclude_fields": "automations._links", + "count": 1000, + "since_create_time": "2022-12-31T23:59:59.001000Z", + "before_create_time": "2023-01-31T23:59:59.001000Z", + }, + ), + HttpResponse(json.dumps({"automations": find_template("automations", __file__)["automations"] * 1002}), 200), + ) + self.r_mock.get( + HttpRequest( + url="https://us10.api.mailchimp.com/3.0/automations", + query_params={ + "sort_field": "create_time", + "sort_dir": "ASC", + "exclude_fields": "automations._links", + "count": 1000, + "offset": 1002, + "since_create_time": "2022-12-31T23:59:59.001000Z", + "before_create_time": "2023-01-31T23:59:59.001000Z", + }, + ), + HttpResponse(json.dumps(find_template("automations", __file__)), 200), + ) + source = SourceMailchimp() + actual_messages = read(source, config=_CONFIG, catalog=_create_catalog()) + + assert len(actual_messages.records) == 1003 + + def test_when_read_incrementally_then_emit_state_message(self): + """Ensure incremental sync emits correct stream state message""" + + self.r_mock.get( + HttpRequest( + url="https://us10.api.mailchimp.com/3.0/automations", + query_params={ + "sort_field": "create_time", + "sort_dir": "ASC", + "exclude_fields": "automations._links", + "count": 1000, + "since_create_time": "2022-12-31T23:59:59.001000Z", + "before_create_time": "2023-01-31T23:59:59.001000Z", + }, + ), + HttpResponse(json.dumps(find_template("automations", __file__)), 200), + ) + + source = SourceMailchimp() + actual_messages = read( + source, + config=_CONFIG, + catalog=_create_catalog(sync_mode=SyncMode.incremental), + state=StateBuilder().with_stream_state("automations", {"create_time": "2220-11-23T05:42:11+00:00"}).build(), + ) + actual_messages.state_messages[0].state.stream.stream_state == {"create_time": "2220-11-23T05:42:11+00:00"} diff --git a/airbyte-integrations/connectors/source-twilio/unit_tests copy/resource/http/response/automations.json b/airbyte-integrations/connectors/source-twilio/unit_tests copy/resource/http/response/automations.json new file mode 100644 index 000000000000..3be7ec7ee972 --- /dev/null +++ b/airbyte-integrations/connectors/source-twilio/unit_tests copy/resource/http/response/automations.json @@ -0,0 +1,68 @@ +{ + "automations": [ + { + "id": "string", + "create_time": "2019-08-24T14:15:22Z", + "start_time": "2019-08-24T14:15:22Z", + "status": "save", + "emails_sent": 0, + "recipients": { + "list_id": "string", + "list_is_active": true, + "list_name": "string", + "segment_opts": { + "saved_segment_id": 0, + "match": "any", + "conditions": [null] + }, + "store_id": "1a2df69xxx" + }, + "settings": { + "title": "string", + "from_name": "string", + "reply_to": "string", + "use_conversation": true, + "to_name": "string", + "authenticate": true, + "auto_footer": true, + "inline_css": true + }, + "tracking": { + "opens": true, + "html_clicks": true, + "text_clicks": true, + "goal_tracking": true, + "ecomm360": true, + "google_analytics": "string", + "clicktale": "string", + "salesforce": { + "campaign": true, + "notes": true + }, + "capsule": { + "notes": true + } + }, + "trigger_settings": { + "workflow_type": "abandonedBrowse", + "workflow_title": "string", + "runtime": { + "days": ["sunday"], + "hours": { + "type": "send_asap" + } + }, + "workflow_emails_count": 0 + }, + "report_summary": { + "opens": 0, + "unique_opens": 0, + "open_rate": 0, + "clicks": 0, + "subscriber_clicks": 0, + "click_rate": 0 + } + } + ], + "total_items": 2 +} diff --git a/airbyte-integrations/connectors/source-twilio/unit_tests copy/test_component_custom_email_activity_extractor.py b/airbyte-integrations/connectors/source-twilio/unit_tests copy/test_component_custom_email_activity_extractor.py new file mode 100644 index 000000000000..7f9ad40d0c7f --- /dev/null +++ b/airbyte-integrations/connectors/source-twilio/unit_tests copy/test_component_custom_email_activity_extractor.py @@ -0,0 +1,66 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import json + +import requests +from airbyte_cdk.sources.declarative.decoders import JsonDecoder +from source_mailchimp.components import MailChimpRecordExtractorEmailActivity + + +def test_email_activity_extractor(): + decoder = JsonDecoder(parameters={}) + field_path = ["emails"] + config = {"response_override": "stop_if_you_see_me"} + extractor = MailChimpRecordExtractorEmailActivity(field_path=field_path, decoder=decoder, config=config, parameters={}) + + body = { + "emails": [ + { + "campaign_id": "string", + "list_id": "string", + "list_is_active": True, + "email_id": "string", + "email_address": "AirbyteMailchimpUser@gmail.com", + "activity": [ + {"action": "close", "type": "string", "timestamp": "2019-08-24T14:15:22Z", "url": "string", "ip": "string"}, + {"action": "open", "type": "string", "timestamp": "2019-08-24T14:15:22Z", "url": "string", "ip": "string"}, + ], + } + ], + "campaign_id": "string", + "total_items": 0, + } + response = requests.Response() + response._content = json.dumps(body).encode("utf-8") + + expected_records = [ + { + "action": "close", + "campaign_id": "string", + "email_address": "AirbyteMailchimpUser@gmail.com", + "email_id": "string", + "ip": "string", + "list_id": "string", + "list_is_active": True, + "timestamp": "2019-08-24T14:15:22Z", + "type": "string", + "url": "string", + }, + { + "action": "open", + "campaign_id": "string", + "email_address": "AirbyteMailchimpUser@gmail.com", + "email_id": "string", + "ip": "string", + "list_id": "string", + "list_is_active": True, + "timestamp": "2019-08-24T14:15:22Z", + "type": "string", + "url": "string", + }, + ] + + assert list(extractor.extract_records(response=response)) == expected_records diff --git a/airbyte-integrations/connectors/source-twilio/unit_tests copy/test_config_datacenter_migration.py b/airbyte-integrations/connectors/source-twilio/unit_tests copy/test_config_datacenter_migration.py new file mode 100644 index 000000000000..20fe8312352e --- /dev/null +++ b/airbyte-integrations/connectors/source-twilio/unit_tests copy/test_config_datacenter_migration.py @@ -0,0 +1,37 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# +import json +import os +from typing import Any, Mapping + +import pytest +from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource +from source_mailchimp import SourceMailchimp +from source_mailchimp.config_migrations import MigrateDataCenter + +# BASE ARGS +SOURCE: YamlDeclarativeSource = SourceMailchimp() + + +# HELPERS +def load_config(config_path: str) -> Mapping[str, Any]: + with open(config_path, "r") as config: + return json.load(config) + + +@pytest.mark.parametrize( + "config_path", + [ + (f"{os.path.dirname(__file__)}/test_configs/test_config_api_key.json"), + (f"{os.path.dirname(__file__)}/test_configs/test_config_oauth.json"), + ], + ids=["test_requester_datacenter_with_api_key", "test_requester_datacenter_with_oauth_flow"], +) +def test_mailchimp_config_migration(config_path: str, requests_mock): + requests_mock.get("https://login.mailchimp.com/oauth2/metadata", json={"dc": "us10"}) + + migration_instance = MigrateDataCenter + migration_instance.migrate(["check", "--config", config_path], SOURCE) + test_migrated_config = load_config(config_path) + assert test_migrated_config.get("data_center") == "us10" diff --git a/airbyte-integrations/connectors/source-twilio/unit_tests copy/test_configs/test_config_api_key.json b/airbyte-integrations/connectors/source-twilio/unit_tests copy/test_configs/test_config_api_key.json new file mode 100644 index 000000000000..20866d9bfcf7 --- /dev/null +++ b/airbyte-integrations/connectors/source-twilio/unit_tests copy/test_configs/test_config_api_key.json @@ -0,0 +1,4 @@ +{ + "credentials": { "auth_type": "apikey", "apikey": "random_api_key-us10" }, + "data_center": "us10" +} diff --git a/airbyte-integrations/connectors/source-twilio/unit_tests copy/test_configs/test_config_oauth.json b/airbyte-integrations/connectors/source-twilio/unit_tests copy/test_configs/test_config_oauth.json new file mode 100644 index 000000000000..ef7ef97ee241 --- /dev/null +++ b/airbyte-integrations/connectors/source-twilio/unit_tests copy/test_configs/test_config_oauth.json @@ -0,0 +1,8 @@ +{ + "credentials": { + "auth_type": "oauth2.0", + "client_id": "client_id", + "client_secret": "client_secret", + "access_token": "access_token" + } +} From 01a9da2b21d2236c5b6bae8f868355ce28e45575 Mon Sep 17 00:00:00 2001 From: "cristina.mariscal" Date: Fri, 28 Jun 2024 16:07:19 -0600 Subject: [PATCH 13/37] WIP LOCAL DO NOT PUSH --- .../connectors/source-twilio/migrations.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/airbyte-integrations/connectors/source-twilio/migrations.py b/airbyte-integrations/connectors/source-twilio/migrations.py index df9c77db0179..34d108acf23a 100644 --- a/airbyte-integrations/connectors/source-twilio/migrations.py +++ b/airbyte-integrations/connectors/source-twilio/migrations.py @@ -19,6 +19,21 @@ class MigrateDataCenter: STREAMS = ["calls"] + def migrate_state(cls, stream_name, state: Mapping[str, Any]) -> Mapping[str, Any]: + """ + This method migrates the state. + Args: + - stream_name (str): The stream name. + - state (Mapping[str, Any]): The state to migrate. + Returns: + - Mapping[str, Any]: The migrated state. + """ + if stream_name in cls.STREAMS: + if "partition" in state: + return state + return {"partition": {"data_center": "us1"}, "cursor": state} + return state + @classmethod def migrate(cls, args: List[str], source: Source) -> None: """ From 898f32392ea82d6ace05a8e738d894d90309faae Mon Sep 17 00:00:00 2001 From: "cristina.mariscal" Date: Thu, 4 Jul 2024 23:07:21 -0600 Subject: [PATCH 14/37] wip --- .../integration_tests/expected_records.jsonl | 2 + .../integration_tests/sample_state.json | 175 +++++++++++++----- .../source-twilio/source_twilio/manifest.yaml | 45 +---- 3 files changed, 140 insertions(+), 82 deletions(-) diff --git a/airbyte-integrations/connectors/source-twilio/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-twilio/integration_tests/expected_records.jsonl index 65bb012b98d0..90e3dffea268 100644 --- a/airbyte-integrations/connectors/source-twilio/integration_tests/expected_records.jsonl +++ b/airbyte-integrations/connectors/source-twilio/integration_tests/expected_records.jsonl @@ -53,3 +53,5 @@ {"stream": "verify_services", "data": {"default_template_sid": null, "tts_name": null, "psd2_enabled": false, "do_not_share_warning_enabled": false, "mailer_sid": null, "friendly_name": "MyServiceName", "url": "https://verify.twilio.com/v2/Services/VAf7a6bc96c5a594a56b3ccacfaf4c2e6e", "account_sid": "ACdade166c12e160e9ed0a6088226718fb", "date_updated": "2022-12-01T04:35:52Z", "totp": {"time_step": 30.0, "skew": 1.0, "code_length": 6.0, "issuer": "MyServiceName"}, "code_length": 6.0, "custom_code_enabled": false, "sid": "VAf7a6bc96c5a594a56b3ccacfaf4c2e6e", "push": {"apn_credential_sid": null, "include_date": false, "fcm_credential_sid": null}, "date_created": "2022-12-01T04:35:52Z", "dtmf_input_required": true, "skip_sms_to_landlines": false, "lookup_enabled": false, "links": {"verification_checks": "https://verify.twilio.com/v2/Services/VAf7a6bc96c5a594a56b3ccacfaf4c2e6e/VerificationCheck", "rate_limits": "https://verify.twilio.com/v2/Services/VAf7a6bc96c5a594a56b3ccacfaf4c2e6e/RateLimits", "entities": "https://verify.twilio.com/v2/Services/VAf7a6bc96c5a594a56b3ccacfaf4c2e6e/Entities", "access_tokens": "https://verify.twilio.com/v2/Services/VAf7a6bc96c5a594a56b3ccacfaf4c2e6e/AccessTokens", "verifications": "https://verify.twilio.com/v2/Services/VAf7a6bc96c5a594a56b3ccacfaf4c2e6e/Verifications", "webhooks": "https://verify.twilio.com/v2/Services/VAf7a6bc96c5a594a56b3ccacfaf4c2e6e/Webhooks", "messaging_configurations": "https://verify.twilio.com/v2/Services/VAf7a6bc96c5a594a56b3ccacfaf4c2e6e/MessagingConfigurations"}}, "emitted_at": 1691420020246} {"stream": "verify_services", "data": {"default_template_sid": null, "tts_name": null, "psd2_enabled": false, "do_not_share_warning_enabled": false, "mailer_sid": null, "friendly_name": "MyServiceName", "url": "https://verify.twilio.com/v2/Services/VA5d72879e05e83a236433c203b4d8ecf4", "account_sid": "ACdade166c12e160e9ed0a6088226718fb", "date_updated": "2022-12-02T18:08:14Z", "totp": {"time_step": 30.0, "skew": 1.0, "code_length": 6.0, "issuer": "MyServiceName"}, "code_length": 6.0, "custom_code_enabled": false, "sid": "VA5d72879e05e83a236433c203b4d8ecf4", "push": {"apn_credential_sid": null, "include_date": false, "fcm_credential_sid": null}, "date_created": "2022-12-02T18:08:14Z", "dtmf_input_required": true, "skip_sms_to_landlines": false, "lookup_enabled": false, "links": {"verification_checks": "https://verify.twilio.com/v2/Services/VA5d72879e05e83a236433c203b4d8ecf4/VerificationCheck", "rate_limits": "https://verify.twilio.com/v2/Services/VA5d72879e05e83a236433c203b4d8ecf4/RateLimits", "entities": "https://verify.twilio.com/v2/Services/VA5d72879e05e83a236433c203b4d8ecf4/Entities", "access_tokens": "https://verify.twilio.com/v2/Services/VA5d72879e05e83a236433c203b4d8ecf4/AccessTokens", "verifications": "https://verify.twilio.com/v2/Services/VA5d72879e05e83a236433c203b4d8ecf4/Verifications", "webhooks": "https://verify.twilio.com/v2/Services/VA5d72879e05e83a236433c203b4d8ecf4/Webhooks", "messaging_configurations": "https://verify.twilio.com/v2/Services/VA5d72879e05e83a236433c203b4d8ecf4/MessagingConfigurations"}}, "emitted_at": 1691420020247} {"stream": "verify_services", "data": {"default_template_sid": null, "tts_name": null, "psd2_enabled": false, "do_not_share_warning_enabled": false, "mailer_sid": null, "friendly_name": "MyServiceName", "url": "https://verify.twilio.com/v2/Services/VA3c8282c56e66bd5b015a9196c09be5be", "account_sid": "ACdade166c12e160e9ed0a6088226718fb", "date_updated": "2022-12-01T04:38:44Z", "totp": {"time_step": 30.0, "skew": 1.0, "code_length": 6.0, "issuer": "MyServiceName"}, "code_length": 6.0, "custom_code_enabled": false, "sid": "VA3c8282c56e66bd5b015a9196c09be5be", "push": {"apn_credential_sid": null, "include_date": false, "fcm_credential_sid": null}, "date_created": "2022-12-01T04:38:44Z", "dtmf_input_required": true, "skip_sms_to_landlines": false, "lookup_enabled": false, "links": {"verification_checks": "https://verify.twilio.com/v2/Services/VA3c8282c56e66bd5b015a9196c09be5be/VerificationCheck", "rate_limits": "https://verify.twilio.com/v2/Services/VA3c8282c56e66bd5b015a9196c09be5be/RateLimits", "entities": "https://verify.twilio.com/v2/Services/VA3c8282c56e66bd5b015a9196c09be5be/Entities", "access_tokens": "https://verify.twilio.com/v2/Services/VA3c8282c56e66bd5b015a9196c09be5be/AccessTokens", "verifications": "https://verify.twilio.com/v2/Services/VA3c8282c56e66bd5b015a9196c09be5be/Verifications", "webhooks": "https://verify.twilio.com/v2/Services/VA3c8282c56e66bd5b015a9196c09be5be/Webhooks", "messaging_configurations": "https://verify.twilio.com/v2/Services/VA3c8282c56e66bd5b015a9196c09be5be/MessagingConfigurations"}}, "emitted_at": 1691420020248} +{"stream": "available_phone_numbers_toll-Free", "data": { "friendly_name": "+37280010004", "phone_number": "+37280010004", "latitude": "0.000000", "longitude": "0.000000", "iso_country": "EE", "address_requirements": "any", "beta": false, "capabilities": { "voice": true, "SMS": false, "MMS": false } } , "emitted_at": 1691419684732} +{"stream": "user_conversations", "data": { "notification_level": "default", "user_sid": "US4373c40fffca48dcab7498989c484a0d", "friendly_name": "Friendly Conversation", "conversation_sid": "CH0ed7b4c3498e455a96fa09fcccee720e", "created_by": "system", "account_sid": "ACdade166c12e160e9ed0a6088226718fb", "date_created": "2023-03-21T13:39:44Z", "url": "https://conversations.twilio.com/v1/Users/US4373c40fffca48dcab7498989c484a0d/Conversations/CH0ed7b4c3498e455a96fa09fcccee720e", "date_updated": "2023-03-21T13:39:44Z", "attributes": "{}", "participant_sid": "MB0a984a4238f14b828cf277becf880bd4", "conversation_state": "active", "chat_service_sid": "IS5fcc074f7ead44c99a0a24a374a7e19f", "links": { "conversation": "https://conversations.twilio.com/v1/Conversations/CH0ed7b4c3498e455a96fa09fcccee720e", "participant": "https://conversations.twilio.com/v1/Conversations/CH0ed7b4c3498e455a96fa09fcccee720e/Participants/MB0a984a4238f14b828cf277becf880bd4" } } , "emitted_at": 1691419684732} diff --git a/airbyte-integrations/connectors/source-twilio/integration_tests/sample_state.json b/airbyte-integrations/connectors/source-twilio/integration_tests/sample_state.json index 075389e3382c..775f2a45d3f3 100644 --- a/airbyte-integrations/connectors/source-twilio/integration_tests/sample_state.json +++ b/airbyte-integrations/connectors/source-twilio/integration_tests/sample_state.json @@ -1,79 +1,168 @@ [ { - "type": "STREAM", - "stream": { - "stream_state": { - "end_time": "2022-06-11T00:00:00Z" + "type": "STATE", + "state": { + "type": "STREAM", + "stream": { + "stream_descriptor": { + "name": "calls", + "namespace": null + }, + "stream_state": { + "states": [ + { + "partition": { + "account_sid": "ACdade166c12e160e9ed0a6088226718fb", + "parent_slice": {} + }, + "cursor": { + "end_time": "Tue, 25 Jun 2024 20:19:28 +0000" + } + } + ] + } }, - "stream_descriptor": { - "name": "calls" + "sourceStats": { + "recordCount": 0.0 } } }, { - "type": "STREAM", - "stream": { - "stream_state": { - "date_updated": "2020-01-01T00:00:00Z" + "type": "STATE", + "state": { + "type": "STREAM", + "stream": { + "stream_descriptor": { + "name": "conferences", + "namespace": null + }, + "stream_state": { + "states": [ + { + "partition": { + "account_sid": "ACdade166c12e160e9ed0a6088226718fb", + "parent_slice": {} + }, + "cursor": { + "date_created": "Tue, 25 Jun 2024 20:19:17 +0000" + } + } + ] + } }, - "stream_descriptor": { - "name": "conferences" + "sourceStats": { + "recordCount": 11.0 } } }, { - "type": "STREAM", - "stream": { - "stream_state": { - "date_created": "2020-01-01T00:00:00Z" + "type": "STATE", + "state": { + "type": "STREAM", + "stream": { + "stream_descriptor": { + "name": "recordings", + "namespace": null + }, + "stream_state": { + "states": [ + { + "partition": { + "account_sid": "ACdade166c12e160e9ed0a6088226718fb", + "parent_slice": {} + }, + "cursor": { + "date_created": "Mon, 10 Jun 2024 19:10:08 +0000" + } + } + ] + } }, - "stream_descriptor": { - "name": "recordings" + "sourceStats": { + "recordCount": 0.0 } } }, { - "type": "STREAM", - "stream": { - "stream_state": { - "date_sent": "2020-01-01T00:00:00Z" + "type": "STATE", + "state": { + "type": "STREAM", + "stream": { + "stream_descriptor": { + "name": "messages", + "namespace": null + }, + "stream_state": { + "states": [ + { + "partition": { + "account_sid": "ACdade166c12e160e9ed0a6088226718fb", + "parent_slice": {} + }, + "cursor": { + "date_sent": "Wed, 08 May 2024 23:28:12 +0000" + } + } + ] + } }, - "stream_descriptor": { - "name": "messages" + "sourceStats": { + "recordCount": 0.0 } } }, { - "type": "STREAM", - "stream": { - "stream_state": { - "date_created": "2020-01-01T00:00:00Z" + "type": "STATE", + "state": { + "type": "STREAM", + "stream": { + "stream_descriptor": { + "name": "message_media", + "namespace": null + }, + "stream_state": { + "states": [] + } }, - "stream_descriptor": { - "name": "message_media" + "sourceStats": { + "recordCount": 0.0 } } }, { - "type": "STREAM", - "stream": { - "stream_state": { - "start_date": "2020-01-01" + "type": "STATE", + "state": { + "type": "STREAM", + "stream": { + "stream_descriptor": { + "name": "usage_records", + "namespace": null + }, + "stream_state": { + "states": [] + } }, - "stream_descriptor": { - "name": "usage_records" + "sourceStats": { + "recordCount": 443.0 } } }, { - "type": "STREAM", - "stream": { - "stream_state": { - "date_updated": "2020-01-01T00:00:00Z" + "type": "STATE", + "state": { + "type": "STREAM", + "stream": { + "stream_descriptor": { + "name": "alerts", + "namespace": null + }, + "stream_state": { + "date_generated": "2024-06-21T21:27:56Z" + } }, - "stream_descriptor": { - "name": "alerts" + "sourceStats": { + "recordCount": 1.0 } } } -] +] \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-twilio/source_twilio/manifest.yaml b/airbyte-integrations/connectors/source-twilio/source_twilio/manifest.yaml index 2a6c3759c2fd..e8ac9b251d75 100644 --- a/airbyte-integrations/connectors/source-twilio/source_twilio/manifest.yaml +++ b/airbyte-integrations/connectors/source-twilio/source_twilio/manifest.yaml @@ -460,7 +460,7 @@ definitions: type: DatetimeBasedCursor cursor_field: date_generated cursor_datetime_formats: - - '%a, %d %b %Y %H:%M:%S %z' + - '%Y-%m-%dT%H:%M:%SZ' datetime_format: '%Y-%m-%d' start_datetime: type: MinMaxDatetime @@ -1469,7 +1469,7 @@ definitions: type: DeclarativeStream name: user_conversations primary_key: - - account_sid + - conversation_sid retriever: type: SimpleRetriever requester: @@ -1482,7 +1482,8 @@ definitions: type: RecordSelector extractor: type: DpathExtractor - field_path: [] + field_path: + - conversations partition_router: type: SubstreamPartitionRouter parent_stream_configs: @@ -3999,7 +4000,8 @@ schemas: - string date_created: description: The date and time when the transcription was created. - format: date-time + format: date-time + datetime_format: '%a, %d %b %Y %H:%M:%S %z' type: - "null" - string @@ -5069,13 +5071,6 @@ schemas: $schema: http://json-schema.org/schema# additionalProperties: true properties: - account_sid: - description: - The unique identifier of the Twilio account the conversation belongs - to. - type: - - "null" - - string attributes: description: Additional metadata or attributes associated with the conversation. type: @@ -5116,13 +5111,6 @@ schemas: type: - "null" - string - last_read_message_index: - description: - The index of the last message in the conversation that has been read - by the user. - type: - - "null" - - integer links: description: Contains links related to the user conversation data. properties: @@ -5149,27 +5137,6 @@ schemas: type: - "null" - string - timers: - description: Contains timers or time-related information for the user conversation. - properties: - chat_service_sid: - description: Additional timer related to the chat service. - type: - - "null" - - string - type: - - "null" - - object - unique_name: - description: A unique name assigned to the conversation. - type: - - "null" - - string - unread_messages_count: - description: The count of unread messages in the conversation. - type: - - "null" - - integer url: description: The URL to access the conversation details. type: From 59912a6c9e47f11406f723027f732e4815876b13 Mon Sep 17 00:00:00 2001 From: "cristina.mariscal" Date: Fri, 5 Jul 2024 04:27:43 -0600 Subject: [PATCH 15/37] wip --- .../integration_tests/sample_state.json | 2 +- .../source-twilio/source_twilio/manifest.yaml | 103 ++++++------------ 2 files changed, 37 insertions(+), 68 deletions(-) diff --git a/airbyte-integrations/connectors/source-twilio/integration_tests/sample_state.json b/airbyte-integrations/connectors/source-twilio/integration_tests/sample_state.json index 775f2a45d3f3..a08921ccb3e1 100644 --- a/airbyte-integrations/connectors/source-twilio/integration_tests/sample_state.json +++ b/airbyte-integrations/connectors/source-twilio/integration_tests/sample_state.json @@ -165,4 +165,4 @@ } } } -] \ No newline at end of file +] diff --git a/airbyte-integrations/connectors/source-twilio/source_twilio/manifest.yaml b/airbyte-integrations/connectors/source-twilio/source_twilio/manifest.yaml index e8ac9b251d75..675e24a1119f 100644 --- a/airbyte-integrations/connectors/source-twilio/source_twilio/manifest.yaml +++ b/airbyte-integrations/connectors/source-twilio/source_twilio/manifest.yaml @@ -460,12 +460,12 @@ definitions: type: DatetimeBasedCursor cursor_field: date_generated cursor_datetime_formats: - - '%Y-%m-%dT%H:%M:%SZ' - datetime_format: '%Y-%m-%d' + - "%Y-%m-%dT%H:%M:%SZ" + datetime_format: "%Y-%m-%d" start_datetime: type: MinMaxDatetime datetime: '{{ config["start_date"] }}' - datetime_format: '%Y-%m-%dT%H:%M:%SZ' + datetime_format: "%Y-%m-%dT%H:%M:%SZ" start_time_option: type: RequestOption field_name: StartDate= @@ -476,8 +476,8 @@ definitions: inject_into: request_parameter end_datetime: type: MinMaxDatetime - datetime: '{{ now_utc().strftime(''%Y-%m-%dT%H:%M:%SZ'') }}' - datetime_format: '%Y-%m-%dT%H:%M:%SZ' + datetime: "{{ now_utc().strftime('%Y-%m-%dT%H:%M:%SZ') }}" + datetime_format: "%Y-%m-%dT%H:%M:%SZ" schema_loader: type: InlineSchemaLoader schema: @@ -625,12 +625,12 @@ definitions: type: DatetimeBasedCursor cursor_field: end_time cursor_datetime_formats: - - '%a, %d %b %Y %H:%M:%S %z' - datetime_format: '%Y-%m-%d' + - "%a, %d %b %Y %H:%M:%S %z" + datetime_format: "%Y-%m-%d" start_datetime: type: MinMaxDatetime datetime: '{{ config["start_date"] }}' - datetime_format: '%Y-%m-%dT%H:%M:%SZ' + datetime_format: "%Y-%m-%dT%H:%M:%SZ" start_time_option: type: RequestOption field_name: EndTime> @@ -641,8 +641,8 @@ definitions: inject_into: request_parameter end_datetime: type: MinMaxDatetime - datetime: '{{ now_utc().strftime(''%Y-%m-%dT%H:%M:%SZ'') }}' - datetime_format: '%Y-%m-%dT%H:%M:%SZ' + datetime: "{{ now_utc().strftime('%Y-%m-%dT%H:%M:%SZ') }}" + datetime_format: "%Y-%m-%dT%H:%M:%SZ" schema_loader: type: InlineSchemaLoader schema: @@ -692,12 +692,12 @@ definitions: type: DatetimeBasedCursor cursor_field: date_created cursor_datetime_formats: - - '%a, %d %b %Y %H:%M:%S %z' - datetime_format: '%Y-%m-%d' + - "%a, %d %b %Y %H:%M:%S %z" + datetime_format: "%Y-%m-%d" start_datetime: type: MinMaxDatetime datetime: '{{ config["start_date"] }}' - datetime_format: '%Y-%m-%dT%H:%M:%SZ' + datetime_format: "%Y-%m-%dT%H:%M:%SZ" start_time_option: type: RequestOption field_name: DateCreated> @@ -708,8 +708,8 @@ definitions: inject_into: request_parameter end_datetime: type: MinMaxDatetime - datetime: '{{ now_utc().strftime(''%Y-%m-%dT%H:%M:%SZ'') }}' - datetime_format: '%Y-%m-%dT%H:%M:%SZ' + datetime: "{{ now_utc().strftime('%Y-%m-%dT%H:%M:%SZ') }}" + datetime_format: "%Y-%m-%dT%H:%M:%SZ" schema_loader: type: InlineSchemaLoader schema: @@ -902,12 +902,12 @@ definitions: type: DatetimeBasedCursor cursor_field: date_created cursor_datetime_formats: - - '%a, %d %b %Y %H:%M:%S %z' - datetime_format: '%Y-%m-%d' + - "%a, %d %b %Y %H:%M:%S %z" + datetime_format: "%Y-%m-%d" start_datetime: type: MinMaxDatetime datetime: '{{ config["start_date"] }}' - datetime_format: '%Y-%m-%dT%H:%M:%SZ' + datetime_format: "%Y-%m-%dT%H:%M:%SZ" start_time_option: type: RequestOption field_name: DateCreated> @@ -918,8 +918,8 @@ definitions: inject_into: request_parameter end_datetime: type: MinMaxDatetime - datetime: '{{ now_utc().strftime(''%Y-%m-%dT%H:%M:%SZ'') }}' - datetime_format: '%Y-%m-%dT%H:%M:%SZ' + datetime: "{{ now_utc().strftime('%Y-%m-%dT%H:%M:%SZ') }}" + datetime_format: "%Y-%m-%dT%H:%M:%SZ" schema_loader: type: InlineSchemaLoader schema: @@ -1151,12 +1151,12 @@ definitions: type: DatetimeBasedCursor cursor_field: date_sent cursor_datetime_formats: - - '%a, %d %b %Y %H:%M:%S %z' - datetime_format: '%Y-%m-%d' + - "%a, %d %b %Y %H:%M:%S %z" + datetime_format: "%Y-%m-%d" start_datetime: type: MinMaxDatetime datetime: '{{ config["start_date"] }}' - datetime_format: '%Y-%m-%dT%H:%M:%SZ' + datetime_format: "%Y-%m-%dT%H:%M:%SZ" start_time_option: type: RequestOption field_name: DateSent> @@ -1167,8 +1167,8 @@ definitions: inject_into: request_parameter end_datetime: type: MinMaxDatetime - datetime: '{{ now_utc().strftime(''%Y-%m-%dT%H:%M:%SZ'') }}' - datetime_format: '%Y-%m-%dT%H:%M:%SZ' + datetime: "{{ now_utc().strftime('%Y-%m-%dT%H:%M:%SZ') }}" + datetime_format: "%Y-%m-%dT%H:%M:%SZ" schema_loader: type: InlineSchemaLoader schema: @@ -1226,12 +1226,12 @@ definitions: type: DatetimeBasedCursor cursor_field: date_created cursor_datetime_formats: - - '%a, %d %b %Y %H:%M:%S %z' - datetime_format: '%Y-%m-%d' + - "%a, %d %b %Y %H:%M:%S %z" + datetime_format: "%Y-%m-%d" start_datetime: type: MinMaxDatetime datetime: '{{ config["start_date"] }}' - datetime_format: '%Y-%m-%dT%H:%M:%SZ' + datetime_format: "%Y-%m-%dT%H:%M:%SZ" start_time_option: type: RequestOption field_name: DateCreated> @@ -1242,8 +1242,8 @@ definitions: inject_into: request_parameter end_datetime: type: MinMaxDatetime - datetime: '{{ now_utc().strftime(''%Y-%m-%dT%H:%M:%SZ'') }}' - datetime_format: '%Y-%m-%dT%H:%M:%SZ' + datetime: "{{ now_utc().strftime('%Y-%m-%dT%H:%M:%SZ') }}" + datetime_format: "%Y-%m-%dT%H:%M:%SZ" schema_loader: type: InlineSchemaLoader schema: @@ -1294,12 +1294,12 @@ definitions: type: DatetimeBasedCursor cursor_field: start_date cursor_datetime_formats: - - '%a, %d %b %Y %H:%M:%S %z' - datetime_format: '%Y-%m-%d' + - "%a, %d %b %Y %H:%M:%S %z" + datetime_format: "%Y-%m-%d" start_datetime: type: MinMaxDatetime datetime: '{{ config["start_date"] }}' - datetime_format: '%Y-%m-%dT%H:%M:%SZ' + datetime_format: "%Y-%m-%dT%H:%M:%SZ" start_time_option: type: RequestOption field_name: StartDate> @@ -1310,8 +1310,8 @@ definitions: inject_into: request_parameter end_datetime: type: MinMaxDatetime - datetime: '{{ now_utc().strftime(''%Y-%m-%dT%H:%M:%SZ'') }}' - datetime_format: '%Y-%m-%dT%H:%M:%SZ' + datetime: "{{ now_utc().strftime('%Y-%m-%dT%H:%M:%SZ') }}" + datetime_format: "%Y-%m-%dT%H:%M:%SZ" schema_loader: type: InlineSchemaLoader schema: @@ -1764,7 +1764,6 @@ schemas: - sid Addresses: type: object - $schema: http://json-schema.org/schema# additionalProperties: true properties: account_sid: @@ -1853,7 +1852,6 @@ schemas: - sid Available Phone Number Countries: type: object - $schema: http://json-schema.org/schema# additionalProperties: true properties: beta: @@ -1900,7 +1898,6 @@ schemas: - string Available Phone Numbers Local: type: object - $schema: http://json-schema.org/schema# additionalProperties: true properties: address_requirements: @@ -1994,7 +1991,6 @@ schemas: - string Available Phone Numbers Mobile: type: object - $schema: http://json-schema.org/schema# additionalProperties: true properties: address_requirements: @@ -2088,7 +2084,6 @@ schemas: - string Available Phone Numbers Toll-Free: type: object - $schema: http://json-schema.org/schema# additionalProperties: true properties: address_requirements: @@ -2184,7 +2179,6 @@ schemas: - string Incoming Phone Numbers: type: object - $schema: http://json-schema.org/schema# additionalProperties: true properties: account_sid: @@ -2405,7 +2399,6 @@ schemas: - string Keys: type: object - $schema: http://json-schema.org/schema# additionalProperties: true properties: date_created: @@ -2432,7 +2425,6 @@ schemas: - string Flows: type: object - $schema: http://json-schema.org/schema# additionalProperties: true properties: account_sid: @@ -2494,7 +2486,6 @@ schemas: - integer Step: type: object - $schema: http://json-schema.org/schema# additionalProperties: true properties: account_sid: @@ -2572,7 +2563,6 @@ schemas: - string Alerts: type: object - $schema: http://json-schema.org/schema# additionalProperties: true properties: account_sid: @@ -2655,7 +2645,6 @@ schemas: - string Applications: type: object - $schema: http://json-schema.org/schema# additionalProperties: true properties: account_sid: @@ -2777,7 +2766,6 @@ schemas: - sid Dependent Phone Numbers: type: object - $schema: http://json-schema.org/schema# additionalProperties: true properties: account_sid: @@ -2939,7 +2927,6 @@ schemas: - sid Calls: type: object - $schema: http://json-schema.org/schema# additionalProperties: true properties: account_sid: @@ -3137,7 +3124,6 @@ schemas: - sid Conferences: type: object - $schema: http://json-schema.org/schema# additionalProperties: true properties: account_sid: @@ -3218,7 +3204,6 @@ schemas: - sid Conference Participants: type: object - $schema: http://json-schema.org/schema# additionalProperties: true properties: account_sid: @@ -3418,7 +3403,6 @@ schemas: - string Recordings: type: object - $schema: http://json-schema.org/schema# additionalProperties: true properties: account_sid: @@ -3557,7 +3541,6 @@ schemas: - sid Services: type: object - $schema: http://json-schema.org/schema# additionalProperties: true properties: account_sid: @@ -3732,7 +3715,6 @@ schemas: - string Verify Services: type: object - $schema: http://json-schema.org/schema# additionalProperties: true properties: account_sid: @@ -3925,7 +3907,6 @@ schemas: - sid Roles: type: object - $schema: http://json-schema.org/schema# additionalProperties: true properties: account_sid: @@ -3983,7 +3964,6 @@ schemas: - sid Transcriptions: type: object - $schema: http://json-schema.org/schema# additionalProperties: true properties: account_sid: @@ -4000,8 +3980,7 @@ schemas: - string date_created: description: The date and time when the transcription was created. - format: date-time - datetime_format: '%a, %d %b %Y %H:%M:%S %z' + format: date-time type: - "null" - string @@ -4062,7 +4041,6 @@ schemas: - sid Trunks: type: object - $schema: http://json-schema.org/schema# additionalProperties: true properties: account_sid: @@ -4198,7 +4176,6 @@ schemas: - sid Queues: type: object - $schema: http://json-schema.org/schema# additionalProperties: true properties: account_sid: @@ -4266,7 +4243,6 @@ schemas: - sid Messages: type: object - $schema: http://json-schema.org/schema# additionalProperties: true properties: account_sid: @@ -4440,7 +4416,6 @@ schemas: - string Usage Records: type: object - $schema: http://json-schema.org/schema# additionalProperties: true properties: account_sid: @@ -4580,7 +4555,6 @@ schemas: - start_date Usage Triggers: type: object - $schema: http://json-schema.org/schema# additionalProperties: true properties: account_sid: @@ -4670,7 +4644,6 @@ schemas: - sid Conversations: type: object - $schema: http://json-schema.org/schema# additionalProperties: true properties: account_sid: @@ -4748,7 +4721,6 @@ schemas: - sid Conversation Participants: type: object - $schema: http://json-schema.org/schema# additionalProperties: true properties: account_sid: @@ -4833,7 +4805,6 @@ schemas: - sid Conversation Messages: type: object - $schema: http://json-schema.org/schema# additionalProperties: true properties: account_sid: @@ -4988,7 +4959,6 @@ schemas: - sid Users: type: object - $schema: http://json-schema.org/schema# additionalProperties: true properties: account_sid: @@ -5068,7 +5038,6 @@ schemas: - sid User Conversations: type: object - $schema: http://json-schema.org/schema# additionalProperties: true properties: attributes: From 57fd0592aa8d652264eb81a004c8ecffac4d99af Mon Sep 17 00:00:00 2001 From: "cristina.mariscal" Date: Fri, 5 Jul 2024 05:55:24 -0600 Subject: [PATCH 16/37] adding tranformations --- .../source-twilio/source_twilio/manifest.yaml | 58 +++++++++++++++++++ 1 file changed, 58 insertions(+) diff --git a/airbyte-integrations/connectors/source-twilio/source_twilio/manifest.yaml b/airbyte-integrations/connectors/source-twilio/source_twilio/manifest.yaml index 675e24a1119f..3b09a724ab16 100644 --- a/airbyte-integrations/connectors/source-twilio/source_twilio/manifest.yaml +++ b/airbyte-integrations/connectors/source-twilio/source_twilio/manifest.yaml @@ -1102,6 +1102,19 @@ definitions: partition_field: account_sid stream: $ref: "#/definitions/streams/Accounts" + transformations: + - type: AddFields + fields: + - type: AddedFieldDefinition + path: + - date_created + value: '{% set datetime_rfc2822 = format_datetime_string(record["date_created"], "%a, %d %b %Y %H:%M:%S %z") %}{% set formatted_datetime = format_datetime(datetime_rfc2822, "%Y-%m-%dT%H:%M:%SZ") %}{{ formatted_datetime }}' + - type: AddFields + fields: + - type: AddedFieldDefinition + path: + - date_updated + value: '{% set datetime_rfc2822 = format_datetime_string(record["date_updated"], "%a, %d %b %Y %H:%M:%S %z") %}{% set formatted_datetime = format_datetime(datetime_rfc2822, "%Y-%m-%dT%H:%M:%SZ") %}{{ formatted_datetime }}' schema_loader: type: InlineSchemaLoader schema: @@ -1169,6 +1182,25 @@ definitions: type: MinMaxDatetime datetime: "{{ now_utc().strftime('%Y-%m-%dT%H:%M:%SZ') }}" datetime_format: "%Y-%m-%dT%H:%M:%SZ" + transformations: + - type: AddFields + fields: + - type: AddedFieldDefinition + path: + - date_created + value: '{% set datetime_rfc2822 = format_datetime_string(record["date_created"], "%a, %d %b %Y %H:%M:%S %z") %}{% set formatted_datetime = format_datetime(datetime_rfc2822, "%Y-%m-%dT%H:%M:%SZ") %}{{ formatted_datetime }}' + - type: AddFields + fields: + - type: AddedFieldDefinition + path: + - date_updated + value: '{% set datetime_rfc2822 = format_datetime_string(record["date_updated"], "%a, %d %b %Y %H:%M:%S %z") %}{% set formatted_datetime = format_datetime(datetime_rfc2822, "%Y-%m-%dT%H:%M:%SZ") %}{{ formatted_datetime }}' + - type: AddFields + fields: + - type: AddedFieldDefinition + path: + - date_sent + value: '{% set datetime_rfc2822 = format_datetime_string(record["date_sent"], "%a, %d %b %Y %H:%M:%S %z") %}{% set formatted_datetime = format_datetime(datetime_rfc2822, "%Y-%m-%dT%H:%M:%SZ") %}{{ formatted_datetime }}' schema_loader: type: InlineSchemaLoader schema: @@ -1357,6 +1389,19 @@ definitions: partition_field: account_sid stream: $ref: "#/definitions/streams/Accounts" + transformations: + - type: AddFields + fields: + - type: AddedFieldDefinition + path: + - date_created + value: '{% set datetime_rfc2822 = format_datetime_string(record["date_created"], "%a, %d %b %Y %H:%M:%S %z") %}{% set formatted_datetime = format_datetime(datetime_rfc2822, "%Y-%m-%dT%H:%M:%SZ") %}{{ formatted_datetime }}' + - type: AddFields + fields: + - type: AddedFieldDefinition + path: + - date_updated + value: '{% set datetime_rfc2822 = format_datetime_string(record["date_updated"], "%a, %d %b %Y %H:%M:%S %z") %}{% set formatted_datetime = format_datetime(datetime_rfc2822, "%Y-%m-%dT%H:%M:%SZ") %}{{ formatted_datetime }}' schema_loader: type: InlineSchemaLoader schema: @@ -1492,6 +1537,19 @@ definitions: partition_field: user_sid stream: $ref: "#/definitions/streams/Users" + transformations: + - type: AddFields + fields: + - type: AddedFieldDefinition + path: + - date_created + value: '{% set datetime_rfc2822 = format_datetime_string(record["date_created"], "%a, %d %b %Y %H:%M:%S %z") %}{% set formatted_datetime = format_datetime(datetime_rfc2822, "%Y-%m-%dT%H:%M:%SZ") %}{{ formatted_datetime }}' + - type: AddFields + fields: + - type: AddedFieldDefinition + path: + - date_updated + value: '{% set datetime_rfc2822 = format_datetime_string(record["date_updated"], "%a, %d %b %Y %H:%M:%S %z") %}{% set formatted_datetime = format_datetime(datetime_rfc2822, "%Y-%m-%dT%H:%M:%SZ") %}{{ formatted_datetime }}' schema_loader: type: InlineSchemaLoader schema: From 9491be5eb67a558815e13aff5e816abdb2e66847 Mon Sep 17 00:00:00 2001 From: "cristina.mariscal" Date: Fri, 5 Jul 2024 15:35:14 -0600 Subject: [PATCH 17/37] adding transformations --- .../source-twilio/source_twilio/manifest.yaml | 378 +++++++++++++++++- 1 file changed, 367 insertions(+), 11 deletions(-) diff --git a/airbyte-integrations/connectors/source-twilio/source_twilio/manifest.yaml b/airbyte-integrations/connectors/source-twilio/source_twilio/manifest.yaml index 3b09a724ab16..24b44c6547dd 100644 --- a/airbyte-integrations/connectors/source-twilio/source_twilio/manifest.yaml +++ b/airbyte-integrations/connectors/source-twilio/source_twilio/manifest.yaml @@ -33,6 +33,19 @@ definitions: type: DpathExtractor field_path: - accounts + transformations: + - type: AddFields + fields: + - type: AddedFieldDefinition + path: + - date_created + value: '{% set formatted_datetime = format_datetime(record["date_created"], "%Y-%m-%dT%H:%M:%SZ") %}{{ formatted_datetime }}' + - type: AddFields + fields: + - type: AddedFieldDefinition + path: + - date_updated + value: '{% set formatted_datetime = format_datetime(record["date_updated"], "%Y-%m-%dT%H:%M:%SZ") %}{{ formatted_datetime }}' schema_loader: type: InlineSchemaLoader schema: @@ -78,6 +91,19 @@ definitions: partition_field: account_sid stream: $ref: "#/definitions/streams/Accounts" + transformations: + - type: AddFields + fields: + - type: AddedFieldDefinition + path: + - date_created + value: '{% set formatted_datetime = format_datetime(record["date_created"], "%Y-%m-%dT%H:%M:%SZ") %}{{ formatted_datetime }}' + - type: AddFields + fields: + - type: AddedFieldDefinition + path: + - date_updated + value: '{% set formatted_datetime = format_datetime(record["date_updated"], "%Y-%m-%dT%H:%M:%SZ") %}{{ formatted_datetime }}' schema_loader: type: InlineSchemaLoader schema: @@ -232,6 +258,14 @@ definitions: path: >- https://api.twilio.com/2010-04-01/Accounts/{{stream_partition.account_sid}}/AvailablePhoneNumbers/{{stream_partition.country_code}}/TollFree.json http_method: GET + error_handler: + type: CompositeErrorHandler + error_handlers: + - type: DefaultErrorHandler + response_filters: + - http_codes: [404] + action: IGNORE + error_message: "The requested resource was not found." record_selector: type: RecordSelector extractor: @@ -309,6 +343,19 @@ definitions: partition_field: account_sid stream: $ref: "#/definitions/streams/Accounts" + transformations: + - type: AddFields + fields: + - type: AddedFieldDefinition + path: + - date_created + value: '{% set formatted_datetime = format_datetime(record["date_created"], "%Y-%m-%dT%H:%M:%SZ") %}{{ formatted_datetime }}' + - type: AddFields + fields: + - type: AddedFieldDefinition + path: + - date_updated + value: '{% set formatted_datetime = format_datetime(record["date_updated"], "%Y-%m-%dT%H:%M:%SZ") %}{{ formatted_datetime }}' schema_loader: type: InlineSchemaLoader schema: @@ -351,6 +398,19 @@ definitions: partition_field: account_sid stream: $ref: "#/definitions/streams/Accounts" + transformations: + - type: AddFields + fields: + - type: AddedFieldDefinition + path: + - date_created + value: '{% set formatted_datetime = format_datetime(record["date_created"], "%Y-%m-%dT%H:%M:%SZ") %}{{ formatted_datetime }}' + - type: AddFields + fields: + - type: AddedFieldDefinition + path: + - date_updated + value: '{% set formatted_datetime = format_datetime(record["date_updated"], "%Y-%m-%dT%H:%M:%SZ") %}{{ formatted_datetime }}' schema_loader: type: InlineSchemaLoader schema: @@ -386,6 +446,19 @@ definitions: type: PageIncrement page_size: 1000 start_from_page: 0 + transformations: + - type: AddFields + fields: + - type: AddedFieldDefinition + path: + - date_created + value: '{% set formatted_datetime = format_datetime(record["date_created"], "%Y-%m-%dT%H:%M:%SZ") %}{{ formatted_datetime }}' + - type: AddFields + fields: + - type: AddedFieldDefinition + path: + - date_updated + value: '{% set formatted_datetime = format_datetime(record["date_updated"], "%Y-%m-%dT%H:%M:%SZ") %}{{ formatted_datetime }}' schema_loader: type: InlineSchemaLoader schema: @@ -437,6 +510,19 @@ definitions: partition_field: execution_sid stream: $ref: "#/definitions/streams/Executions" + transformations: + - type: AddFields + fields: + - type: AddedFieldDefinition + path: + - date_created + value: '{% set formatted_datetime = format_datetime(record["date_created"], "%Y-%m-%dT%H:%M:%SZ") %}{{ formatted_datetime }}' + - type: AddFields + fields: + - type: AddedFieldDefinition + path: + - date_updated + value: '{% set formatted_datetime = format_datetime(record["date_updated"], "%Y-%m-%dT%H:%M:%SZ") %}{{ formatted_datetime }}' schema_loader: type: InlineSchemaLoader schema: @@ -478,6 +564,25 @@ definitions: type: MinMaxDatetime datetime: "{{ now_utc().strftime('%Y-%m-%dT%H:%M:%SZ') }}" datetime_format: "%Y-%m-%dT%H:%M:%SZ" + transformations: + - type: AddFields + fields: + - type: AddedFieldDefinition + path: + - date_created + value: '{% set formatted_datetime = format_datetime(record["date_created"], "%Y-%m-%dT%H:%M:%SZ") %}{{ formatted_datetime }}' + - type: AddFields + fields: + - type: AddedFieldDefinition + path: + - date_updated + value: '{% set formatted_datetime = format_datetime(record["date_updated"], "%Y-%m-%dT%H:%M:%SZ") %}{{ formatted_datetime }}' + - type: AddFields + fields: + - type: AddedFieldDefinition + path: + - date_generated + value: '{% set formatted_datetime = format_datetime(record["date_generated"], "%Y-%m-%dT%H:%M:%SZ") %}{{ formatted_datetime }}' schema_loader: type: InlineSchemaLoader schema: @@ -523,6 +628,19 @@ definitions: partition_field: account_sid stream: $ref: "#/definitions/streams/Accounts" + transformations: + - type: AddFields + fields: + - type: AddedFieldDefinition + path: + - date_created + value: '{% set formatted_datetime = format_datetime(record["date_created"], "%Y-%m-%dT%H:%M:%SZ") %}{{ formatted_datetime }}' + - type: AddFields + fields: + - type: AddedFieldDefinition + path: + - date_updated + value: '{% set formatted_datetime = format_datetime(record["date_updated"], "%Y-%m-%dT%H:%M:%SZ") %}{{ formatted_datetime }}' schema_loader: type: InlineSchemaLoader schema: @@ -576,6 +694,19 @@ definitions: partition_field: address_sid stream: $ref: "#/definitions/streams/Addresses" + transformations: + - type: AddFields + fields: + - type: AddedFieldDefinition + path: + - date_created + value: '{% set formatted_datetime = format_datetime(record["date_created"], "%Y-%m-%dT%H:%M:%SZ") %}{{ formatted_datetime }}' + - type: AddFields + fields: + - type: AddedFieldDefinition + path: + - date_updated + value: '{% set formatted_datetime = format_datetime(record["date_updated"], "%Y-%m-%dT%H:%M:%SZ") %}{{ formatted_datetime }}' schema_loader: type: InlineSchemaLoader schema: @@ -643,6 +774,19 @@ definitions: type: MinMaxDatetime datetime: "{{ now_utc().strftime('%Y-%m-%dT%H:%M:%SZ') }}" datetime_format: "%Y-%m-%dT%H:%M:%SZ" + transformations: + - type: AddFields + fields: + - type: AddedFieldDefinition + path: + - date_created + value: '{% set formatted_datetime = format_datetime(record["date_created"], "%Y-%m-%dT%H:%M:%SZ") %}{{ formatted_datetime }}' + - type: AddFields + fields: + - type: AddedFieldDefinition + path: + - date_updated + value: '{% set formatted_datetime = format_datetime(record["date_updated"], "%Y-%m-%dT%H:%M:%SZ") %}{{ formatted_datetime }}' schema_loader: type: InlineSchemaLoader schema: @@ -710,6 +854,19 @@ definitions: type: MinMaxDatetime datetime: "{{ now_utc().strftime('%Y-%m-%dT%H:%M:%SZ') }}" datetime_format: "%Y-%m-%dT%H:%M:%SZ" + transformations: + - type: AddFields + fields: + - type: AddedFieldDefinition + path: + - date_created + value: '{% set formatted_datetime = format_datetime(record["date_created"], "%Y-%m-%dT%H:%M:%SZ") %}{{ formatted_datetime }}' + - type: AddFields + fields: + - type: AddedFieldDefinition + path: + - date_updated + value: '{% set formatted_datetime = format_datetime(record["date_updated"], "%Y-%m-%dT%H:%M:%SZ") %}{{ formatted_datetime }}' schema_loader: type: InlineSchemaLoader schema: @@ -764,6 +921,19 @@ definitions: partition_field: conference_sid stream: $ref: "#/definitions/streams/Conferences" + transformations: + - type: AddFields + fields: + - type: AddedFieldDefinition + path: + - date_created + value: '{% set formatted_datetime = format_datetime(record["date_created"], "%Y-%m-%dT%H:%M:%SZ") %}{{ formatted_datetime }}' + - type: AddFields + fields: + - type: AddedFieldDefinition + path: + - date_updated + value: '{% set formatted_datetime = format_datetime(record["date_updated"], "%Y-%m-%dT%H:%M:%SZ") %}{{ formatted_datetime }}' schema_loader: type: InlineSchemaLoader schema: @@ -809,6 +979,19 @@ definitions: partition_field: flow_sid stream: $ref: "#/definitions/streams/Flows" + transformations: + - type: AddFields + fields: + - type: AddedFieldDefinition + path: + - date_created + value: '{% set formatted_datetime = format_datetime(record["date_created"], "%Y-%m-%dT%H:%M:%SZ") %}{{ formatted_datetime }}' + - type: AddFields + fields: + - type: AddedFieldDefinition + path: + - date_updated + value: '{% set formatted_datetime = format_datetime(record["date_updated"], "%Y-%m-%dT%H:%M:%SZ") %}{{ formatted_datetime }}' schema_loader: type: InlineSchemaLoader schema: @@ -853,6 +1036,19 @@ definitions: partition_field: account_sid stream: $ref: "#/definitions/streams/Accounts" + transformations: + - type: AddFields + fields: + - type: AddedFieldDefinition + path: + - date_created + value: '{% set formatted_datetime = format_datetime(record["date_created"], "%Y-%m-%dT%H:%M:%SZ") %}{{ formatted_datetime }}' + - type: AddFields + fields: + - type: AddedFieldDefinition + path: + - date_updated + value: '{% set formatted_datetime = format_datetime(record["date_updated"], "%Y-%m-%dT%H:%M:%SZ") %}{{ formatted_datetime }}' schema_loader: type: InlineSchemaLoader schema: @@ -920,6 +1116,19 @@ definitions: type: MinMaxDatetime datetime: "{{ now_utc().strftime('%Y-%m-%dT%H:%M:%SZ') }}" datetime_format: "%Y-%m-%dT%H:%M:%SZ" + transformations: + - type: AddFields + fields: + - type: AddedFieldDefinition + path: + - date_created + value: '{% set formatted_datetime = format_datetime(record["date_created"], "%Y-%m-%dT%H:%M:%SZ") %}{{ formatted_datetime }}' + - type: AddFields + fields: + - type: AddedFieldDefinition + path: + - date_updated + value: '{% set formatted_datetime = format_datetime(record["date_updated"], "%Y-%m-%dT%H:%M:%SZ") %}{{ formatted_datetime }}' schema_loader: type: InlineSchemaLoader schema: @@ -939,6 +1148,19 @@ definitions: type: DpathExtractor field_path: - services + transformations: + - type: AddFields + fields: + - type: AddedFieldDefinition + path: + - date_created + value: '{% set formatted_datetime = format_datetime(record["date_created"], "%Y-%m-%dT%H:%M:%SZ") %}{{ formatted_datetime }}' + - type: AddFields + fields: + - type: AddedFieldDefinition + path: + - date_updated + value: '{% set formatted_datetime = format_datetime(record["date_updated"], "%Y-%m-%dT%H:%M:%SZ") %}{{ formatted_datetime }}' schema_loader: type: InlineSchemaLoader schema: @@ -960,6 +1182,19 @@ definitions: type: DpathExtractor field_path: - services + transformations: + - type: AddFields + fields: + - type: AddedFieldDefinition + path: + - date_created + value: '{% set formatted_datetime = format_datetime(record["date_created"], "%Y-%m-%dT%H:%M:%SZ") %}{{ formatted_datetime }}' + - type: AddFields + fields: + - type: AddedFieldDefinition + path: + - date_updated + value: '{% set formatted_datetime = format_datetime(record["date_updated"], "%Y-%m-%dT%H:%M:%SZ") %}{{ formatted_datetime }}' schema_loader: type: InlineSchemaLoader schema: @@ -991,6 +1226,19 @@ definitions: partition_field: service_sid stream: $ref: "#/definitions/streams/Services" + transformations: + - type: AddFields + fields: + - type: AddedFieldDefinition + path: + - date_created + value: '{% set formatted_datetime = format_datetime(record["date_created"], "%Y-%m-%dT%H:%M:%SZ") %}{{ formatted_datetime }}' + - type: AddFields + fields: + - type: AddedFieldDefinition + path: + - date_updated + value: '{% set formatted_datetime = format_datetime(record["date_updated"], "%Y-%m-%dT%H:%M:%SZ") %}{{ formatted_datetime }}' schema_loader: type: InlineSchemaLoader schema: @@ -1036,6 +1284,31 @@ definitions: partition_field: account_sid stream: $ref: "#/definitions/streams/Accounts" + transformations: + - type: AddFields + fields: + - type: AddedFieldDefinition + path: + - date_created + value: '{% set formatted_datetime = format_datetime(record["date_created"], "%Y-%m-%dT%H:%M:%SZ") %}{{ formatted_datetime }}' + - type: AddFields + fields: + - type: AddedFieldDefinition + path: + - date_updated + value: '{% set formatted_datetime = format_datetime(record["date_updated"], "%Y-%m-%dT%H:%M:%SZ") %}{{ formatted_datetime }}' + - type: AddFields + fields: + - type: AddedFieldDefinition + path: + - duration + value: '{% set duration = record["duration"] | int %}{{ duration }}' + - type: AddFields + fields: + - type: AddedFieldDefinition + path: + - price + value: '{% set price = record["price"] | number %}{{ price }}' schema_loader: type: InlineSchemaLoader schema: @@ -1057,6 +1330,19 @@ definitions: type: DpathExtractor field_path: - trunks + transformations: + - type: AddFields + fields: + - type: AddedFieldDefinition + path: + - date_created + value: '{% set formatted_datetime = format_datetime(record["date_created"], "%Y-%m-%dT%H:%M:%SZ") %}{{ formatted_datetime }}' + - type: AddFields + fields: + - type: AddedFieldDefinition + path: + - date_updated + value: '{% set formatted_datetime = format_datetime(record["date_updated"], "%Y-%m-%dT%H:%M:%SZ") %}{{ formatted_datetime }}' schema_loader: type: InlineSchemaLoader schema: @@ -1108,13 +1394,13 @@ definitions: - type: AddedFieldDefinition path: - date_created - value: '{% set datetime_rfc2822 = format_datetime_string(record["date_created"], "%a, %d %b %Y %H:%M:%S %z") %}{% set formatted_datetime = format_datetime(datetime_rfc2822, "%Y-%m-%dT%H:%M:%SZ") %}{{ formatted_datetime }}' + value: '{% set formatted_datetime = format_datetime(record["date_created"], "%Y-%m-%dT%H:%M:%SZ") %}{{ formatted_datetime }}' - type: AddFields fields: - type: AddedFieldDefinition path: - date_updated - value: '{% set datetime_rfc2822 = format_datetime_string(record["date_updated"], "%a, %d %b %Y %H:%M:%S %z") %}{% set formatted_datetime = format_datetime(datetime_rfc2822, "%Y-%m-%dT%H:%M:%SZ") %}{{ formatted_datetime }}' + value: '{% set formatted_datetime = format_datetime(record["date_updated"], "%Y-%m-%dT%H:%M:%SZ") %}{{ formatted_datetime }}' schema_loader: type: InlineSchemaLoader schema: @@ -1188,19 +1474,19 @@ definitions: - type: AddedFieldDefinition path: - date_created - value: '{% set datetime_rfc2822 = format_datetime_string(record["date_created"], "%a, %d %b %Y %H:%M:%S %z") %}{% set formatted_datetime = format_datetime(datetime_rfc2822, "%Y-%m-%dT%H:%M:%SZ") %}{{ formatted_datetime }}' + value: '{% set formatted_datetime = format_datetime(record["date_created"], "%Y-%m-%dT%H:%M:%SZ") %}{{ formatted_datetime }}' - type: AddFields fields: - type: AddedFieldDefinition path: - date_updated - value: '{% set datetime_rfc2822 = format_datetime_string(record["date_updated"], "%a, %d %b %Y %H:%M:%S %z") %}{% set formatted_datetime = format_datetime(datetime_rfc2822, "%Y-%m-%dT%H:%M:%SZ") %}{{ formatted_datetime }}' + value: '{% set formatted_datetime = format_datetime(record["date_updated"], "%Y-%m-%dT%H:%M:%SZ") %}{{ formatted_datetime }}' - type: AddFields fields: - type: AddedFieldDefinition path: - date_sent - value: '{% set datetime_rfc2822 = format_datetime_string(record["date_sent"], "%a, %d %b %Y %H:%M:%S %z") %}{% set formatted_datetime = format_datetime(datetime_rfc2822, "%Y-%m-%dT%H:%M:%SZ") %}{{ formatted_datetime }}' + value: '{% set formatted_datetime = format_datetime(record["date_sent"], "%Y-%m-%dT%H:%M:%SZ") %}{{ formatted_datetime }}' schema_loader: type: InlineSchemaLoader schema: @@ -1276,6 +1562,19 @@ definitions: type: MinMaxDatetime datetime: "{{ now_utc().strftime('%Y-%m-%dT%H:%M:%SZ') }}" datetime_format: "%Y-%m-%dT%H:%M:%SZ" + transformations: + - type: AddFields + fields: + - type: AddedFieldDefinition + path: + - date_created + value: '{% set formatted_datetime = format_datetime(record["date_created"], "%Y-%m-%dT%H:%M:%SZ") %}{{ formatted_datetime }}' + - type: AddFields + fields: + - type: AddedFieldDefinition + path: + - date_updated + value: '{% set formatted_datetime = format_datetime(record["date_updated"], "%Y-%m-%dT%H:%M:%SZ") %}{{ formatted_datetime }}' schema_loader: type: InlineSchemaLoader schema: @@ -1395,13 +1694,19 @@ definitions: - type: AddedFieldDefinition path: - date_created - value: '{% set datetime_rfc2822 = format_datetime_string(record["date_created"], "%a, %d %b %Y %H:%M:%S %z") %}{% set formatted_datetime = format_datetime(datetime_rfc2822, "%Y-%m-%dT%H:%M:%SZ") %}{{ formatted_datetime }}' + value: '{% set formatted_datetime = format_datetime(record["date_created"], "%Y-%m-%dT%H:%M:%SZ") %}{{ formatted_datetime }}' - type: AddFields fields: - type: AddedFieldDefinition path: - date_updated - value: '{% set datetime_rfc2822 = format_datetime_string(record["date_updated"], "%a, %d %b %Y %H:%M:%S %z") %}{% set formatted_datetime = format_datetime(datetime_rfc2822, "%Y-%m-%dT%H:%M:%SZ") %}{{ formatted_datetime }}' + value: '{% set formatted_datetime = format_datetime(record["date_updated"], "%Y-%m-%dT%H:%M:%SZ") %}{{ formatted_datetime }}' + - type: AddFields + fields: + - type: AddedFieldDefinition + path: + - date_fired + value: '{% set formatted_datetime = format_datetime(record["date_fired"], "%Y-%m-%dT%H:%M:%SZ") %}{{ formatted_datetime }}' schema_loader: type: InlineSchemaLoader schema: @@ -1423,6 +1728,19 @@ definitions: type: DpathExtractor field_path: - conversations + transformations: + - type: AddFields + fields: + - type: AddedFieldDefinition + path: + - date_created + value: '{% set formatted_datetime = format_datetime(record["date_created"], "%Y-%m-%dT%H:%M:%SZ") %}{{ formatted_datetime }}' + - type: AddFields + fields: + - type: AddedFieldDefinition + path: + - date_updated + value: '{% set formatted_datetime = format_datetime(record["date_updated"], "%Y-%m-%dT%H:%M:%SZ") %}{{ formatted_datetime }}' schema_loader: type: InlineSchemaLoader schema: @@ -1454,6 +1772,19 @@ definitions: partition_field: conversation_sid stream: $ref: "#/definitions/streams/Conversations" + transformations: + - type: AddFields + fields: + - type: AddedFieldDefinition + path: + - date_created + value: '{% set formatted_datetime = format_datetime(record["date_created"], "%Y-%m-%dT%H:%M:%SZ") %}{{ formatted_datetime }}' + - type: AddFields + fields: + - type: AddedFieldDefinition + path: + - date_updated + value: '{% set formatted_datetime = format_datetime(record["date_updated"], "%Y-%m-%dT%H:%M:%SZ") %}{{ formatted_datetime }}' schema_loader: type: InlineSchemaLoader schema: @@ -1485,6 +1816,19 @@ definitions: partition_field: conversation_sid stream: $ref: "#/definitions/streams/Conversations" + transformations: + - type: AddFields + fields: + - type: AddedFieldDefinition + path: + - date_created + value: '{% set formatted_datetime = format_datetime(record["date_created"], "%Y-%m-%dT%H:%M:%SZ") %}{{ formatted_datetime }}' + - type: AddFields + fields: + - type: AddedFieldDefinition + path: + - date_updated + value: '{% set formatted_datetime = format_datetime(record["date_updated"], "%Y-%m-%dT%H:%M:%SZ") %}{{ formatted_datetime }}' schema_loader: type: InlineSchemaLoader schema: @@ -1506,6 +1850,19 @@ definitions: type: DpathExtractor field_path: - users + transformations: + - type: AddFields + fields: + - type: AddedFieldDefinition + path: + - date_created + value: '{% set formatted_datetime = format_datetime(record["date_created"], "%Y-%m-%dT%H:%M:%SZ") %}{{ formatted_datetime }}' + - type: AddFields + fields: + - type: AddedFieldDefinition + path: + - date_updated + value: '{% set formatted_datetime = format_datetime(record["date_updated"], "%Y-%m-%dT%H:%M:%SZ") %}{{ formatted_datetime }}' schema_loader: type: InlineSchemaLoader schema: @@ -1543,13 +1900,13 @@ definitions: - type: AddedFieldDefinition path: - date_created - value: '{% set datetime_rfc2822 = format_datetime_string(record["date_created"], "%a, %d %b %Y %H:%M:%S %z") %}{% set formatted_datetime = format_datetime(datetime_rfc2822, "%Y-%m-%dT%H:%M:%SZ") %}{{ formatted_datetime }}' + value: '{% set formatted_datetime = format_datetime(record["date_created"], "%Y-%m-%dT%H:%M:%SZ") %}{{ formatted_datetime }}' - type: AddFields fields: - type: AddedFieldDefinition path: - date_updated - value: '{% set datetime_rfc2822 = format_datetime_string(record["date_updated"], "%a, %d %b %Y %H:%M:%S %z") %}{% set formatted_datetime = format_datetime(datetime_rfc2822, "%Y-%m-%dT%H:%M:%SZ") %}{{ formatted_datetime }}' + value: '{% set formatted_datetime = format_datetime(record["date_updated"], "%Y-%m-%dT%H:%M:%SZ") %}{{ formatted_datetime }}' schema_loader: type: InlineSchemaLoader schema: @@ -5174,5 +5531,4 @@ schemas: type: - "null" - string - required: - - conversations + From 424afb2d94b42032630d7b8ae9a4f312322d1328 Mon Sep 17 00:00:00 2001 From: "cristina.mariscal" Date: Mon, 8 Jul 2024 07:50:08 -0600 Subject: [PATCH 18/37] updating transformations --- .../connectors/source-twilio/poetry.lock | 317 +++++++++++------- .../connectors/source-twilio/pyproject.toml | 2 +- .../source-twilio/source_twilio/manifest.yaml | 309 +++++++---------- 3 files changed, 317 insertions(+), 311 deletions(-) diff --git a/airbyte-integrations/connectors/source-twilio/poetry.lock b/airbyte-integrations/connectors/source-twilio/poetry.lock index 1e513f6f603e..1e3f883d8373 100644 --- a/airbyte-integrations/connectors/source-twilio/poetry.lock +++ b/airbyte-integrations/connectors/source-twilio/poetry.lock @@ -2,17 +2,17 @@ [[package]] name = "airbyte-cdk" -version = "1.8.0" +version = "2.4.0" description = "A framework for writing Airbyte Connectors." optional = false python-versions = "<4.0,>=3.9" files = [ - {file = "airbyte_cdk-1.8.0-py3-none-any.whl", hash = "sha256:ca23d7877005fe87ffc4a3a3de29ee55eed625d874eb59b49664b156f9ae9ee2"}, - {file = "airbyte_cdk-1.8.0.tar.gz", hash = "sha256:ac82fbfd6b650b7ed015900748e30fdd2a4c574caa54d1bcc03cb584a17f1533"}, + {file = "airbyte_cdk-2.4.0-py3-none-any.whl", hash = "sha256:39470b2fe97f28959fcecb839d3080a8aba4a64a29dddf54a39f11f93f9f9ef7"}, + {file = "airbyte_cdk-2.4.0.tar.gz", hash = "sha256:f973d2e17a6dd0416c4395139e2761a10b38aafa61e097eaacffebbe6164ef45"}, ] [package.dependencies] -airbyte-protocol-models = ">=0.9.0,<1.0" +airbyte-protocol-models-pdv2 = ">=0.12.2,<0.13.0" backoff = "*" cachetools = "*" cryptography = ">=42.0.5,<43.0.0" @@ -25,7 +25,7 @@ jsonref = ">=0.2,<0.3" jsonschema = ">=3.2.0,<3.3.0" langchain_core = "0.1.42" pendulum = "<3.0.0" -pydantic = ">=1.10.8,<2.0.0" +pydantic = ">=2.7,<3.0" pyjwt = ">=2.8.0,<3.0.0" pyrate-limiter = ">=3.1.0,<3.2.0" python-dateutil = "*" @@ -41,18 +41,29 @@ sphinx-docs = ["Sphinx (>=4.2,<4.3)", "sphinx-rtd-theme (>=1.0,<1.1)"] vector-db-based = ["cohere (==4.21)", "langchain (==0.1.16)", "openai[embeddings] (==0.27.9)", "tiktoken (==0.4.0)"] [[package]] -name = "airbyte-protocol-models" +name = "airbyte-protocol-models-pdv2" version = "0.12.2" description = "Declares the Airbyte Protocol." optional = false python-versions = ">=3.8" files = [ - {file = "airbyte_protocol_models-0.12.2-py3-none-any.whl", hash = "sha256:1780db5b26285865b858d26502933def8e11919c9436ccf7b8b9cb0170b07c2a"}, - {file = "airbyte_protocol_models-0.12.2.tar.gz", hash = "sha256:b7c4d9a7c32c0691601c2b9416af090a858e126666e2c8c880d7a1798eb519f0"}, + {file = "airbyte_protocol_models_pdv2-0.12.2-py3-none-any.whl", hash = "sha256:8b3f9d0388928547cdf2e9134c0d589e4bcaa6f63bf71a21299f6824bfb7ad0e"}, + {file = "airbyte_protocol_models_pdv2-0.12.2.tar.gz", hash = "sha256:130c9ab289f3f53749ce63ff1abbfb67a44b7e5bd2794865315a2976138b672b"}, ] [package.dependencies] -pydantic = ">=1.9.2,<2.0.0" +pydantic = ">=2.7.2,<3.0.0" + +[[package]] +name = "annotated-types" +version = "0.7.0" +description = "Reusable constraint types to use with typing.Annotated" +optional = false +python-versions = ">=3.8" +files = [ + {file = "annotated_types-0.7.0-py3-none-any.whl", hash = "sha256:1f02e8b43a8fbbc3f3e0d4f0f4bfc8131bcb4eebe8849b8e5c773f3a1c582a53"}, + {file = "annotated_types-0.7.0.tar.gz", hash = "sha256:aff07c09a53a08bc8cfccb9c85b05f1aa9a2a6f23728d790723543408344ce89"}, +] [[package]] name = "atomicwrites" @@ -143,13 +154,13 @@ ujson = ["ujson (>=5.7.0)"] [[package]] name = "certifi" -version = "2024.6.2" +version = "2024.7.4" description = "Python package for providing Mozilla's CA Bundle." optional = false python-versions = ">=3.6" files = [ - {file = "certifi-2024.6.2-py3-none-any.whl", hash = "sha256:ddc6c8ce995e6987e7faf5e3f1b02b302836a0e5d98ece18392cb1a36c72ad56"}, - {file = "certifi-2024.6.2.tar.gz", hash = "sha256:3cd43f1c6fa7dedc5899d69d3ad0398fd018ad1a17fba83ddaf78aa46c747516"}, + {file = "certifi-2024.7.4-py3-none-any.whl", hash = "sha256:c198e21b1289c2ab85ee4e67bb4b4ef3ead0892059901a8d5b622f24a1101e90"}, + {file = "certifi-2024.7.4.tar.gz", hash = "sha256:5a1e7645bc0ec61a09e26c36f6106dd4cf40c6db3a1fb6352b0244e7fb057c7b"}, ] [[package]] @@ -580,13 +591,13 @@ extended-testing = ["jinja2 (>=3,<4)"] [[package]] name = "langsmith" -version = "0.1.82" +version = "0.1.84" description = "Client library to connect to the LangSmith LLM Tracing and Evaluation Platform." optional = false python-versions = "<4.0,>=3.8.1" files = [ - {file = "langsmith-0.1.82-py3-none-any.whl", hash = "sha256:9b3653e7d316036b0c60bf0bc3e280662d660f485a4ebd8e5c9d84f9831ae79c"}, - {file = "langsmith-0.1.82.tar.gz", hash = "sha256:c02e2bbc488c10c13b52c69d271eb40bd38da078d37b6ae7ae04a18bd48140be"}, + {file = "langsmith-0.1.84-py3-none-any.whl", hash = "sha256:01f3c6390dba26c583bac8dd0e551ce3d0509c7f55cad714db0b5c8d36e4c7ff"}, + {file = "langsmith-0.1.84.tar.gz", hash = "sha256:5220c0439838b9a5bd320fd3686be505c5083dcee22d2452006c23891153bea1"}, ] [package.dependencies] @@ -665,57 +676,62 @@ files = [ [[package]] name = "orjson" -version = "3.10.5" +version = "3.10.6" description = "Fast, correct Python JSON library supporting dataclasses, datetimes, and numpy" optional = false python-versions = ">=3.8" files = [ - {file = "orjson-3.10.5-cp310-cp310-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:545d493c1f560d5ccfc134803ceb8955a14c3fcb47bbb4b2fee0232646d0b932"}, - {file = "orjson-3.10.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f4324929c2dd917598212bfd554757feca3e5e0fa60da08be11b4aa8b90013c1"}, - {file = "orjson-3.10.5-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8c13ca5e2ddded0ce6a927ea5a9f27cae77eee4c75547b4297252cb20c4d30e6"}, - {file = "orjson-3.10.5-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b6c8e30adfa52c025f042a87f450a6b9ea29649d828e0fec4858ed5e6caecf63"}, - {file = "orjson-3.10.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:338fd4f071b242f26e9ca802f443edc588fa4ab60bfa81f38beaedf42eda226c"}, - {file = "orjson-3.10.5-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:6970ed7a3126cfed873c5d21ece1cd5d6f83ca6c9afb71bbae21a0b034588d96"}, - {file = "orjson-3.10.5-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:235dadefb793ad12f7fa11e98a480db1f7c6469ff9e3da5e73c7809c700d746b"}, - {file = "orjson-3.10.5-cp310-none-win32.whl", hash = "sha256:be79e2393679eda6a590638abda16d167754393f5d0850dcbca2d0c3735cebe2"}, - {file = "orjson-3.10.5-cp310-none-win_amd64.whl", hash = "sha256:c4a65310ccb5c9910c47b078ba78e2787cb3878cdded1702ac3d0da71ddc5228"}, - {file = "orjson-3.10.5-cp311-cp311-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:cdf7365063e80899ae3a697def1277c17a7df7ccfc979990a403dfe77bb54d40"}, - {file = "orjson-3.10.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6b68742c469745d0e6ca5724506858f75e2f1e5b59a4315861f9e2b1df77775a"}, - {file = "orjson-3.10.5-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7d10cc1b594951522e35a3463da19e899abe6ca95f3c84c69e9e901e0bd93d38"}, - {file = "orjson-3.10.5-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:dcbe82b35d1ac43b0d84072408330fd3295c2896973112d495e7234f7e3da2e1"}, - {file = "orjson-3.10.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:10c0eb7e0c75e1e486c7563fe231b40fdd658a035ae125c6ba651ca3b07936f5"}, - {file = "orjson-3.10.5-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:53ed1c879b10de56f35daf06dbc4a0d9a5db98f6ee853c2dbd3ee9d13e6f302f"}, - {file = "orjson-3.10.5-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:099e81a5975237fda3100f918839af95f42f981447ba8f47adb7b6a3cdb078fa"}, - {file = "orjson-3.10.5-cp311-none-win32.whl", hash = "sha256:1146bf85ea37ac421594107195db8bc77104f74bc83e8ee21a2e58596bfb2f04"}, - {file = "orjson-3.10.5-cp311-none-win_amd64.whl", hash = "sha256:36a10f43c5f3a55c2f680efe07aa93ef4a342d2960dd2b1b7ea2dd764fe4a37c"}, - {file = "orjson-3.10.5-cp312-cp312-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:68f85ecae7af14a585a563ac741b0547a3f291de81cd1e20903e79f25170458f"}, - {file = "orjson-3.10.5-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:28afa96f496474ce60d3340fe8d9a263aa93ea01201cd2bad844c45cd21f5268"}, - {file = "orjson-3.10.5-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9cd684927af3e11b6e754df80b9ffafd9fb6adcaa9d3e8fdd5891be5a5cad51e"}, - {file = "orjson-3.10.5-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3d21b9983da032505f7050795e98b5d9eee0df903258951566ecc358f6696969"}, - {file = "orjson-3.10.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1ad1de7fef79736dde8c3554e75361ec351158a906d747bd901a52a5c9c8d24b"}, - {file = "orjson-3.10.5-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:2d97531cdfe9bdd76d492e69800afd97e5930cb0da6a825646667b2c6c6c0211"}, - {file = "orjson-3.10.5-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:d69858c32f09c3e1ce44b617b3ebba1aba030e777000ebdf72b0d8e365d0b2b3"}, - {file = "orjson-3.10.5-cp312-none-win32.whl", hash = "sha256:64c9cc089f127e5875901ac05e5c25aa13cfa5dbbbd9602bda51e5c611d6e3e2"}, - {file = "orjson-3.10.5-cp312-none-win_amd64.whl", hash = "sha256:b2efbd67feff8c1f7728937c0d7f6ca8c25ec81373dc8db4ef394c1d93d13dc5"}, - {file = "orjson-3.10.5-cp38-cp38-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:03b565c3b93f5d6e001db48b747d31ea3819b89abf041ee10ac6988886d18e01"}, - {file = "orjson-3.10.5-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:584c902ec19ab7928fd5add1783c909094cc53f31ac7acfada817b0847975f26"}, - {file = "orjson-3.10.5-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5a35455cc0b0b3a1eaf67224035f5388591ec72b9b6136d66b49a553ce9eb1e6"}, - {file = "orjson-3.10.5-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1670fe88b116c2745a3a30b0f099b699a02bb3482c2591514baf5433819e4f4d"}, - {file = "orjson-3.10.5-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:185c394ef45b18b9a7d8e8f333606e2e8194a50c6e3c664215aae8cf42c5385e"}, - {file = "orjson-3.10.5-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:ca0b3a94ac8d3886c9581b9f9de3ce858263865fdaa383fbc31c310b9eac07c9"}, - {file = "orjson-3.10.5-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:dfc91d4720d48e2a709e9c368d5125b4b5899dced34b5400c3837dadc7d6271b"}, - {file = "orjson-3.10.5-cp38-none-win32.whl", hash = "sha256:c05f16701ab2a4ca146d0bca950af254cb7c02f3c01fca8efbbad82d23b3d9d4"}, - {file = "orjson-3.10.5-cp38-none-win_amd64.whl", hash = "sha256:8a11d459338f96a9aa7f232ba95679fc0c7cedbd1b990d736467894210205c09"}, - {file = "orjson-3.10.5-cp39-cp39-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:85c89131d7b3218db1b24c4abecea92fd6c7f9fab87441cfc342d3acc725d807"}, - {file = "orjson-3.10.5-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fb66215277a230c456f9038d5e2d84778141643207f85336ef8d2a9da26bd7ca"}, - {file = "orjson-3.10.5-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:51bbcdea96cdefa4a9b4461e690c75ad4e33796530d182bdd5c38980202c134a"}, - {file = "orjson-3.10.5-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:dbead71dbe65f959b7bd8cf91e0e11d5338033eba34c114f69078d59827ee139"}, - {file = "orjson-3.10.5-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5df58d206e78c40da118a8c14fc189207fffdcb1f21b3b4c9c0c18e839b5a214"}, - {file = "orjson-3.10.5-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:c4057c3b511bb8aef605616bd3f1f002a697c7e4da6adf095ca5b84c0fd43595"}, - {file = "orjson-3.10.5-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:b39e006b00c57125ab974362e740c14a0c6a66ff695bff44615dcf4a70ce2b86"}, - {file = "orjson-3.10.5-cp39-none-win32.whl", hash = "sha256:eded5138cc565a9d618e111c6d5c2547bbdd951114eb822f7f6309e04db0fb47"}, - {file = "orjson-3.10.5-cp39-none-win_amd64.whl", hash = "sha256:cc28e90a7cae7fcba2493953cff61da5a52950e78dc2dacfe931a317ee3d8de7"}, - {file = "orjson-3.10.5.tar.gz", hash = "sha256:7a5baef8a4284405d96c90c7c62b755e9ef1ada84c2406c24a9ebec86b89f46d"}, + {file = "orjson-3.10.6-cp310-cp310-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:fb0ee33124db6eaa517d00890fc1a55c3bfe1cf78ba4a8899d71a06f2d6ff5c7"}, + {file = "orjson-3.10.6-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9c1c4b53b24a4c06547ce43e5fee6ec4e0d8fe2d597f4647fc033fd205707365"}, + {file = "orjson-3.10.6-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:eadc8fd310edb4bdbd333374f2c8fec6794bbbae99b592f448d8214a5e4050c0"}, + {file = "orjson-3.10.6-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:61272a5aec2b2661f4fa2b37c907ce9701e821b2c1285d5c3ab0207ebd358d38"}, + {file = "orjson-3.10.6-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:57985ee7e91d6214c837936dc1608f40f330a6b88bb13f5a57ce5257807da143"}, + {file = "orjson-3.10.6-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:633a3b31d9d7c9f02d49c4ab4d0a86065c4a6f6adc297d63d272e043472acab5"}, + {file = "orjson-3.10.6-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:1c680b269d33ec444afe2bdc647c9eb73166fa47a16d9a75ee56a374f4a45f43"}, + {file = "orjson-3.10.6-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:f759503a97a6ace19e55461395ab0d618b5a117e8d0fbb20e70cfd68a47327f2"}, + {file = "orjson-3.10.6-cp310-none-win32.whl", hash = "sha256:95a0cce17f969fb5391762e5719575217bd10ac5a189d1979442ee54456393f3"}, + {file = "orjson-3.10.6-cp310-none-win_amd64.whl", hash = "sha256:df25d9271270ba2133cc88ee83c318372bdc0f2cd6f32e7a450809a111efc45c"}, + {file = "orjson-3.10.6-cp311-cp311-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:b1ec490e10d2a77c345def52599311849fc063ae0e67cf4f84528073152bb2ba"}, + {file = "orjson-3.10.6-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:55d43d3feb8f19d07e9f01e5b9be4f28801cf7c60d0fa0d279951b18fae1932b"}, + {file = "orjson-3.10.6-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:ac3045267e98fe749408eee1593a142e02357c5c99be0802185ef2170086a863"}, + {file = "orjson-3.10.6-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c27bc6a28ae95923350ab382c57113abd38f3928af3c80be6f2ba7eb8d8db0b0"}, + {file = "orjson-3.10.6-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d27456491ca79532d11e507cadca37fb8c9324a3976294f68fb1eff2dc6ced5a"}, + {file = "orjson-3.10.6-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:05ac3d3916023745aa3b3b388e91b9166be1ca02b7c7e41045da6d12985685f0"}, + {file = "orjson-3.10.6-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1335d4ef59ab85cab66fe73fd7a4e881c298ee7f63ede918b7faa1b27cbe5212"}, + {file = "orjson-3.10.6-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:4bbc6d0af24c1575edc79994c20e1b29e6fb3c6a570371306db0993ecf144dc5"}, + {file = "orjson-3.10.6-cp311-none-win32.whl", hash = "sha256:450e39ab1f7694465060a0550b3f6d328d20297bf2e06aa947b97c21e5241fbd"}, + {file = "orjson-3.10.6-cp311-none-win_amd64.whl", hash = "sha256:227df19441372610b20e05bdb906e1742ec2ad7a66ac8350dcfd29a63014a83b"}, + {file = "orjson-3.10.6-cp312-cp312-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:ea2977b21f8d5d9b758bb3f344a75e55ca78e3ff85595d248eee813ae23ecdfb"}, + {file = "orjson-3.10.6-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b6f3d167d13a16ed263b52dbfedff52c962bfd3d270b46b7518365bcc2121eed"}, + {file = "orjson-3.10.6-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:f710f346e4c44a4e8bdf23daa974faede58f83334289df80bc9cd12fe82573c7"}, + {file = "orjson-3.10.6-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7275664f84e027dcb1ad5200b8b18373e9c669b2a9ec33d410c40f5ccf4b257e"}, + {file = "orjson-3.10.6-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:0943e4c701196b23c240b3d10ed8ecd674f03089198cf503105b474a4f77f21f"}, + {file = "orjson-3.10.6-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:446dee5a491b5bc7d8f825d80d9637e7af43f86a331207b9c9610e2f93fee22a"}, + {file = "orjson-3.10.6-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:64c81456d2a050d380786413786b057983892db105516639cb5d3ee3c7fd5148"}, + {file = "orjson-3.10.6-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:960db0e31c4e52fa0fc3ecbaea5b2d3b58f379e32a95ae6b0ebeaa25b93dfd34"}, + {file = "orjson-3.10.6-cp312-none-win32.whl", hash = "sha256:a6ea7afb5b30b2317e0bee03c8d34c8181bc5a36f2afd4d0952f378972c4efd5"}, + {file = "orjson-3.10.6-cp312-none-win_amd64.whl", hash = "sha256:874ce88264b7e655dde4aeaacdc8fd772a7962faadfb41abe63e2a4861abc3dc"}, + {file = "orjson-3.10.6-cp38-cp38-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:66680eae4c4e7fc193d91cfc1353ad6d01b4801ae9b5314f17e11ba55e934183"}, + {file = "orjson-3.10.6-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:caff75b425db5ef8e8f23af93c80f072f97b4fb3afd4af44482905c9f588da28"}, + {file = "orjson-3.10.6-cp38-cp38-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:3722fddb821b6036fd2a3c814f6bd9b57a89dc6337b9924ecd614ebce3271394"}, + {file = "orjson-3.10.6-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c2c116072a8533f2fec435fde4d134610f806bdac20188c7bd2081f3e9e0133f"}, + {file = "orjson-3.10.6-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:6eeb13218c8cf34c61912e9df2de2853f1d009de0e46ea09ccdf3d757896af0a"}, + {file = "orjson-3.10.6-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:965a916373382674e323c957d560b953d81d7a8603fbeee26f7b8248638bd48b"}, + {file = "orjson-3.10.6-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:03c95484d53ed8e479cade8628c9cea00fd9d67f5554764a1110e0d5aa2de96e"}, + {file = "orjson-3.10.6-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:e060748a04cccf1e0a6f2358dffea9c080b849a4a68c28b1b907f272b5127e9b"}, + {file = "orjson-3.10.6-cp38-none-win32.whl", hash = "sha256:738dbe3ef909c4b019d69afc19caf6b5ed0e2f1c786b5d6215fbb7539246e4c6"}, + {file = "orjson-3.10.6-cp38-none-win_amd64.whl", hash = "sha256:d40f839dddf6a7d77114fe6b8a70218556408c71d4d6e29413bb5f150a692ff7"}, + {file = "orjson-3.10.6-cp39-cp39-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:697a35a083c4f834807a6232b3e62c8b280f7a44ad0b759fd4dce748951e70db"}, + {file = "orjson-3.10.6-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fd502f96bf5ea9a61cbc0b2b5900d0dd68aa0da197179042bdd2be67e51a1e4b"}, + {file = "orjson-3.10.6-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:f215789fb1667cdc874c1b8af6a84dc939fd802bf293a8334fce185c79cd359b"}, + {file = "orjson-3.10.6-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a2debd8ddce948a8c0938c8c93ade191d2f4ba4649a54302a7da905a81f00b56"}, + {file = "orjson-3.10.6-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5410111d7b6681d4b0d65e0f58a13be588d01b473822483f77f513c7f93bd3b2"}, + {file = "orjson-3.10.6-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bb1f28a137337fdc18384079fa5726810681055b32b92253fa15ae5656e1dddb"}, + {file = "orjson-3.10.6-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:bf2fbbce5fe7cd1aa177ea3eab2b8e6a6bc6e8592e4279ed3db2d62e57c0e1b2"}, + {file = "orjson-3.10.6-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:79b9b9e33bd4c517445a62b90ca0cc279b0f1f3970655c3df9e608bc3f91741a"}, + {file = "orjson-3.10.6-cp39-none-win32.whl", hash = "sha256:30b0a09a2014e621b1adf66a4f705f0809358350a757508ee80209b2d8dae219"}, + {file = "orjson-3.10.6-cp39-none-win_amd64.whl", hash = "sha256:49e3bc615652617d463069f91b867a4458114c5b104e13b7ae6872e5f79d0844"}, + {file = "orjson-3.10.6.tar.gz", hash = "sha256:e54b63d0a7c6c54a5f5f726bc93a2078111ef060fec4ecbf34c5db800ca3b3a7"}, ] [[package]] @@ -818,62 +834,123 @@ files = [ [[package]] name = "pydantic" -version = "1.10.17" -description = "Data validation and settings management using python type hints" +version = "2.8.2" +description = "Data validation using Python type hints" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "pydantic-1.10.17-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:0fa51175313cc30097660b10eec8ca55ed08bfa07acbfe02f7a42f6c242e9a4b"}, - {file = "pydantic-1.10.17-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:c7e8988bb16988890c985bd2093df9dd731bfb9d5e0860db054c23034fab8f7a"}, - {file = "pydantic-1.10.17-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:371dcf1831f87c9e217e2b6a0c66842879a14873114ebb9d0861ab22e3b5bb1e"}, - {file = "pydantic-1.10.17-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4866a1579c0c3ca2c40575398a24d805d4db6cb353ee74df75ddeee3c657f9a7"}, - {file = "pydantic-1.10.17-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:543da3c6914795b37785703ffc74ba4d660418620cc273490d42c53949eeeca6"}, - {file = "pydantic-1.10.17-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:7623b59876f49e61c2e283551cc3647616d2fbdc0b4d36d3d638aae8547ea681"}, - {file = "pydantic-1.10.17-cp310-cp310-win_amd64.whl", hash = "sha256:409b2b36d7d7d19cd8310b97a4ce6b1755ef8bd45b9a2ec5ec2b124db0a0d8f3"}, - {file = "pydantic-1.10.17-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:fa43f362b46741df8f201bf3e7dff3569fa92069bcc7b4a740dea3602e27ab7a"}, - {file = "pydantic-1.10.17-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:2a72d2a5ff86a3075ed81ca031eac86923d44bc5d42e719d585a8eb547bf0c9b"}, - {file = "pydantic-1.10.17-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b4ad32aed3bf5eea5ca5decc3d1bbc3d0ec5d4fbcd72a03cdad849458decbc63"}, - {file = "pydantic-1.10.17-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:aeb4e741782e236ee7dc1fb11ad94dc56aabaf02d21df0e79e0c21fe07c95741"}, - {file = "pydantic-1.10.17-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:d2f89a719411cb234105735a520b7c077158a81e0fe1cb05a79c01fc5eb59d3c"}, - {file = "pydantic-1.10.17-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:db3b48d9283d80a314f7a682f7acae8422386de659fffaba454b77a083c3937d"}, - {file = "pydantic-1.10.17-cp311-cp311-win_amd64.whl", hash = "sha256:9c803a5113cfab7bbb912f75faa4fc1e4acff43e452c82560349fff64f852e1b"}, - {file = "pydantic-1.10.17-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:820ae12a390c9cbb26bb44913c87fa2ff431a029a785642c1ff11fed0a095fcb"}, - {file = "pydantic-1.10.17-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:c1e51d1af306641b7d1574d6d3307eaa10a4991542ca324f0feb134fee259815"}, - {file = "pydantic-1.10.17-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9e53fb834aae96e7b0dadd6e92c66e7dd9cdf08965340ed04c16813102a47fab"}, - {file = "pydantic-1.10.17-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0e2495309b1266e81d259a570dd199916ff34f7f51f1b549a0d37a6d9b17b4dc"}, - {file = "pydantic-1.10.17-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:098ad8de840c92ea586bf8efd9e2e90c6339d33ab5c1cfbb85be66e4ecf8213f"}, - {file = "pydantic-1.10.17-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:525bbef620dac93c430d5d6bdbc91bdb5521698d434adf4434a7ef6ffd5c4b7f"}, - {file = "pydantic-1.10.17-cp312-cp312-win_amd64.whl", hash = "sha256:6654028d1144df451e1da69a670083c27117d493f16cf83da81e1e50edce72ad"}, - {file = "pydantic-1.10.17-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:c87cedb4680d1614f1d59d13fea353faf3afd41ba5c906a266f3f2e8c245d655"}, - {file = "pydantic-1.10.17-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:11289fa895bcbc8f18704efa1d8020bb9a86314da435348f59745473eb042e6b"}, - {file = "pydantic-1.10.17-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:94833612d6fd18b57c359a127cbfd932d9150c1b72fea7c86ab58c2a77edd7c7"}, - {file = "pydantic-1.10.17-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:d4ecb515fa7cb0e46e163ecd9d52f9147ba57bc3633dca0e586cdb7a232db9e3"}, - {file = "pydantic-1.10.17-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:7017971ffa7fd7808146880aa41b266e06c1e6e12261768a28b8b41ba55c8076"}, - {file = "pydantic-1.10.17-cp37-cp37m-win_amd64.whl", hash = "sha256:e840e6b2026920fc3f250ea8ebfdedf6ea7a25b77bf04c6576178e681942ae0f"}, - {file = "pydantic-1.10.17-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:bfbb18b616abc4df70591b8c1ff1b3eabd234ddcddb86b7cac82657ab9017e33"}, - {file = "pydantic-1.10.17-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:ebb249096d873593e014535ab07145498957091aa6ae92759a32d40cb9998e2e"}, - {file = "pydantic-1.10.17-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d8c209af63ccd7b22fba94b9024e8b7fd07feffee0001efae50dd99316b27768"}, - {file = "pydantic-1.10.17-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d4b40c9e13a0b61583e5599e7950490c700297b4a375b55b2b592774332798b7"}, - {file = "pydantic-1.10.17-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:c31d281c7485223caf6474fc2b7cf21456289dbaa31401844069b77160cab9c7"}, - {file = "pydantic-1.10.17-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:ae5184e99a060a5c80010a2d53c99aee76a3b0ad683d493e5f0620b5d86eeb75"}, - {file = "pydantic-1.10.17-cp38-cp38-win_amd64.whl", hash = "sha256:ad1e33dc6b9787a6f0f3fd132859aa75626528b49cc1f9e429cdacb2608ad5f0"}, - {file = "pydantic-1.10.17-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:7e17c0ee7192e54a10943f245dc79e36d9fe282418ea05b886e1c666063a7b54"}, - {file = "pydantic-1.10.17-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:cafb9c938f61d1b182dfc7d44a7021326547b7b9cf695db5b68ec7b590214773"}, - {file = "pydantic-1.10.17-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:95ef534e3c22e5abbdbdd6f66b6ea9dac3ca3e34c5c632894f8625d13d084cbe"}, - {file = "pydantic-1.10.17-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:62d96b8799ae3d782df7ec9615cb59fc32c32e1ed6afa1b231b0595f6516e8ab"}, - {file = "pydantic-1.10.17-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:ab2f976336808fd5d539fdc26eb51f9aafc1f4b638e212ef6b6f05e753c8011d"}, - {file = "pydantic-1.10.17-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:b8ad363330557beac73159acfbeed220d5f1bfcd6b930302a987a375e02f74fd"}, - {file = "pydantic-1.10.17-cp39-cp39-win_amd64.whl", hash = "sha256:48db882e48575ce4b39659558b2f9f37c25b8d348e37a2b4e32971dd5a7d6227"}, - {file = "pydantic-1.10.17-py3-none-any.whl", hash = "sha256:e41b5b973e5c64f674b3b4720286ded184dcc26a691dd55f34391c62c6934688"}, - {file = "pydantic-1.10.17.tar.gz", hash = "sha256:f434160fb14b353caf634149baaf847206406471ba70e64657c1e8330277a991"}, + {file = "pydantic-2.8.2-py3-none-any.whl", hash = "sha256:73ee9fddd406dc318b885c7a2eab8a6472b68b8fb5ba8150949fc3db939f23c8"}, + {file = "pydantic-2.8.2.tar.gz", hash = "sha256:6f62c13d067b0755ad1c21a34bdd06c0c12625a22b0fc09c6b149816604f7c2a"}, ] [package.dependencies] -typing-extensions = ">=4.2.0" +annotated-types = ">=0.4.0" +pydantic-core = "2.20.1" +typing-extensions = {version = ">=4.6.1", markers = "python_version < \"3.13\""} [package.extras] -dotenv = ["python-dotenv (>=0.10.4)"] -email = ["email-validator (>=1.0.3)"] +email = ["email-validator (>=2.0.0)"] + +[[package]] +name = "pydantic-core" +version = "2.20.1" +description = "Core functionality for Pydantic validation and serialization" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pydantic_core-2.20.1-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:3acae97ffd19bf091c72df4d726d552c473f3576409b2a7ca36b2f535ffff4a3"}, + {file = "pydantic_core-2.20.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:41f4c96227a67a013e7de5ff8f20fb496ce573893b7f4f2707d065907bffdbd6"}, + {file = "pydantic_core-2.20.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5f239eb799a2081495ea659d8d4a43a8f42cd1fe9ff2e7e436295c38a10c286a"}, + {file = "pydantic_core-2.20.1-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:53e431da3fc53360db73eedf6f7124d1076e1b4ee4276b36fb25514544ceb4a3"}, + {file = "pydantic_core-2.20.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f1f62b2413c3a0e846c3b838b2ecd6c7a19ec6793b2a522745b0869e37ab5bc1"}, + {file = "pydantic_core-2.20.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5d41e6daee2813ecceea8eda38062d69e280b39df793f5a942fa515b8ed67953"}, + {file = "pydantic_core-2.20.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3d482efec8b7dc6bfaedc0f166b2ce349df0011f5d2f1f25537ced4cfc34fd98"}, + {file = "pydantic_core-2.20.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:e93e1a4b4b33daed65d781a57a522ff153dcf748dee70b40c7258c5861e1768a"}, + {file = "pydantic_core-2.20.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:e7c4ea22b6739b162c9ecaaa41d718dfad48a244909fe7ef4b54c0b530effc5a"}, + {file = "pydantic_core-2.20.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:4f2790949cf385d985a31984907fecb3896999329103df4e4983a4a41e13e840"}, + {file = "pydantic_core-2.20.1-cp310-none-win32.whl", hash = "sha256:5e999ba8dd90e93d57410c5e67ebb67ffcaadcea0ad973240fdfd3a135506250"}, + {file = "pydantic_core-2.20.1-cp310-none-win_amd64.whl", hash = "sha256:512ecfbefef6dac7bc5eaaf46177b2de58cdf7acac8793fe033b24ece0b9566c"}, + {file = "pydantic_core-2.20.1-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:d2a8fa9d6d6f891f3deec72f5cc668e6f66b188ab14bb1ab52422fe8e644f312"}, + {file = "pydantic_core-2.20.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:175873691124f3d0da55aeea1d90660a6ea7a3cfea137c38afa0a5ffabe37b88"}, + {file = "pydantic_core-2.20.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:37eee5b638f0e0dcd18d21f59b679686bbd18917b87db0193ae36f9c23c355fc"}, + {file = "pydantic_core-2.20.1-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:25e9185e2d06c16ee438ed39bf62935ec436474a6ac4f9358524220f1b236e43"}, + {file = "pydantic_core-2.20.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:150906b40ff188a3260cbee25380e7494ee85048584998c1e66df0c7a11c17a6"}, + {file = "pydantic_core-2.20.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8ad4aeb3e9a97286573c03df758fc7627aecdd02f1da04516a86dc159bf70121"}, + {file = "pydantic_core-2.20.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d3f3ed29cd9f978c604708511a1f9c2fdcb6c38b9aae36a51905b8811ee5cbf1"}, + {file = "pydantic_core-2.20.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:b0dae11d8f5ded51699c74d9548dcc5938e0804cc8298ec0aa0da95c21fff57b"}, + {file = "pydantic_core-2.20.1-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:faa6b09ee09433b87992fb5a2859efd1c264ddc37280d2dd5db502126d0e7f27"}, + {file = "pydantic_core-2.20.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:9dc1b507c12eb0481d071f3c1808f0529ad41dc415d0ca11f7ebfc666e66a18b"}, + {file = "pydantic_core-2.20.1-cp311-none-win32.whl", hash = "sha256:fa2fddcb7107e0d1808086ca306dcade7df60a13a6c347a7acf1ec139aa6789a"}, + {file = "pydantic_core-2.20.1-cp311-none-win_amd64.whl", hash = "sha256:40a783fb7ee353c50bd3853e626f15677ea527ae556429453685ae32280c19c2"}, + {file = "pydantic_core-2.20.1-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:595ba5be69b35777474fa07f80fc260ea71255656191adb22a8c53aba4479231"}, + {file = "pydantic_core-2.20.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:a4f55095ad087474999ee28d3398bae183a66be4823f753cd7d67dd0153427c9"}, + {file = "pydantic_core-2.20.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f9aa05d09ecf4c75157197f27cdc9cfaeb7c5f15021c6373932bf3e124af029f"}, + {file = "pydantic_core-2.20.1-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:e97fdf088d4b31ff4ba35db26d9cc472ac7ef4a2ff2badeabf8d727b3377fc52"}, + {file = "pydantic_core-2.20.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:bc633a9fe1eb87e250b5c57d389cf28998e4292336926b0b6cdaee353f89a237"}, + {file = "pydantic_core-2.20.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d573faf8eb7e6b1cbbcb4f5b247c60ca8be39fe2c674495df0eb4318303137fe"}, + {file = "pydantic_core-2.20.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:26dc97754b57d2fd00ac2b24dfa341abffc380b823211994c4efac7f13b9e90e"}, + {file = "pydantic_core-2.20.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:33499e85e739a4b60c9dac710c20a08dc73cb3240c9a0e22325e671b27b70d24"}, + {file = "pydantic_core-2.20.1-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:bebb4d6715c814597f85297c332297c6ce81e29436125ca59d1159b07f423eb1"}, + {file = "pydantic_core-2.20.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:516d9227919612425c8ef1c9b869bbbee249bc91912c8aaffb66116c0b447ebd"}, + {file = "pydantic_core-2.20.1-cp312-none-win32.whl", hash = "sha256:469f29f9093c9d834432034d33f5fe45699e664f12a13bf38c04967ce233d688"}, + {file = "pydantic_core-2.20.1-cp312-none-win_amd64.whl", hash = "sha256:035ede2e16da7281041f0e626459bcae33ed998cca6a0a007a5ebb73414ac72d"}, + {file = "pydantic_core-2.20.1-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:0827505a5c87e8aa285dc31e9ec7f4a17c81a813d45f70b1d9164e03a813a686"}, + {file = "pydantic_core-2.20.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:19c0fa39fa154e7e0b7f82f88ef85faa2a4c23cc65aae2f5aea625e3c13c735a"}, + {file = "pydantic_core-2.20.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4aa223cd1e36b642092c326d694d8bf59b71ddddc94cdb752bbbb1c5c91d833b"}, + {file = "pydantic_core-2.20.1-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:c336a6d235522a62fef872c6295a42ecb0c4e1d0f1a3e500fe949415761b8a19"}, + {file = "pydantic_core-2.20.1-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7eb6a0587eded33aeefea9f916899d42b1799b7b14b8f8ff2753c0ac1741edac"}, + {file = "pydantic_core-2.20.1-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:70c8daf4faca8da5a6d655f9af86faf6ec2e1768f4b8b9d0226c02f3d6209703"}, + {file = "pydantic_core-2.20.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e9fa4c9bf273ca41f940bceb86922a7667cd5bf90e95dbb157cbb8441008482c"}, + {file = "pydantic_core-2.20.1-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:11b71d67b4725e7e2a9f6e9c0ac1239bbc0c48cce3dc59f98635efc57d6dac83"}, + {file = "pydantic_core-2.20.1-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:270755f15174fb983890c49881e93f8f1b80f0b5e3a3cc1394a255706cabd203"}, + {file = "pydantic_core-2.20.1-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:c81131869240e3e568916ef4c307f8b99583efaa60a8112ef27a366eefba8ef0"}, + {file = "pydantic_core-2.20.1-cp313-none-win32.whl", hash = "sha256:b91ced227c41aa29c672814f50dbb05ec93536abf8f43cd14ec9521ea09afe4e"}, + {file = "pydantic_core-2.20.1-cp313-none-win_amd64.whl", hash = "sha256:65db0f2eefcaad1a3950f498aabb4875c8890438bc80b19362cf633b87a8ab20"}, + {file = "pydantic_core-2.20.1-cp38-cp38-macosx_10_12_x86_64.whl", hash = "sha256:4745f4ac52cc6686390c40eaa01d48b18997cb130833154801a442323cc78f91"}, + {file = "pydantic_core-2.20.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:a8ad4c766d3f33ba8fd692f9aa297c9058970530a32c728a2c4bfd2616d3358b"}, + {file = "pydantic_core-2.20.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:41e81317dd6a0127cabce83c0c9c3fbecceae981c8391e6f1dec88a77c8a569a"}, + {file = "pydantic_core-2.20.1-cp38-cp38-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:04024d270cf63f586ad41fff13fde4311c4fc13ea74676962c876d9577bcc78f"}, + {file = "pydantic_core-2.20.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:eaad4ff2de1c3823fddf82f41121bdf453d922e9a238642b1dedb33c4e4f98ad"}, + {file = "pydantic_core-2.20.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:26ab812fa0c845df815e506be30337e2df27e88399b985d0bb4e3ecfe72df31c"}, + {file = "pydantic_core-2.20.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3c5ebac750d9d5f2706654c638c041635c385596caf68f81342011ddfa1e5598"}, + {file = "pydantic_core-2.20.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:2aafc5a503855ea5885559eae883978c9b6d8c8993d67766ee73d82e841300dd"}, + {file = "pydantic_core-2.20.1-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:4868f6bd7c9d98904b748a2653031fc9c2f85b6237009d475b1008bfaeb0a5aa"}, + {file = "pydantic_core-2.20.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:aa2f457b4af386254372dfa78a2eda2563680d982422641a85f271c859df1987"}, + {file = "pydantic_core-2.20.1-cp38-none-win32.whl", hash = "sha256:225b67a1f6d602de0ce7f6c1c3ae89a4aa25d3de9be857999e9124f15dab486a"}, + {file = "pydantic_core-2.20.1-cp38-none-win_amd64.whl", hash = "sha256:6b507132dcfc0dea440cce23ee2182c0ce7aba7054576efc65634f080dbe9434"}, + {file = "pydantic_core-2.20.1-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:b03f7941783b4c4a26051846dea594628b38f6940a2fdc0df00b221aed39314c"}, + {file = "pydantic_core-2.20.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:1eedfeb6089ed3fad42e81a67755846ad4dcc14d73698c120a82e4ccf0f1f9f6"}, + {file = "pydantic_core-2.20.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:635fee4e041ab9c479e31edda27fcf966ea9614fff1317e280d99eb3e5ab6fe2"}, + {file = "pydantic_core-2.20.1-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:77bf3ac639c1ff567ae3b47f8d4cc3dc20f9966a2a6dd2311dcc055d3d04fb8a"}, + {file = "pydantic_core-2.20.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7ed1b0132f24beeec5a78b67d9388656d03e6a7c837394f99257e2d55b461611"}, + {file = "pydantic_core-2.20.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c6514f963b023aeee506678a1cf821fe31159b925c4b76fe2afa94cc70b3222b"}, + {file = "pydantic_core-2.20.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:10d4204d8ca33146e761c79f83cc861df20e7ae9f6487ca290a97702daf56006"}, + {file = "pydantic_core-2.20.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:2d036c7187b9422ae5b262badb87a20a49eb6c5238b2004e96d4da1231badef1"}, + {file = "pydantic_core-2.20.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:9ebfef07dbe1d93efb94b4700f2d278494e9162565a54f124c404a5656d7ff09"}, + {file = "pydantic_core-2.20.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:6b9d9bb600328a1ce523ab4f454859e9d439150abb0906c5a1983c146580ebab"}, + {file = "pydantic_core-2.20.1-cp39-none-win32.whl", hash = "sha256:784c1214cb6dd1e3b15dd8b91b9a53852aed16671cc3fbe4786f4f1db07089e2"}, + {file = "pydantic_core-2.20.1-cp39-none-win_amd64.whl", hash = "sha256:d2fe69c5434391727efa54b47a1e7986bb0186e72a41b203df8f5b0a19a4f669"}, + {file = "pydantic_core-2.20.1-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:a45f84b09ac9c3d35dfcf6a27fd0634d30d183205230a0ebe8373a0e8cfa0906"}, + {file = "pydantic_core-2.20.1-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:d02a72df14dfdbaf228424573a07af10637bd490f0901cee872c4f434a735b94"}, + {file = "pydantic_core-2.20.1-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d2b27e6af28f07e2f195552b37d7d66b150adbaa39a6d327766ffd695799780f"}, + {file = "pydantic_core-2.20.1-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:084659fac3c83fd674596612aeff6041a18402f1e1bc19ca39e417d554468482"}, + {file = "pydantic_core-2.20.1-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:242b8feb3c493ab78be289c034a1f659e8826e2233786e36f2893a950a719bb6"}, + {file = "pydantic_core-2.20.1-pp310-pypy310_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:38cf1c40a921d05c5edc61a785c0ddb4bed67827069f535d794ce6bcded919fc"}, + {file = "pydantic_core-2.20.1-pp310-pypy310_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:e0bbdd76ce9aa5d4209d65f2b27fc6e5ef1312ae6c5333c26db3f5ade53a1e99"}, + {file = "pydantic_core-2.20.1-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:254ec27fdb5b1ee60684f91683be95e5133c994cc54e86a0b0963afa25c8f8a6"}, + {file = "pydantic_core-2.20.1-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:407653af5617f0757261ae249d3fba09504d7a71ab36ac057c938572d1bc9331"}, + {file = "pydantic_core-2.20.1-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:c693e916709c2465b02ca0ad7b387c4f8423d1db7b4649c551f27a529181c5ad"}, + {file = "pydantic_core-2.20.1-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5b5ff4911aea936a47d9376fd3ab17e970cc543d1b68921886e7f64bd28308d1"}, + {file = "pydantic_core-2.20.1-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:177f55a886d74f1808763976ac4efd29b7ed15c69f4d838bbd74d9d09cf6fa86"}, + {file = "pydantic_core-2.20.1-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:964faa8a861d2664f0c7ab0c181af0bea66098b1919439815ca8803ef136fc4e"}, + {file = "pydantic_core-2.20.1-pp39-pypy39_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:4dd484681c15e6b9a977c785a345d3e378d72678fd5f1f3c0509608da24f2ac0"}, + {file = "pydantic_core-2.20.1-pp39-pypy39_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:f6d6cff3538391e8486a431569b77921adfcdef14eb18fbf19b7c0a5294d4e6a"}, + {file = "pydantic_core-2.20.1-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:a6d511cc297ff0883bc3708b465ff82d7560193169a8b93260f74ecb0a5e08a7"}, + {file = "pydantic_core-2.20.1.tar.gz", hash = "sha256:26ca695eeee5f9f1aeeb211ffc12f10bcb6f71e2989988fda61dabd65db878d4"}, +] + +[package.dependencies] +typing-extensions = ">=4.6.0,<4.7.0 || >4.7.0" [[package]] name = "pyjwt" @@ -1155,18 +1232,18 @@ fixture = ["fixtures"] [[package]] name = "setuptools" -version = "70.1.1" +version = "70.2.0" description = "Easily download, build, install, upgrade, and uninstall Python packages" optional = false python-versions = ">=3.8" files = [ - {file = "setuptools-70.1.1-py3-none-any.whl", hash = "sha256:a58a8fde0541dab0419750bcc521fbdf8585f6e5cb41909df3a472ef7b81ca95"}, - {file = "setuptools-70.1.1.tar.gz", hash = "sha256:937a48c7cdb7a21eb53cd7f9b59e525503aa8abaf3584c730dc5f7a5bec3a650"}, + {file = "setuptools-70.2.0-py3-none-any.whl", hash = "sha256:b8b8060bb426838fbe942479c90296ce976249451118ef566a5a0b7d8b78fb05"}, + {file = "setuptools-70.2.0.tar.gz", hash = "sha256:bd63e505105011b25c3c11f753f7e3b8465ea739efddaccef8f0efac2137bac1"}, ] [package.extras] -docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "pyproject-hooks (!=1.1)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier"] -testing = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "importlib-metadata", "ini2toml[lite] (>=0.14)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "jaraco.test", "mypy (==1.10.0)", "packaging (>=23.2)", "pip (>=19.1)", "pyproject-hooks (!=1.1)", "pytest (>=6,!=8.1.1)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-home (>=0.5)", "pytest-mypy", "pytest-perf", "pytest-ruff (>=0.3.2)", "pytest-subprocess", "pytest-timeout", "pytest-xdist (>=3)", "tomli", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] +doc = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "pyproject-hooks (!=1.1)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier"] +test = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "importlib-metadata", "ini2toml[lite] (>=0.14)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "jaraco.test", "mypy (==1.10.0)", "packaging (>=23.2)", "pip (>=19.1)", "pyproject-hooks (!=1.1)", "pytest (>=6,!=8.1.*)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-home (>=0.5)", "pytest-mypy", "pytest-perf", "pytest-ruff (>=0.3.2)", "pytest-subprocess", "pytest-timeout", "pytest-xdist (>=3)", "tomli", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] [[package]] name = "six" @@ -1181,13 +1258,13 @@ files = [ [[package]] name = "tenacity" -version = "8.4.2" +version = "8.5.0" description = "Retry code until it succeeds" optional = false python-versions = ">=3.8" files = [ - {file = "tenacity-8.4.2-py3-none-any.whl", hash = "sha256:9e6f7cf7da729125c7437222f8a522279751cdfbe6b67bfe64f75d3a348661b2"}, - {file = "tenacity-8.4.2.tar.gz", hash = "sha256:cd80a53a79336edba8489e767f729e4f391c896956b57140b5d7511a64bbd3ef"}, + {file = "tenacity-8.5.0-py3-none-any.whl", hash = "sha256:b594c2a5945830c267ce6b79a166228323ed52718f30302c1359836112346687"}, + {file = "tenacity-8.5.0.tar.gz", hash = "sha256:8bc6c0c8a09b31e6cad13c47afbed1a567518250a9a171418582ed8d9c20ca78"}, ] [package.extras] @@ -1343,4 +1420,4 @@ files = [ [metadata] lock-version = "2.0" python-versions = "^3.9,<3.12" -content-hash = "8e983cc17e259e1923dd43d392d98ac081169e47e2d7c23d488798fbc8ae348e" +content-hash = "5c9e0f9a32f316f08b2fa20279601e968bad24cb2f47815af067b91689933b20" diff --git a/airbyte-integrations/connectors/source-twilio/pyproject.toml b/airbyte-integrations/connectors/source-twilio/pyproject.toml index f9a85ca32c45..6531bf26413e 100644 --- a/airbyte-integrations/connectors/source-twilio/pyproject.toml +++ b/airbyte-integrations/connectors/source-twilio/pyproject.toml @@ -17,7 +17,7 @@ include = "source_twilio" [tool.poetry.dependencies] python = "^3.9,<3.12" -airbyte-cdk = "^1" +airbyte-cdk = "^2" pendulum = "==2.1.2" requests = "==2.31.0" diff --git a/airbyte-integrations/connectors/source-twilio/source_twilio/manifest.yaml b/airbyte-integrations/connectors/source-twilio/source_twilio/manifest.yaml index 24b44c6547dd..454cb378d0cb 100644 --- a/airbyte-integrations/connectors/source-twilio/source_twilio/manifest.yaml +++ b/airbyte-integrations/connectors/source-twilio/source_twilio/manifest.yaml @@ -39,13 +39,13 @@ definitions: - type: AddedFieldDefinition path: - date_created - value: '{% set formatted_datetime = format_datetime(record["date_created"], "%Y-%m-%dT%H:%M:%SZ") %}{{ formatted_datetime }}' + value: '{% set formatted_datetime = format_datetime(record["date_created"], "%Y-%m-%dT%H:%M:%SZ", "%a, %d %b %Y %H:%M:%S %z") %}{{ formatted_datetime }}' - type: AddFields fields: - type: AddedFieldDefinition path: - date_updated - value: '{% set formatted_datetime = format_datetime(record["date_updated"], "%Y-%m-%dT%H:%M:%SZ") %}{{ formatted_datetime }}' + value: '{% set formatted_datetime = format_datetime(record["date_updated"], "%Y-%m-%dT%H:%M:%SZ", "%a, %d %b %Y %H:%M:%S %z") %}{{ formatted_datetime }}' schema_loader: type: InlineSchemaLoader schema: @@ -97,13 +97,13 @@ definitions: - type: AddedFieldDefinition path: - date_created - value: '{% set formatted_datetime = format_datetime(record["date_created"], "%Y-%m-%dT%H:%M:%SZ") %}{{ formatted_datetime }}' + value: '{% set formatted_datetime = format_datetime(record["date_created"], "%Y-%m-%dT%H:%M:%SZ", "%a, %d %b %Y %H:%M:%S %z") %}{{ formatted_datetime }}' - type: AddFields fields: - type: AddedFieldDefinition path: - date_updated - value: '{% set formatted_datetime = format_datetime(record["date_updated"], "%Y-%m-%dT%H:%M:%SZ") %}{{ formatted_datetime }}' + value: '{% set formatted_datetime = format_datetime(record["date_updated"], "%Y-%m-%dT%H:%M:%SZ", "%a, %d %b %Y %H:%M:%S %z") %}{{ formatted_datetime }}' schema_loader: type: InlineSchemaLoader schema: @@ -349,13 +349,13 @@ definitions: - type: AddedFieldDefinition path: - date_created - value: '{% set formatted_datetime = format_datetime(record["date_created"], "%Y-%m-%dT%H:%M:%SZ") %}{{ formatted_datetime }}' + value: '{% set formatted_datetime = format_datetime(record["date_created"], "%Y-%m-%dT%H:%M:%SZ", "%a, %d %b %Y %H:%M:%S %z") %}{{ formatted_datetime }}' - type: AddFields fields: - type: AddedFieldDefinition path: - date_updated - value: '{% set formatted_datetime = format_datetime(record["date_updated"], "%Y-%m-%dT%H:%M:%SZ") %}{{ formatted_datetime }}' + value: '{% set formatted_datetime = format_datetime(record["date_updated"], "%Y-%m-%dT%H:%M:%SZ", "%a, %d %b %Y %H:%M:%S %z") %}{{ formatted_datetime }}' schema_loader: type: InlineSchemaLoader schema: @@ -404,13 +404,13 @@ definitions: - type: AddedFieldDefinition path: - date_created - value: '{% set formatted_datetime = format_datetime(record["date_created"], "%Y-%m-%dT%H:%M:%SZ") %}{{ formatted_datetime }}' + value: '{% set formatted_datetime = format_datetime(record["date_created"], "%Y-%m-%dT%H:%M:%SZ", "%a, %d %b %Y %H:%M:%S %z") %}{{ formatted_datetime }}' - type: AddFields fields: - type: AddedFieldDefinition path: - date_updated - value: '{% set formatted_datetime = format_datetime(record["date_updated"], "%Y-%m-%dT%H:%M:%SZ") %}{{ formatted_datetime }}' + value: '{% set formatted_datetime = format_datetime(record["date_updated"], "%Y-%m-%dT%H:%M:%SZ", "%a, %d %b %Y %H:%M:%S %z") %}{{ formatted_datetime }}' schema_loader: type: InlineSchemaLoader schema: @@ -446,19 +446,6 @@ definitions: type: PageIncrement page_size: 1000 start_from_page: 0 - transformations: - - type: AddFields - fields: - - type: AddedFieldDefinition - path: - - date_created - value: '{% set formatted_datetime = format_datetime(record["date_created"], "%Y-%m-%dT%H:%M:%SZ") %}{{ formatted_datetime }}' - - type: AddFields - fields: - - type: AddedFieldDefinition - path: - - date_updated - value: '{% set formatted_datetime = format_datetime(record["date_updated"], "%Y-%m-%dT%H:%M:%SZ") %}{{ formatted_datetime }}' schema_loader: type: InlineSchemaLoader schema: @@ -516,13 +503,13 @@ definitions: - type: AddedFieldDefinition path: - date_created - value: '{% set formatted_datetime = format_datetime(record["date_created"], "%Y-%m-%dT%H:%M:%SZ") %}{{ formatted_datetime }}' + value: '{% set formatted_datetime = format_datetime(record["date_created"], "%Y-%m-%dT%H:%M:%SZ", "%a, %d %b %Y %H:%M:%S %z") %}{{ formatted_datetime }}' - type: AddFields fields: - type: AddedFieldDefinition path: - date_updated - value: '{% set formatted_datetime = format_datetime(record["date_updated"], "%Y-%m-%dT%H:%M:%SZ") %}{{ formatted_datetime }}' + value: '{% set formatted_datetime = format_datetime(record["date_updated"], "%Y-%m-%dT%H:%M:%SZ", "%a, %d %b %Y %H:%M:%S %z") %}{{ formatted_datetime }}' schema_loader: type: InlineSchemaLoader schema: @@ -570,19 +557,19 @@ definitions: - type: AddedFieldDefinition path: - date_created - value: '{% set formatted_datetime = format_datetime(record["date_created"], "%Y-%m-%dT%H:%M:%SZ") %}{{ formatted_datetime }}' + value: '{% set formatted_datetime = format_datetime(record["date_created"], "%Y-%m-%dT%H:%M:%SZ", "%a, %d %b %Y %H:%M:%S %z") %}{{ formatted_datetime }}' - type: AddFields fields: - type: AddedFieldDefinition path: - date_updated - value: '{% set formatted_datetime = format_datetime(record["date_updated"], "%Y-%m-%dT%H:%M:%SZ") %}{{ formatted_datetime }}' + value: '{% set formatted_datetime = format_datetime(record["date_updated"], "%Y-%m-%dT%H:%M:%SZ", "%a, %d %b %Y %H:%M:%S %z") %}{{ formatted_datetime }}' - type: AddFields fields: - type: AddedFieldDefinition path: - date_generated - value: '{% set formatted_datetime = format_datetime(record["date_generated"], "%Y-%m-%dT%H:%M:%SZ") %}{{ formatted_datetime }}' + value: '{% set formatted_datetime = format_datetime(record["date_generated"], "%Y-%m-%dT%H:%M:%SZ", "%a, %d %b %Y %H:%M:%S %z") %}{{ formatted_datetime }}' schema_loader: type: InlineSchemaLoader schema: @@ -634,13 +621,13 @@ definitions: - type: AddedFieldDefinition path: - date_created - value: '{% set formatted_datetime = format_datetime(record["date_created"], "%Y-%m-%dT%H:%M:%SZ") %}{{ formatted_datetime }}' + value: '{% set formatted_datetime = format_datetime(record["date_created"], "%Y-%m-%dT%H:%M:%SZ", "%a, %d %b %Y %H:%M:%S %z") %}{{ formatted_datetime }}' - type: AddFields fields: - type: AddedFieldDefinition path: - date_updated - value: '{% set formatted_datetime = format_datetime(record["date_updated"], "%Y-%m-%dT%H:%M:%SZ") %}{{ formatted_datetime }}' + value: '{% set formatted_datetime = format_datetime(record["date_updated"], "%Y-%m-%dT%H:%M:%SZ", "%a, %d %b %Y %H:%M:%S %z") %}{{ formatted_datetime }}' schema_loader: type: InlineSchemaLoader schema: @@ -700,13 +687,13 @@ definitions: - type: AddedFieldDefinition path: - date_created - value: '{% set formatted_datetime = format_datetime(record["date_created"], "%Y-%m-%dT%H:%M:%SZ") %}{{ formatted_datetime }}' + value: '{% set formatted_datetime = format_datetime(record["date_created"], "%Y-%m-%dT%H:%M:%SZ", "%a, %d %b %Y %H:%M:%S %z") %}{{ formatted_datetime }}' - type: AddFields fields: - type: AddedFieldDefinition path: - date_updated - value: '{% set formatted_datetime = format_datetime(record["date_updated"], "%Y-%m-%dT%H:%M:%SZ") %}{{ formatted_datetime }}' + value: '{% set formatted_datetime = format_datetime(record["date_updated"], "%Y-%m-%dT%H:%M:%SZ", "%a, %d %b %Y %H:%M:%S %z") %}{{ formatted_datetime }}' schema_loader: type: InlineSchemaLoader schema: @@ -756,7 +743,7 @@ definitions: type: DatetimeBasedCursor cursor_field: end_time cursor_datetime_formats: - - "%a, %d %b %Y %H:%M:%S %z" + - "%Y-%m-%dT%H:%M:%SZ" datetime_format: "%Y-%m-%d" start_datetime: type: MinMaxDatetime @@ -780,13 +767,43 @@ definitions: - type: AddedFieldDefinition path: - date_created - value: '{% set formatted_datetime = format_datetime(record["date_created"], "%Y-%m-%dT%H:%M:%SZ") %}{{ formatted_datetime }}' + value: '{% set formatted_datetime = format_datetime(record["date_created"], "%Y-%m-%dT%H:%M:%SZ", "%a, %d %b %Y %H:%M:%S %z") %}{{ formatted_datetime }}' - type: AddFields fields: - type: AddedFieldDefinition path: - date_updated - value: '{% set formatted_datetime = format_datetime(record["date_updated"], "%Y-%m-%dT%H:%M:%SZ") %}{{ formatted_datetime }}' + value: '{% set formatted_datetime = format_datetime(record["date_updated"], "%Y-%m-%dT%H:%M:%SZ", "%a, %d %b %Y %H:%M:%S %z") %}{{ formatted_datetime }}' + - type: AddFields + fields: + - type: AddedFieldDefinition + path: + - start_time + value: '{% set formatted_datetime = format_datetime(record["start_time"], "%Y-%m-%dT%H:%M:%SZ", "%a, %d %b %Y %H:%M:%S %z") %}{{ formatted_datetime }}' + - type: AddFields + fields: + - type: AddedFieldDefinition + path: + - end_time + value: '{% set formatted_datetime = format_datetime(record["end_time"], "%Y-%m-%dT%H:%M:%SZ", "%a, %d %b %Y %H:%M:%S %z") %}{{ formatted_datetime }}' + - type: AddFields + fields: + - type: AddedFieldDefinition + path: + - duration + value: '{% set duration = record["duration"] | int %}{{ duration }}' + - type: AddFields + fields: + - type: AddedFieldDefinition + path: + - price + value: '{% set price = record["price"] | float %}{{ price }}' + - type: AddFields + fields: + - type: AddedFieldDefinition + path: + - queue_time + value: '{% set queue_time = record["queue_time"] | int %}{{ queue_time }}' schema_loader: type: InlineSchemaLoader schema: @@ -836,7 +853,7 @@ definitions: type: DatetimeBasedCursor cursor_field: date_created cursor_datetime_formats: - - "%a, %d %b %Y %H:%M:%S %z" + - "%Y-%m-%dT%H:%M:%SZ" datetime_format: "%Y-%m-%d" start_datetime: type: MinMaxDatetime @@ -860,13 +877,13 @@ definitions: - type: AddedFieldDefinition path: - date_created - value: '{% set formatted_datetime = format_datetime(record["date_created"], "%Y-%m-%dT%H:%M:%SZ") %}{{ formatted_datetime }}' + value: '{% set formatted_datetime = format_datetime(record["date_created"], "%Y-%m-%dT%H:%M:%SZ", "%a, %d %b %Y %H:%M:%S %z") %}{{ formatted_datetime }}' - type: AddFields fields: - type: AddedFieldDefinition path: - date_updated - value: '{% set formatted_datetime = format_datetime(record["date_updated"], "%Y-%m-%dT%H:%M:%SZ") %}{{ formatted_datetime }}' + value: '{% set formatted_datetime = format_datetime(record["date_updated"], "%Y-%m-%dT%H:%M:%SZ", "%a, %d %b %Y %H:%M:%S %z") %}{{ formatted_datetime }}' schema_loader: type: InlineSchemaLoader schema: @@ -927,13 +944,13 @@ definitions: - type: AddedFieldDefinition path: - date_created - value: '{% set formatted_datetime = format_datetime(record["date_created"], "%Y-%m-%dT%H:%M:%SZ") %}{{ formatted_datetime }}' + value: '{% set formatted_datetime = format_datetime(record["date_created"], "%Y-%m-%dT%H:%M:%SZ", "%a, %d %b %Y %H:%M:%S %z") %}{{ formatted_datetime }}' - type: AddFields fields: - type: AddedFieldDefinition path: - date_updated - value: '{% set formatted_datetime = format_datetime(record["date_updated"], "%Y-%m-%dT%H:%M:%SZ") %}{{ formatted_datetime }}' + value: '{% set formatted_datetime = format_datetime(record["date_updated"], "%Y-%m-%dT%H:%M:%SZ", "%a, %d %b %Y %H:%M:%S %z") %}{{ formatted_datetime }}' schema_loader: type: InlineSchemaLoader schema: @@ -985,13 +1002,13 @@ definitions: - type: AddedFieldDefinition path: - date_created - value: '{% set formatted_datetime = format_datetime(record["date_created"], "%Y-%m-%dT%H:%M:%SZ") %}{{ formatted_datetime }}' + value: '{% set formatted_datetime = format_datetime(record["date_created"], "%Y-%m-%dT%H:%M:%SZ", "%a, %d %b %Y %H:%M:%S %z") %}{{ formatted_datetime }}' - type: AddFields fields: - type: AddedFieldDefinition path: - date_updated - value: '{% set formatted_datetime = format_datetime(record["date_updated"], "%Y-%m-%dT%H:%M:%SZ") %}{{ formatted_datetime }}' + value: '{% set formatted_datetime = format_datetime(record["date_updated"], "%Y-%m-%dT%H:%M:%SZ", "%a, %d %b %Y %H:%M:%S %z") %}{{ formatted_datetime }}' schema_loader: type: InlineSchemaLoader schema: @@ -1042,13 +1059,13 @@ definitions: - type: AddedFieldDefinition path: - date_created - value: '{% set formatted_datetime = format_datetime(record["date_created"], "%Y-%m-%dT%H:%M:%SZ") %}{{ formatted_datetime }}' + value: '{% set formatted_datetime = format_datetime(record["date_created"], "%Y-%m-%dT%H:%M:%SZ", "%a, %d %b %Y %H:%M:%S %z") %}{{ formatted_datetime }}' - type: AddFields fields: - type: AddedFieldDefinition path: - date_updated - value: '{% set formatted_datetime = format_datetime(record["date_updated"], "%Y-%m-%dT%H:%M:%SZ") %}{{ formatted_datetime }}' + value: '{% set formatted_datetime = format_datetime(record["date_updated"], "%Y-%m-%dT%H:%M:%SZ", "%a, %d %b %Y %H:%M:%S %z") %}{{ formatted_datetime }}' schema_loader: type: InlineSchemaLoader schema: @@ -1098,7 +1115,7 @@ definitions: type: DatetimeBasedCursor cursor_field: date_created cursor_datetime_formats: - - "%a, %d %b %Y %H:%M:%S %z" + - "%Y-%m-%dT%H:%M:%SZ" datetime_format: "%Y-%m-%d" start_datetime: type: MinMaxDatetime @@ -1122,13 +1139,31 @@ definitions: - type: AddedFieldDefinition path: - date_created - value: '{% set formatted_datetime = format_datetime(record["date_created"], "%Y-%m-%dT%H:%M:%SZ") %}{{ formatted_datetime }}' + value: '{% set formatted_datetime = format_datetime(record["date_created"], "%Y-%m-%dT%H:%M:%SZ", "%a, %d %b %Y %H:%M:%S %z") %}{{ formatted_datetime }}' - type: AddFields fields: - type: AddedFieldDefinition path: - date_updated - value: '{% set formatted_datetime = format_datetime(record["date_updated"], "%Y-%m-%dT%H:%M:%SZ") %}{{ formatted_datetime }}' + value: '{% set formatted_datetime = format_datetime(record["date_updated"], "%Y-%m-%dT%H:%M:%SZ", "%a, %d %b %Y %H:%M:%S %z") %}{{ formatted_datetime }}' + - type: AddFields + fields: + - type: AddedFieldDefinition + path: + - start_time + value: '{% set formatted_datetime = format_datetime(record["start_time"], "%Y-%m-%dT%H:%M:%SZ", "%a, %d %b %Y %H:%M:%S %z") %}{{ formatted_datetime }}' + - type: AddFields + fields: + - type: AddedFieldDefinition + path: + - duration + value: '{% set duration = record["duration"] | int %}{{ duration }}' + - type: AddFields + fields: + - type: AddedFieldDefinition + path: + - price + value: '{% set price = record["price"] | float %}{{ price }}' schema_loader: type: InlineSchemaLoader schema: @@ -1148,19 +1183,6 @@ definitions: type: DpathExtractor field_path: - services - transformations: - - type: AddFields - fields: - - type: AddedFieldDefinition - path: - - date_created - value: '{% set formatted_datetime = format_datetime(record["date_created"], "%Y-%m-%dT%H:%M:%SZ") %}{{ formatted_datetime }}' - - type: AddFields - fields: - - type: AddedFieldDefinition - path: - - date_updated - value: '{% set formatted_datetime = format_datetime(record["date_updated"], "%Y-%m-%dT%H:%M:%SZ") %}{{ formatted_datetime }}' schema_loader: type: InlineSchemaLoader schema: @@ -1182,19 +1204,6 @@ definitions: type: DpathExtractor field_path: - services - transformations: - - type: AddFields - fields: - - type: AddedFieldDefinition - path: - - date_created - value: '{% set formatted_datetime = format_datetime(record["date_created"], "%Y-%m-%dT%H:%M:%SZ") %}{{ formatted_datetime }}' - - type: AddFields - fields: - - type: AddedFieldDefinition - path: - - date_updated - value: '{% set formatted_datetime = format_datetime(record["date_updated"], "%Y-%m-%dT%H:%M:%SZ") %}{{ formatted_datetime }}' schema_loader: type: InlineSchemaLoader schema: @@ -1226,19 +1235,6 @@ definitions: partition_field: service_sid stream: $ref: "#/definitions/streams/Services" - transformations: - - type: AddFields - fields: - - type: AddedFieldDefinition - path: - - date_created - value: '{% set formatted_datetime = format_datetime(record["date_created"], "%Y-%m-%dT%H:%M:%SZ") %}{{ formatted_datetime }}' - - type: AddFields - fields: - - type: AddedFieldDefinition - path: - - date_updated - value: '{% set formatted_datetime = format_datetime(record["date_updated"], "%Y-%m-%dT%H:%M:%SZ") %}{{ formatted_datetime }}' schema_loader: type: InlineSchemaLoader schema: @@ -1290,13 +1286,13 @@ definitions: - type: AddedFieldDefinition path: - date_created - value: '{% set formatted_datetime = format_datetime(record["date_created"], "%Y-%m-%dT%H:%M:%SZ") %}{{ formatted_datetime }}' + value: '{% set formatted_datetime = format_datetime(record["date_created"], "%Y-%m-%dT%H:%M:%SZ", "%a, %d %b %Y %H:%M:%S %z") %}{{ formatted_datetime }}' - type: AddFields fields: - type: AddedFieldDefinition path: - date_updated - value: '{% set formatted_datetime = format_datetime(record["date_updated"], "%Y-%m-%dT%H:%M:%SZ") %}{{ formatted_datetime }}' + value: '{% set formatted_datetime = format_datetime(record["date_updated"], "%Y-%m-%dT%H:%M:%SZ", "%a, %d %b %Y %H:%M:%S %z") %}{{ formatted_datetime }}' - type: AddFields fields: - type: AddedFieldDefinition @@ -1308,7 +1304,7 @@ definitions: - type: AddedFieldDefinition path: - price - value: '{% set price = record["price"] | number %}{{ price }}' + value: '{% set price = record["price"] | float %}{{ price }}' schema_loader: type: InlineSchemaLoader schema: @@ -1330,19 +1326,6 @@ definitions: type: DpathExtractor field_path: - trunks - transformations: - - type: AddFields - fields: - - type: AddedFieldDefinition - path: - - date_created - value: '{% set formatted_datetime = format_datetime(record["date_created"], "%Y-%m-%dT%H:%M:%SZ") %}{{ formatted_datetime }}' - - type: AddFields - fields: - - type: AddedFieldDefinition - path: - - date_updated - value: '{% set formatted_datetime = format_datetime(record["date_updated"], "%Y-%m-%dT%H:%M:%SZ") %}{{ formatted_datetime }}' schema_loader: type: InlineSchemaLoader schema: @@ -1394,13 +1377,13 @@ definitions: - type: AddedFieldDefinition path: - date_created - value: '{% set formatted_datetime = format_datetime(record["date_created"], "%Y-%m-%dT%H:%M:%SZ") %}{{ formatted_datetime }}' + value: '{% set formatted_datetime = format_datetime(record["date_created"], "%Y-%m-%dT%H:%M:%SZ", "%a, %d %b %Y %H:%M:%S %z") %}{{ formatted_datetime }}' - type: AddFields fields: - type: AddedFieldDefinition path: - date_updated - value: '{% set formatted_datetime = format_datetime(record["date_updated"], "%Y-%m-%dT%H:%M:%SZ") %}{{ formatted_datetime }}' + value: '{% set formatted_datetime = format_datetime(record["date_updated"], "%Y-%m-%dT%H:%M:%SZ", "%a, %d %b %Y %H:%M:%S %z") %}{{ formatted_datetime }}' schema_loader: type: InlineSchemaLoader schema: @@ -1450,7 +1433,7 @@ definitions: type: DatetimeBasedCursor cursor_field: date_sent cursor_datetime_formats: - - "%a, %d %b %Y %H:%M:%S %z" + - "%Y-%m-%dT%H:%M:%SZ" datetime_format: "%Y-%m-%d" start_datetime: type: MinMaxDatetime @@ -1474,19 +1457,37 @@ definitions: - type: AddedFieldDefinition path: - date_created - value: '{% set formatted_datetime = format_datetime(record["date_created"], "%Y-%m-%dT%H:%M:%SZ") %}{{ formatted_datetime }}' + value: '{% set formatted_datetime = format_datetime(record["date_created"], "%Y-%m-%dT%H:%M:%SZ", "%a, %d %b %Y %H:%M:%S %z") %}{{ formatted_datetime }}' - type: AddFields fields: - type: AddedFieldDefinition path: - date_updated - value: '{% set formatted_datetime = format_datetime(record["date_updated"], "%Y-%m-%dT%H:%M:%SZ") %}{{ formatted_datetime }}' + value: '{% set formatted_datetime = format_datetime(record["date_updated"], "%Y-%m-%dT%H:%M:%SZ", "%a, %d %b %Y %H:%M:%S %z") %}{{ formatted_datetime }}' - type: AddFields fields: - type: AddedFieldDefinition path: - date_sent - value: '{% set formatted_datetime = format_datetime(record["date_sent"], "%Y-%m-%dT%H:%M:%SZ") %}{{ formatted_datetime }}' + value: '{% set formatted_datetime = format_datetime(record["date_sent"], "%Y-%m-%dT%H:%M:%SZ", "%a, %d %b %Y %H:%M:%S %z") %}{{ formatted_datetime }}' + - type: AddFields + fields: + - type: AddedFieldDefinition + path: + - num_media + value: '{% set num_media = record["num_media"] | int %}{{ num_media }}' + - type: AddFields + fields: + - type: AddedFieldDefinition + path: + - num_segments + value: '{% set num_segments = record["num_segments"] | int %}{{ num_segments }}' + - type: AddFields + fields: + - type: AddedFieldDefinition + path: + - price + value: '{% set price = record["price"] | float %}{{ price }}' schema_loader: type: InlineSchemaLoader schema: @@ -1544,7 +1545,7 @@ definitions: type: DatetimeBasedCursor cursor_field: date_created cursor_datetime_formats: - - "%a, %d %b %Y %H:%M:%S %z" + - "%Y-%m-%dT%H:%M:%SZ" datetime_format: "%Y-%m-%d" start_datetime: type: MinMaxDatetime @@ -1562,19 +1563,6 @@ definitions: type: MinMaxDatetime datetime: "{{ now_utc().strftime('%Y-%m-%dT%H:%M:%SZ') }}" datetime_format: "%Y-%m-%dT%H:%M:%SZ" - transformations: - - type: AddFields - fields: - - type: AddedFieldDefinition - path: - - date_created - value: '{% set formatted_datetime = format_datetime(record["date_created"], "%Y-%m-%dT%H:%M:%SZ") %}{{ formatted_datetime }}' - - type: AddFields - fields: - - type: AddedFieldDefinition - path: - - date_updated - value: '{% set formatted_datetime = format_datetime(record["date_updated"], "%Y-%m-%dT%H:%M:%SZ") %}{{ formatted_datetime }}' schema_loader: type: InlineSchemaLoader schema: @@ -1625,7 +1613,7 @@ definitions: type: DatetimeBasedCursor cursor_field: start_date cursor_datetime_formats: - - "%a, %d %b %Y %H:%M:%S %z" + - "%Y-%m-%dT%H:%M:%SZ" datetime_format: "%Y-%m-%d" start_datetime: type: MinMaxDatetime @@ -1694,19 +1682,25 @@ definitions: - type: AddedFieldDefinition path: - date_created - value: '{% set formatted_datetime = format_datetime(record["date_created"], "%Y-%m-%dT%H:%M:%SZ") %}{{ formatted_datetime }}' + value: '{% set formatted_datetime = format_datetime(record["date_created"], "%Y-%m-%dT%H:%M:%SZ", "%a, %d %b %Y %H:%M:%S %z") %}{{ formatted_datetime }}' - type: AddFields fields: - type: AddedFieldDefinition path: - date_updated - value: '{% set formatted_datetime = format_datetime(record["date_updated"], "%Y-%m-%dT%H:%M:%SZ") %}{{ formatted_datetime }}' + value: '{% set formatted_datetime = format_datetime(record["date_updated"], "%Y-%m-%dT%H:%M:%SZ", "%a, %d %b %Y %H:%M:%S %z") %}{{ formatted_datetime }}' - type: AddFields fields: - type: AddedFieldDefinition path: - - date_fired - value: '{% set formatted_datetime = format_datetime(record["date_fired"], "%Y-%m-%dT%H:%M:%SZ") %}{{ formatted_datetime }}' + - current_value + value: '{% set current_value = record["current_value"] | float %}{{ current_value }}' + - type: AddFields + fields: + - type: AddedFieldDefinition + path: + - trigger_value + value: '{% set trigger_value = record["trigger_value"] | float %}{{ trigger_value }}' schema_loader: type: InlineSchemaLoader schema: @@ -1728,19 +1722,6 @@ definitions: type: DpathExtractor field_path: - conversations - transformations: - - type: AddFields - fields: - - type: AddedFieldDefinition - path: - - date_created - value: '{% set formatted_datetime = format_datetime(record["date_created"], "%Y-%m-%dT%H:%M:%SZ") %}{{ formatted_datetime }}' - - type: AddFields - fields: - - type: AddedFieldDefinition - path: - - date_updated - value: '{% set formatted_datetime = format_datetime(record["date_updated"], "%Y-%m-%dT%H:%M:%SZ") %}{{ formatted_datetime }}' schema_loader: type: InlineSchemaLoader schema: @@ -1772,19 +1753,6 @@ definitions: partition_field: conversation_sid stream: $ref: "#/definitions/streams/Conversations" - transformations: - - type: AddFields - fields: - - type: AddedFieldDefinition - path: - - date_created - value: '{% set formatted_datetime = format_datetime(record["date_created"], "%Y-%m-%dT%H:%M:%SZ") %}{{ formatted_datetime }}' - - type: AddFields - fields: - - type: AddedFieldDefinition - path: - - date_updated - value: '{% set formatted_datetime = format_datetime(record["date_updated"], "%Y-%m-%dT%H:%M:%SZ") %}{{ formatted_datetime }}' schema_loader: type: InlineSchemaLoader schema: @@ -1816,19 +1784,6 @@ definitions: partition_field: conversation_sid stream: $ref: "#/definitions/streams/Conversations" - transformations: - - type: AddFields - fields: - - type: AddedFieldDefinition - path: - - date_created - value: '{% set formatted_datetime = format_datetime(record["date_created"], "%Y-%m-%dT%H:%M:%SZ") %}{{ formatted_datetime }}' - - type: AddFields - fields: - - type: AddedFieldDefinition - path: - - date_updated - value: '{% set formatted_datetime = format_datetime(record["date_updated"], "%Y-%m-%dT%H:%M:%SZ") %}{{ formatted_datetime }}' schema_loader: type: InlineSchemaLoader schema: @@ -1850,19 +1805,6 @@ definitions: type: DpathExtractor field_path: - users - transformations: - - type: AddFields - fields: - - type: AddedFieldDefinition - path: - - date_created - value: '{% set formatted_datetime = format_datetime(record["date_created"], "%Y-%m-%dT%H:%M:%SZ") %}{{ formatted_datetime }}' - - type: AddFields - fields: - - type: AddedFieldDefinition - path: - - date_updated - value: '{% set formatted_datetime = format_datetime(record["date_updated"], "%Y-%m-%dT%H:%M:%SZ") %}{{ formatted_datetime }}' schema_loader: type: InlineSchemaLoader schema: @@ -1894,19 +1836,6 @@ definitions: partition_field: user_sid stream: $ref: "#/definitions/streams/Users" - transformations: - - type: AddFields - fields: - - type: AddedFieldDefinition - path: - - date_created - value: '{% set formatted_datetime = format_datetime(record["date_created"], "%Y-%m-%dT%H:%M:%SZ") %}{{ formatted_datetime }}' - - type: AddFields - fields: - - type: AddedFieldDefinition - path: - - date_updated - value: '{% set formatted_datetime = format_datetime(record["date_updated"], "%Y-%m-%dT%H:%M:%SZ") %}{{ formatted_datetime }}' schema_loader: type: InlineSchemaLoader schema: From 79a777595e722b825cb2a3ee860717c3ffa25bdf Mon Sep 17 00:00:00 2001 From: "cristina.mariscal" Date: Mon, 8 Jul 2024 08:23:31 -0600 Subject: [PATCH 19/37] removing non required files --- .../connectors/source-twilio/migrations.py | 48 ------- .../source-twilio/unit_tests copy/conftest.py | 42 ------- .../unit_tests copy/integration/__init__.py | 0 .../unit_tests copy/integration/config.py | 16 --- .../integration/test_automations.py | 119 ------------------ .../resource/http/response/automations.json | 68 ---------- ...mponent_custom_email_activity_extractor.py | 66 ---------- .../test_config_datacenter_migration.py | 37 ------ .../test_configs/test_config_api_key.json | 4 - .../test_configs/test_config_oauth.json | 8 -- 10 files changed, 408 deletions(-) delete mode 100644 airbyte-integrations/connectors/source-twilio/migrations.py delete mode 100644 airbyte-integrations/connectors/source-twilio/unit_tests copy/conftest.py delete mode 100644 airbyte-integrations/connectors/source-twilio/unit_tests copy/integration/__init__.py delete mode 100644 airbyte-integrations/connectors/source-twilio/unit_tests copy/integration/config.py delete mode 100644 airbyte-integrations/connectors/source-twilio/unit_tests copy/integration/test_automations.py delete mode 100644 airbyte-integrations/connectors/source-twilio/unit_tests copy/resource/http/response/automations.json delete mode 100644 airbyte-integrations/connectors/source-twilio/unit_tests copy/test_component_custom_email_activity_extractor.py delete mode 100644 airbyte-integrations/connectors/source-twilio/unit_tests copy/test_config_datacenter_migration.py delete mode 100644 airbyte-integrations/connectors/source-twilio/unit_tests copy/test_configs/test_config_api_key.json delete mode 100644 airbyte-integrations/connectors/source-twilio/unit_tests copy/test_configs/test_config_oauth.json diff --git a/airbyte-integrations/connectors/source-twilio/migrations.py b/airbyte-integrations/connectors/source-twilio/migrations.py deleted file mode 100644 index 34d108acf23a..000000000000 --- a/airbyte-integrations/connectors/source-twilio/migrations.py +++ /dev/null @@ -1,48 +0,0 @@ -import logging -from typing import Any, List, Mapping - -import requests -from airbyte_cdk.config_observation import create_connector_config_control_message -from airbyte_cdk.entrypoint import AirbyteEntrypoint -from airbyte_cdk.sources import Source -from airbyte_cdk.utils import AirbyteTracedException -from airbyte_protocol.models import FailureType - -logger = logging.getLogger("airbyte_logger") - - -class MigrateDataCenter: - """ - This class stands for migrating the state at runtime, - Set stream state partition property in config based on credential type. - """ - - STREAMS = ["calls"] - - def migrate_state(cls, stream_name, state: Mapping[str, Any]) -> Mapping[str, Any]: - """ - This method migrates the state. - Args: - - stream_name (str): The stream name. - - state (Mapping[str, Any]): The state to migrate. - Returns: - - Mapping[str, Any]: The migrated state. - """ - if stream_name in cls.STREAMS: - if "partition" in state: - return state - return {"partition": {"data_center": "us1"}, "cursor": state} - return state - - @classmethod - def migrate(cls, args: List[str], source: Source) -> None: - """ - Orchestrates the migration process. - Args: - - args (List[str]): List of command-line arguments. - - source (Source): The data source. - """ - config_path = AirbyteEntrypoint(source).extract_config(args) - if config_path: - config = source.read_config(config_path) - cls.emit_control_message(cls.modify_and_save(config_path, source, config)) diff --git a/airbyte-integrations/connectors/source-twilio/unit_tests copy/conftest.py b/airbyte-integrations/connectors/source-twilio/unit_tests copy/conftest.py deleted file mode 100644 index c387004a5110..000000000000 --- a/airbyte-integrations/connectors/source-twilio/unit_tests copy/conftest.py +++ /dev/null @@ -1,42 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from pytest import fixture - - -@fixture(name="data_center") -def data_center_fixture(): - return "some_dc" - - -@fixture(name="config") -def config_fixture(data_center): - return {"apikey": f"API_KEY-{data_center}", "start_date": "2022-01-01T00:00:00.000Z"} - - -@fixture(name="access_token") -def access_token_fixture(): - return "some_access_token" - - -@fixture(name="oauth_config") -def oauth_config_fixture(access_token): - return { - "credentials": { - "auth_type": "oauth2.0", - "client_id": "111111111", - "client_secret": "secret_1111111111", - "access_token": access_token, - } - } - - -@fixture(name="apikey_config") -def apikey_config_fixture(data_center): - return {"credentials": {"auth_type": "apikey", "apikey": f"some_api_key-{data_center}"}} - - -@fixture(name="wrong_config") -def wrong_config_fixture(): - return {"credentials": {"auth_type": "not auth_type"}} diff --git a/airbyte-integrations/connectors/source-twilio/unit_tests copy/integration/__init__.py b/airbyte-integrations/connectors/source-twilio/unit_tests copy/integration/__init__.py deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/airbyte-integrations/connectors/source-twilio/unit_tests copy/integration/config.py b/airbyte-integrations/connectors/source-twilio/unit_tests copy/integration/config.py deleted file mode 100644 index 7e363fea9677..000000000000 --- a/airbyte-integrations/connectors/source-twilio/unit_tests copy/integration/config.py +++ /dev/null @@ -1,16 +0,0 @@ -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. - -from datetime import datetime -from typing import Any, Dict - - -class ConfigBuilder: - def __init__(self) -> None: - self._config: Dict[str, Any] = {"credentials": {"auth_type": "apikey", "apikey": "Mailchimp_token-us10"}, "data_center": "us10"} - - def with_start_date(self, start_datetime: datetime) -> "ConfigBuilder": - self._config["start_date"] = start_datetime.isoformat()[:-3] + "Z" - return self - - def build(self) -> Dict[str, Any]: - return self._config diff --git a/airbyte-integrations/connectors/source-twilio/unit_tests copy/integration/test_automations.py b/airbyte-integrations/connectors/source-twilio/unit_tests copy/integration/test_automations.py deleted file mode 100644 index 602562aa2db4..000000000000 --- a/airbyte-integrations/connectors/source-twilio/unit_tests copy/integration/test_automations.py +++ /dev/null @@ -1,119 +0,0 @@ -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -import datetime -import json -from unittest import TestCase - -import freezegun -from airbyte_cdk.models import SyncMode -from airbyte_cdk.test.catalog_builder import CatalogBuilder -from airbyte_cdk.test.entrypoint_wrapper import read -from airbyte_cdk.test.mock_http import HttpMocker, HttpRequest, HttpResponse -from airbyte_cdk.test.mock_http.response_builder import find_template -from airbyte_cdk.test.state_builder import StateBuilder -from source_mailchimp import SourceMailchimp - -from .config import ConfigBuilder - -_CONFIG = ConfigBuilder().with_start_date(datetime.datetime(2023, 1, 1, 0, 0, 0, 1000)).build() - - -def _create_catalog(sync_mode: SyncMode = SyncMode.full_refresh): - return CatalogBuilder().with_stream(name="automations", sync_mode=sync_mode).build() - - -@freezegun.freeze_time("2023-01-31T23:59:59.001000Z") -class AutomationsTest(TestCase): - def setUp(self) -> None: - """Base setup for all tests. Enter test mocker.""" - - self.r_mock = HttpMocker() - self.r_mock.__enter__() - - def teardown(self): - """Stops and resets HttpMocker instance.""" - self.r_mock.__exit__() - - def test_read_full_refresh_no_pagination(self): - """Ensure http integration and record extraction""" - self.r_mock.get( - HttpRequest( - url="https://us10.api.mailchimp.com/3.0/automations", - query_params={ - "sort_field": "create_time", - "sort_dir": "ASC", - "exclude_fields": "automations._links", - "count": 1000, - "since_create_time": "2022-12-31T23:59:59.001000Z", - "before_create_time": "2023-01-31T23:59:59.001000Z", - }, - ), - HttpResponse(json.dumps(find_template("automations", __file__)), 200), - ) - - source = SourceMailchimp() - actual_messages = read(source, config=_CONFIG, catalog=_create_catalog()) - - assert len(actual_messages.records) == 1 - - def test_full_refresh_with_pagination(self): - """Ensure pagination""" - self.r_mock.get( - HttpRequest( - url="https://us10.api.mailchimp.com/3.0/automations", - query_params={ - "sort_field": "create_time", - "sort_dir": "ASC", - "exclude_fields": "automations._links", - "count": 1000, - "since_create_time": "2022-12-31T23:59:59.001000Z", - "before_create_time": "2023-01-31T23:59:59.001000Z", - }, - ), - HttpResponse(json.dumps({"automations": find_template("automations", __file__)["automations"] * 1002}), 200), - ) - self.r_mock.get( - HttpRequest( - url="https://us10.api.mailchimp.com/3.0/automations", - query_params={ - "sort_field": "create_time", - "sort_dir": "ASC", - "exclude_fields": "automations._links", - "count": 1000, - "offset": 1002, - "since_create_time": "2022-12-31T23:59:59.001000Z", - "before_create_time": "2023-01-31T23:59:59.001000Z", - }, - ), - HttpResponse(json.dumps(find_template("automations", __file__)), 200), - ) - source = SourceMailchimp() - actual_messages = read(source, config=_CONFIG, catalog=_create_catalog()) - - assert len(actual_messages.records) == 1003 - - def test_when_read_incrementally_then_emit_state_message(self): - """Ensure incremental sync emits correct stream state message""" - - self.r_mock.get( - HttpRequest( - url="https://us10.api.mailchimp.com/3.0/automations", - query_params={ - "sort_field": "create_time", - "sort_dir": "ASC", - "exclude_fields": "automations._links", - "count": 1000, - "since_create_time": "2022-12-31T23:59:59.001000Z", - "before_create_time": "2023-01-31T23:59:59.001000Z", - }, - ), - HttpResponse(json.dumps(find_template("automations", __file__)), 200), - ) - - source = SourceMailchimp() - actual_messages = read( - source, - config=_CONFIG, - catalog=_create_catalog(sync_mode=SyncMode.incremental), - state=StateBuilder().with_stream_state("automations", {"create_time": "2220-11-23T05:42:11+00:00"}).build(), - ) - actual_messages.state_messages[0].state.stream.stream_state == {"create_time": "2220-11-23T05:42:11+00:00"} diff --git a/airbyte-integrations/connectors/source-twilio/unit_tests copy/resource/http/response/automations.json b/airbyte-integrations/connectors/source-twilio/unit_tests copy/resource/http/response/automations.json deleted file mode 100644 index 3be7ec7ee972..000000000000 --- a/airbyte-integrations/connectors/source-twilio/unit_tests copy/resource/http/response/automations.json +++ /dev/null @@ -1,68 +0,0 @@ -{ - "automations": [ - { - "id": "string", - "create_time": "2019-08-24T14:15:22Z", - "start_time": "2019-08-24T14:15:22Z", - "status": "save", - "emails_sent": 0, - "recipients": { - "list_id": "string", - "list_is_active": true, - "list_name": "string", - "segment_opts": { - "saved_segment_id": 0, - "match": "any", - "conditions": [null] - }, - "store_id": "1a2df69xxx" - }, - "settings": { - "title": "string", - "from_name": "string", - "reply_to": "string", - "use_conversation": true, - "to_name": "string", - "authenticate": true, - "auto_footer": true, - "inline_css": true - }, - "tracking": { - "opens": true, - "html_clicks": true, - "text_clicks": true, - "goal_tracking": true, - "ecomm360": true, - "google_analytics": "string", - "clicktale": "string", - "salesforce": { - "campaign": true, - "notes": true - }, - "capsule": { - "notes": true - } - }, - "trigger_settings": { - "workflow_type": "abandonedBrowse", - "workflow_title": "string", - "runtime": { - "days": ["sunday"], - "hours": { - "type": "send_asap" - } - }, - "workflow_emails_count": 0 - }, - "report_summary": { - "opens": 0, - "unique_opens": 0, - "open_rate": 0, - "clicks": 0, - "subscriber_clicks": 0, - "click_rate": 0 - } - } - ], - "total_items": 2 -} diff --git a/airbyte-integrations/connectors/source-twilio/unit_tests copy/test_component_custom_email_activity_extractor.py b/airbyte-integrations/connectors/source-twilio/unit_tests copy/test_component_custom_email_activity_extractor.py deleted file mode 100644 index 7f9ad40d0c7f..000000000000 --- a/airbyte-integrations/connectors/source-twilio/unit_tests copy/test_component_custom_email_activity_extractor.py +++ /dev/null @@ -1,66 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - - -import json - -import requests -from airbyte_cdk.sources.declarative.decoders import JsonDecoder -from source_mailchimp.components import MailChimpRecordExtractorEmailActivity - - -def test_email_activity_extractor(): - decoder = JsonDecoder(parameters={}) - field_path = ["emails"] - config = {"response_override": "stop_if_you_see_me"} - extractor = MailChimpRecordExtractorEmailActivity(field_path=field_path, decoder=decoder, config=config, parameters={}) - - body = { - "emails": [ - { - "campaign_id": "string", - "list_id": "string", - "list_is_active": True, - "email_id": "string", - "email_address": "AirbyteMailchimpUser@gmail.com", - "activity": [ - {"action": "close", "type": "string", "timestamp": "2019-08-24T14:15:22Z", "url": "string", "ip": "string"}, - {"action": "open", "type": "string", "timestamp": "2019-08-24T14:15:22Z", "url": "string", "ip": "string"}, - ], - } - ], - "campaign_id": "string", - "total_items": 0, - } - response = requests.Response() - response._content = json.dumps(body).encode("utf-8") - - expected_records = [ - { - "action": "close", - "campaign_id": "string", - "email_address": "AirbyteMailchimpUser@gmail.com", - "email_id": "string", - "ip": "string", - "list_id": "string", - "list_is_active": True, - "timestamp": "2019-08-24T14:15:22Z", - "type": "string", - "url": "string", - }, - { - "action": "open", - "campaign_id": "string", - "email_address": "AirbyteMailchimpUser@gmail.com", - "email_id": "string", - "ip": "string", - "list_id": "string", - "list_is_active": True, - "timestamp": "2019-08-24T14:15:22Z", - "type": "string", - "url": "string", - }, - ] - - assert list(extractor.extract_records(response=response)) == expected_records diff --git a/airbyte-integrations/connectors/source-twilio/unit_tests copy/test_config_datacenter_migration.py b/airbyte-integrations/connectors/source-twilio/unit_tests copy/test_config_datacenter_migration.py deleted file mode 100644 index 20fe8312352e..000000000000 --- a/airbyte-integrations/connectors/source-twilio/unit_tests copy/test_config_datacenter_migration.py +++ /dev/null @@ -1,37 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# -import json -import os -from typing import Any, Mapping - -import pytest -from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource -from source_mailchimp import SourceMailchimp -from source_mailchimp.config_migrations import MigrateDataCenter - -# BASE ARGS -SOURCE: YamlDeclarativeSource = SourceMailchimp() - - -# HELPERS -def load_config(config_path: str) -> Mapping[str, Any]: - with open(config_path, "r") as config: - return json.load(config) - - -@pytest.mark.parametrize( - "config_path", - [ - (f"{os.path.dirname(__file__)}/test_configs/test_config_api_key.json"), - (f"{os.path.dirname(__file__)}/test_configs/test_config_oauth.json"), - ], - ids=["test_requester_datacenter_with_api_key", "test_requester_datacenter_with_oauth_flow"], -) -def test_mailchimp_config_migration(config_path: str, requests_mock): - requests_mock.get("https://login.mailchimp.com/oauth2/metadata", json={"dc": "us10"}) - - migration_instance = MigrateDataCenter - migration_instance.migrate(["check", "--config", config_path], SOURCE) - test_migrated_config = load_config(config_path) - assert test_migrated_config.get("data_center") == "us10" diff --git a/airbyte-integrations/connectors/source-twilio/unit_tests copy/test_configs/test_config_api_key.json b/airbyte-integrations/connectors/source-twilio/unit_tests copy/test_configs/test_config_api_key.json deleted file mode 100644 index 20866d9bfcf7..000000000000 --- a/airbyte-integrations/connectors/source-twilio/unit_tests copy/test_configs/test_config_api_key.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "credentials": { "auth_type": "apikey", "apikey": "random_api_key-us10" }, - "data_center": "us10" -} diff --git a/airbyte-integrations/connectors/source-twilio/unit_tests copy/test_configs/test_config_oauth.json b/airbyte-integrations/connectors/source-twilio/unit_tests copy/test_configs/test_config_oauth.json deleted file mode 100644 index ef7ef97ee241..000000000000 --- a/airbyte-integrations/connectors/source-twilio/unit_tests copy/test_configs/test_config_oauth.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "credentials": { - "auth_type": "oauth2.0", - "client_id": "client_id", - "client_secret": "client_secret", - "access_token": "access_token" - } -} From b921703cc1a49f3c9bcb6194cca5968803885d40 Mon Sep 17 00:00:00 2001 From: "cristina.mariscal" Date: Tue, 9 Jul 2024 02:20:40 -0600 Subject: [PATCH 20/37] updating start_datetime for messages --- .../connectors/source-twilio/source_twilio/manifest.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/airbyte-integrations/connectors/source-twilio/source_twilio/manifest.yaml b/airbyte-integrations/connectors/source-twilio/source_twilio/manifest.yaml index 454cb378d0cb..24f81923d800 100644 --- a/airbyte-integrations/connectors/source-twilio/source_twilio/manifest.yaml +++ b/airbyte-integrations/connectors/source-twilio/source_twilio/manifest.yaml @@ -1437,7 +1437,7 @@ definitions: datetime_format: "%Y-%m-%d" start_datetime: type: MinMaxDatetime - datetime: '{{ config["start_date"] }}' + datetime: "{{ day_delta(-400, format='%Y-%m-%dT%H:%M:%SZ') }}" datetime_format: "%Y-%m-%dT%H:%M:%SZ" start_time_option: type: RequestOption @@ -1549,7 +1549,7 @@ definitions: datetime_format: "%Y-%m-%d" start_datetime: type: MinMaxDatetime - datetime: '{{ config["start_date"] }}' + datetime: "{{ day_delta(-400, format='%Y-%m-%dT%H:%M:%SZ') }}" datetime_format: "%Y-%m-%dT%H:%M:%SZ" start_time_option: type: RequestOption From ece247b5ff7674844af6922d430c5cc2ad226175 Mon Sep 17 00:00:00 2001 From: "cristina.mariscal" Date: Tue, 9 Jul 2024 02:38:17 -0600 Subject: [PATCH 21/37] fix format --- .../connectors/source-twilio/source_twilio/manifest.yaml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/airbyte-integrations/connectors/source-twilio/source_twilio/manifest.yaml b/airbyte-integrations/connectors/source-twilio/source_twilio/manifest.yaml index 24f81923d800..21c0fa6e7d2e 100644 --- a/airbyte-integrations/connectors/source-twilio/source_twilio/manifest.yaml +++ b/airbyte-integrations/connectors/source-twilio/source_twilio/manifest.yaml @@ -1434,7 +1434,7 @@ definitions: cursor_field: date_sent cursor_datetime_formats: - "%Y-%m-%dT%H:%M:%SZ" - datetime_format: "%Y-%m-%d" + datetime_format: "%Y-%m-%dT%H:%M:%SZ" start_datetime: type: MinMaxDatetime datetime: "{{ day_delta(-400, format='%Y-%m-%dT%H:%M:%SZ') }}" @@ -1546,7 +1546,7 @@ definitions: cursor_field: date_created cursor_datetime_formats: - "%Y-%m-%dT%H:%M:%SZ" - datetime_format: "%Y-%m-%d" + datetime_format: "%Y-%m-%dT%H:%M:%SZ" start_datetime: type: MinMaxDatetime datetime: "{{ day_delta(-400, format='%Y-%m-%dT%H:%M:%SZ') }}" @@ -5460,4 +5460,3 @@ schemas: type: - "null" - string - From 8c4b6a51101be8249cbda58485504e9cfca81772 Mon Sep 17 00:00:00 2001 From: "cristina.mariscal" Date: Tue, 9 Jul 2024 03:10:04 -0600 Subject: [PATCH 22/37] updating breaking changes --- .../connectors/source-twilio/metadata.yaml | 5 ++ .../connectors/source-twilio/pyproject.toml | 2 +- .../source-twilio/source_twilio/manifest.yaml | 22 ++++++- .../integrations/sources/twilio-migrations.md | 59 +++++++++++++++++++ docs/integrations/sources/twilio.md | 2 +- 5 files changed, 86 insertions(+), 4 deletions(-) create mode 100644 docs/integrations/sources/twilio-migrations.md diff --git a/airbyte-integrations/connectors/source-twilio/metadata.yaml b/airbyte-integrations/connectors/source-twilio/metadata.yaml index 8ff25f529f8b..8a5695938160 100644 --- a/airbyte-integrations/connectors/source-twilio/metadata.yaml +++ b/airbyte-integrations/connectors/source-twilio/metadata.yaml @@ -49,4 +49,9 @@ data: secretStore: type: GSM alias: airbyte-connector-testing-secret-store + releases: + breakingChanges: + 1.0.0: + message: This release changes the state format of existing connections. + upgradeDeadline: "2024-12-01" metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-twilio/pyproject.toml b/airbyte-integrations/connectors/source-twilio/pyproject.toml index 6531bf26413e..1fec3c6f8979 100644 --- a/airbyte-integrations/connectors/source-twilio/pyproject.toml +++ b/airbyte-integrations/connectors/source-twilio/pyproject.toml @@ -3,7 +3,7 @@ requires = [ "poetry-core>=1.0.0",] build-backend = "poetry.core.masonry.api" [tool.poetry] -version = "0.11.8" +version = "1.0.0" name = "source-twilio" description = "Source implementation for Twilio." authors = [ "Airbyte ",] diff --git a/airbyte-integrations/connectors/source-twilio/source_twilio/manifest.yaml b/airbyte-integrations/connectors/source-twilio/source_twilio/manifest.yaml index 21c0fa6e7d2e..89050f22530a 100644 --- a/airbyte-integrations/connectors/source-twilio/source_twilio/manifest.yaml +++ b/airbyte-integrations/connectors/source-twilio/source_twilio/manifest.yaml @@ -570,6 +570,12 @@ definitions: path: - date_generated value: '{% set formatted_datetime = format_datetime(record["date_generated"], "%Y-%m-%dT%H:%M:%SZ", "%a, %d %b %Y %H:%M:%S %z") %}{{ formatted_datetime }}' + - type: AddFields + fields: + - type: AddedFieldDefinition + path: + - error_code + value: '{% set error_code = record["error_code"] | string %}{{ error_code }}' schema_loader: type: InlineSchemaLoader schema: @@ -1164,6 +1170,12 @@ definitions: path: - price value: '{% set price = record["price"] | float %}{{ price }}' + - type: AddFields + fields: + - type: AddedFieldDefinition + path: + - error_code + value: '{% set error_code = record["error_code"] | string %}{{ error_code }}' schema_loader: type: InlineSchemaLoader schema: @@ -1437,7 +1449,7 @@ definitions: datetime_format: "%Y-%m-%dT%H:%M:%SZ" start_datetime: type: MinMaxDatetime - datetime: "{{ day_delta(-400, format='%Y-%m-%dT%H:%M:%SZ') }}" + datetime: "{{ format_datetime(max(timestamp(day_delta(-400, format='%Y-%m-%dT%H:%M:%SZ')), timestamp(config['start_time'])), '%Y-%m-%dT%H:%M:%SZ') }}" datetime_format: "%Y-%m-%dT%H:%M:%SZ" start_time_option: type: RequestOption @@ -1488,6 +1500,12 @@ definitions: path: - price value: '{% set price = record["price"] | float %}{{ price }}' + - type: AddFields + fields: + - type: AddedFieldDefinition + path: + - error_code + value: '{% set error_code = record["error_code"] | string %}{{ error_code }}' schema_loader: type: InlineSchemaLoader schema: @@ -1549,7 +1567,7 @@ definitions: datetime_format: "%Y-%m-%dT%H:%M:%SZ" start_datetime: type: MinMaxDatetime - datetime: "{{ day_delta(-400, format='%Y-%m-%dT%H:%M:%SZ') }}" + datetime: "{{ format_datetime(max(timestamp(day_delta(-400, format='%Y-%m-%dT%H:%M:%SZ')), timestamp(config['start_time'])), '%Y-%m-%dT%H:%M:%SZ') }}" datetime_format: "%Y-%m-%dT%H:%M:%SZ" start_time_option: type: RequestOption diff --git a/docs/integrations/sources/twilio-migrations.md b/docs/integrations/sources/twilio-migrations.md new file mode 100644 index 000000000000..da4bd22af3a5 --- /dev/null +++ b/docs/integrations/sources/twilio-migrations.md @@ -0,0 +1,59 @@ +# Twilio Migration Guide + +## Upgrading to 1.0.0 + +### Migration Description for Connector Update to Low Code + +As part of the migration to a low code, we are updating the connector state format. The previous state format will no longer be supported, and all streams must be re-executed to update the state to a valid format. + +- For each affected connector, re-run all streams to ensure the state is updated to the new format. +- Verify that the data is correctly processed and the state is properly updated. + +#### Previous State Format + +```json +{ + "type": "STREAM", + "stream": { + "stream_state": { + "end_time": "2022-06-11T00:00:00Z" + }, + "stream_descriptor": { + "name": "calls" + } + } +} +``` + +#### New State Format + +```json +{ + "type": "STATE", + "state": { + "type": "STREAM", + "stream": { + "stream_descriptor": { + "name": "calls", + "namespace": null + }, + "stream_state": { + "states": [ + { + "partition": { + "account_sid": "", + "parent_slice": {} + }, + "cursor": { + "end_time": "Tue, 25 Jun 2024 20:19:28 +0000" + } + } + ] + } + }, + "sourceStats": { + "recordCount": 0.0 + } + } +} +``` diff --git a/docs/integrations/sources/twilio.md b/docs/integrations/sources/twilio.md index 8a9785c16bd3..677cead0a186 100644 --- a/docs/integrations/sources/twilio.md +++ b/docs/integrations/sources/twilio.md @@ -100,7 +100,7 @@ For more information, see [the Twilio docs for rate limitations](https://support | Version | Date | Pull Request | Subject | | :------ | :--------- | :------------------------------------------------------- | :------------------------------------------------------------------------------------------------------ | -| 0.11.8 | 2024-06-24 | [40248](https://github.com/airbytehq/airbyte/pull/40248) | Twilio accounts stream migration to low code | +| 1.0.0 | 2024-06-24 | [40248](https://github.com/airbytehq/airbyte/pull/40248) | Twilio migration to low code | | 0.11.7 | 2024-06-26 | [40527](https://github.com/airbytehq/airbyte/pull/40527) | Update dependencies | | 0.11.6 | 2024-06-22 | [40030](https://github.com/airbytehq/airbyte/pull/40030) | Update dependencies | | 0.11.5 | 2024-06-06 | [39252](https://github.com/airbytehq/airbyte/pull/39252) | [autopull] Upgrade base image to v1.2.2 | From 5e8f0356310935a76a489843ba706ffc5fd44246 Mon Sep 17 00:00:00 2001 From: "cristina.mariscal" Date: Tue, 9 Jul 2024 03:26:57 -0600 Subject: [PATCH 23/37] fix key --- .../connectors/source-twilio/source_twilio/manifest.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/airbyte-integrations/connectors/source-twilio/source_twilio/manifest.yaml b/airbyte-integrations/connectors/source-twilio/source_twilio/manifest.yaml index 89050f22530a..b5a3f623b1e0 100644 --- a/airbyte-integrations/connectors/source-twilio/source_twilio/manifest.yaml +++ b/airbyte-integrations/connectors/source-twilio/source_twilio/manifest.yaml @@ -1449,7 +1449,7 @@ definitions: datetime_format: "%Y-%m-%dT%H:%M:%SZ" start_datetime: type: MinMaxDatetime - datetime: "{{ format_datetime(max(timestamp(day_delta(-400, format='%Y-%m-%dT%H:%M:%SZ')), timestamp(config['start_time'])), '%Y-%m-%dT%H:%M:%SZ') }}" + datetime: "{{ format_datetime(max(timestamp(day_delta(-400, format='%Y-%m-%dT%H:%M:%SZ')), timestamp(config['start_date'])), '%Y-%m-%dT%H:%M:%SZ') }}" datetime_format: "%Y-%m-%dT%H:%M:%SZ" start_time_option: type: RequestOption @@ -1567,7 +1567,7 @@ definitions: datetime_format: "%Y-%m-%dT%H:%M:%SZ" start_datetime: type: MinMaxDatetime - datetime: "{{ format_datetime(max(timestamp(day_delta(-400, format='%Y-%m-%dT%H:%M:%SZ')), timestamp(config['start_time'])), '%Y-%m-%dT%H:%M:%SZ') }}" + datetime: "{{ format_datetime(max(timestamp(day_delta(-400, format='%Y-%m-%dT%H:%M:%SZ')), timestamp(config['start_date'])), '%Y-%m-%dT%H:%M:%SZ') }}" datetime_format: "%Y-%m-%dT%H:%M:%SZ" start_time_option: type: RequestOption From df598c0f2425346829da51e252c02d27f86f7614 Mon Sep 17 00:00:00 2001 From: "cristina.mariscal" Date: Tue, 9 Jul 2024 06:51:53 -0600 Subject: [PATCH 24/37] retention_window fixes --- airbyte-integrations/connectors/source-twilio/metadata.yaml | 2 +- .../connectors/source-twilio/source_twilio/manifest.yaml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/airbyte-integrations/connectors/source-twilio/metadata.yaml b/airbyte-integrations/connectors/source-twilio/metadata.yaml index 8a5695938160..606e09640a91 100644 --- a/airbyte-integrations/connectors/source-twilio/metadata.yaml +++ b/airbyte-integrations/connectors/source-twilio/metadata.yaml @@ -13,7 +13,7 @@ data: connectorSubtype: api connectorType: source definitionId: b9dc6155-672e-42ea-b10d-9f1f1fb95ab1 - dockerImageTag: 0.11.8 + dockerImageTag: 1.0.0 dockerRepository: airbyte/source-twilio documentationUrl: https://docs.airbyte.com/integrations/sources/twilio githubIssueLabel: source-twilio diff --git a/airbyte-integrations/connectors/source-twilio/source_twilio/manifest.yaml b/airbyte-integrations/connectors/source-twilio/source_twilio/manifest.yaml index b5a3f623b1e0..5aa4127bb257 100644 --- a/airbyte-integrations/connectors/source-twilio/source_twilio/manifest.yaml +++ b/airbyte-integrations/connectors/source-twilio/source_twilio/manifest.yaml @@ -1449,7 +1449,7 @@ definitions: datetime_format: "%Y-%m-%dT%H:%M:%SZ" start_datetime: type: MinMaxDatetime - datetime: "{{ format_datetime(max(timestamp(day_delta(-400, format='%Y-%m-%dT%H:%M:%SZ')), timestamp(config['start_date'])), '%Y-%m-%dT%H:%M:%SZ') }}" + datetime: "{% set retention_window = day_delta(-400, format='%Y-%m-%dT%H:%M:%SZ') %}{{ retention_window if timestamp(retention_window) > timestamp(config['start_date']) else config['start_date'] }}" datetime_format: "%Y-%m-%dT%H:%M:%SZ" start_time_option: type: RequestOption @@ -1567,7 +1567,7 @@ definitions: datetime_format: "%Y-%m-%dT%H:%M:%SZ" start_datetime: type: MinMaxDatetime - datetime: "{{ format_datetime(max(timestamp(day_delta(-400, format='%Y-%m-%dT%H:%M:%SZ')), timestamp(config['start_date'])), '%Y-%m-%dT%H:%M:%SZ') }}" + datetime: "{% set retention_window = day_delta(-400, format='%Y-%m-%dT%H:%M:%SZ') %}{{ retention_window if timestamp(retention_window) > timestamp(config['start_date']) else config['start_date'] }}" datetime_format: "%Y-%m-%dT%H:%M:%SZ" start_time_option: type: RequestOption From d7c84952f4148e9b2c6ed4c257d941a71597f72b Mon Sep 17 00:00:00 2001 From: "cristina.mariscal" Date: Tue, 9 Jul 2024 07:10:03 -0600 Subject: [PATCH 25/37] removing error code transformation --- .../connectors/source-twilio/source_twilio/manifest.yaml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/airbyte-integrations/connectors/source-twilio/source_twilio/manifest.yaml b/airbyte-integrations/connectors/source-twilio/source_twilio/manifest.yaml index 5aa4127bb257..f86e40f94581 100644 --- a/airbyte-integrations/connectors/source-twilio/source_twilio/manifest.yaml +++ b/airbyte-integrations/connectors/source-twilio/source_twilio/manifest.yaml @@ -1170,12 +1170,6 @@ definitions: path: - price value: '{% set price = record["price"] | float %}{{ price }}' - - type: AddFields - fields: - - type: AddedFieldDefinition - path: - - error_code - value: '{% set error_code = record["error_code"] | string %}{{ error_code }}' schema_loader: type: InlineSchemaLoader schema: From d569a5e0dec1bbc93b1b11c8429de1d8e221d32f Mon Sep 17 00:00:00 2001 From: "cristina.mariscal" Date: Tue, 9 Jul 2024 07:53:12 -0600 Subject: [PATCH 26/37] media fixes --- .../source-twilio/source_twilio/manifest.yaml | 31 +++++++++---------- 1 file changed, 15 insertions(+), 16 deletions(-) diff --git a/airbyte-integrations/connectors/source-twilio/source_twilio/manifest.yaml b/airbyte-integrations/connectors/source-twilio/source_twilio/manifest.yaml index f86e40f94581..595625b7d21f 100644 --- a/airbyte-integrations/connectors/source-twilio/source_twilio/manifest.yaml +++ b/airbyte-integrations/connectors/source-twilio/source_twilio/manifest.yaml @@ -1030,27 +1030,13 @@ definitions: $ref: "#/definitions/base_requester" path: >- https://api.twilio.com/2010-04-01/Accounts/{{ - stream_partition.account_sid }}/OutgoingCallerIds + stream_partition.account_sid }}/OutgoingCallerIds.json http_method: GET record_selector: type: RecordSelector extractor: type: DpathExtractor field_path: [] - paginator: - type: DefaultPaginator - page_token_option: - type: RequestOption - inject_into: request_parameter - field_name: Page - page_size_option: - type: RequestOption - field_name: PageSize - inject_into: request_parameter - pagination_strategy: - type: PageIncrement - page_size: 1000 - start_from_page: 0 partition_router: type: SubstreamPartitionRouter parent_stream_configs: @@ -1523,7 +1509,7 @@ definitions: extractor: type: DpathExtractor field_path: - - media + - media_list paginator: type: DefaultPaginator page_token_option: @@ -1575,6 +1561,19 @@ definitions: type: MinMaxDatetime datetime: "{{ now_utc().strftime('%Y-%m-%dT%H:%M:%SZ') }}" datetime_format: "%Y-%m-%dT%H:%M:%SZ" + transformations: + - type: AddFields + fields: + - type: AddedFieldDefinition + path: + - date_created + value: '{% set formatted_datetime = format_datetime(record["date_created"], "%Y-%m-%dT%H:%M:%SZ", "%a, %d %b %Y %H:%M:%S %z") %}{{ formatted_datetime }}' + - type: AddFields + fields: + - type: AddedFieldDefinition + path: + - date_updated + value: '{% set formatted_datetime = format_datetime(record["date_updated"], "%Y-%m-%dT%H:%M:%SZ", "%a, %d %b %Y %H:%M:%S %z") %}{{ formatted_datetime }}' schema_loader: type: InlineSchemaLoader schema: From 67c9442b0b8d064f769c26cb35fb383332d3171b Mon Sep 17 00:00:00 2001 From: "cristina.mariscal" Date: Tue, 9 Jul 2024 08:22:51 -0600 Subject: [PATCH 27/37] removing tranformations from outgoing callers --- .../source-twilio/source_twilio/manifest.yaml | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/airbyte-integrations/connectors/source-twilio/source_twilio/manifest.yaml b/airbyte-integrations/connectors/source-twilio/source_twilio/manifest.yaml index 595625b7d21f..0d6ae50b5126 100644 --- a/airbyte-integrations/connectors/source-twilio/source_twilio/manifest.yaml +++ b/airbyte-integrations/connectors/source-twilio/source_twilio/manifest.yaml @@ -1045,19 +1045,6 @@ definitions: partition_field: account_sid stream: $ref: "#/definitions/streams/Accounts" - transformations: - - type: AddFields - fields: - - type: AddedFieldDefinition - path: - - date_created - value: '{% set formatted_datetime = format_datetime(record["date_created"], "%Y-%m-%dT%H:%M:%SZ", "%a, %d %b %Y %H:%M:%S %z") %}{{ formatted_datetime }}' - - type: AddFields - fields: - - type: AddedFieldDefinition - path: - - date_updated - value: '{% set formatted_datetime = format_datetime(record["date_updated"], "%Y-%m-%dT%H:%M:%SZ", "%a, %d %b %Y %H:%M:%S %z") %}{{ formatted_datetime }}' schema_loader: type: InlineSchemaLoader schema: From 96a26257db9d751d14c4d575c5be7ce35801650e Mon Sep 17 00:00:00 2001 From: "cristina.mariscal" Date: Tue, 9 Jul 2024 08:50:31 -0600 Subject: [PATCH 28/37] fix format --- airbyte-integrations/connectors/source-twilio/metadata.yaml | 2 +- .../connectors/source-twilio/source_twilio/manifest.yaml | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/airbyte-integrations/connectors/source-twilio/metadata.yaml b/airbyte-integrations/connectors/source-twilio/metadata.yaml index 606e09640a91..7d1f5f4a529d 100644 --- a/airbyte-integrations/connectors/source-twilio/metadata.yaml +++ b/airbyte-integrations/connectors/source-twilio/metadata.yaml @@ -52,6 +52,6 @@ data: releases: breakingChanges: 1.0.0: - message: This release changes the state format of existing connections. + message: This release changes the state format of existing connections. upgradeDeadline: "2024-12-01" metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-twilio/source_twilio/manifest.yaml b/airbyte-integrations/connectors/source-twilio/source_twilio/manifest.yaml index 0d6ae50b5126..5189ffef077d 100644 --- a/airbyte-integrations/connectors/source-twilio/source_twilio/manifest.yaml +++ b/airbyte-integrations/connectors/source-twilio/source_twilio/manifest.yaml @@ -1022,8 +1022,6 @@ definitions: Outgoing Caller Ids: type: DeclarativeStream name: outgoing_caller_ids - primary_key: - - sid retriever: type: SimpleRetriever requester: From babcc5290c67cefe9c7a376031ee2bd6160ba462 Mon Sep 17 00:00:00 2001 From: "cristina.mariscal" Date: Tue, 9 Jul 2024 09:10:38 -0600 Subject: [PATCH 29/37] fix outgoing callers --- .../connectors/source-twilio/source_twilio/manifest.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-twilio/source_twilio/manifest.yaml b/airbyte-integrations/connectors/source-twilio/source_twilio/manifest.yaml index 5189ffef077d..fd733a3e5e57 100644 --- a/airbyte-integrations/connectors/source-twilio/source_twilio/manifest.yaml +++ b/airbyte-integrations/connectors/source-twilio/source_twilio/manifest.yaml @@ -1034,7 +1034,8 @@ definitions: type: RecordSelector extractor: type: DpathExtractor - field_path: [] + field_path: + - outgoing_caller_ids partition_router: type: SubstreamPartitionRouter parent_stream_configs: From c7ef0dc8bb32595d7c6a30924b18c717f60e4fb3 Mon Sep 17 00:00:00 2001 From: "cristina.mariscal" Date: Tue, 9 Jul 2024 09:27:13 -0600 Subject: [PATCH 30/37] add back tranformations from outgoing callers --- .../source-twilio/source_twilio/manifest.yaml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/airbyte-integrations/connectors/source-twilio/source_twilio/manifest.yaml b/airbyte-integrations/connectors/source-twilio/source_twilio/manifest.yaml index fd733a3e5e57..d3622718f6c7 100644 --- a/airbyte-integrations/connectors/source-twilio/source_twilio/manifest.yaml +++ b/airbyte-integrations/connectors/source-twilio/source_twilio/manifest.yaml @@ -1022,6 +1022,8 @@ definitions: Outgoing Caller Ids: type: DeclarativeStream name: outgoing_caller_ids + primary_key: + - sid retriever: type: SimpleRetriever requester: @@ -1044,6 +1046,19 @@ definitions: partition_field: account_sid stream: $ref: "#/definitions/streams/Accounts" + transformations: + - type: AddFields + fields: + - type: AddedFieldDefinition + path: + - date_created + value: '{% set formatted_datetime = format_datetime(record["date_created"], "%Y-%m-%dT%H:%M:%SZ", "%a, %d %b %Y %H:%M:%S %z") %}{{ formatted_datetime }}' + - type: AddFields + fields: + - type: AddedFieldDefinition + path: + - date_updated + value: '{% set formatted_datetime = format_datetime(record["date_updated"], "%Y-%m-%dT%H:%M:%SZ", "%a, %d %b %Y %H:%M:%S %z") %}{{ formatted_datetime }}' schema_loader: type: InlineSchemaLoader schema: From 7f54d2d38f384e3a50ccfd150676f313fb7d065f Mon Sep 17 00:00:00 2001 From: "cristina.mariscal" Date: Tue, 9 Jul 2024 15:17:50 -0600 Subject: [PATCH 31/37] fixes at alert --- .../source-twilio/source_twilio/manifest.yaml | 39 ++++--------------- 1 file changed, 8 insertions(+), 31 deletions(-) diff --git a/airbyte-integrations/connectors/source-twilio/source_twilio/manifest.yaml b/airbyte-integrations/connectors/source-twilio/source_twilio/manifest.yaml index d3622718f6c7..c94c3849c3d1 100644 --- a/airbyte-integrations/connectors/source-twilio/source_twilio/manifest.yaml +++ b/airbyte-integrations/connectors/source-twilio/source_twilio/manifest.yaml @@ -462,6 +462,14 @@ definitions: path: >- https://studio.twilio.com/v1/Flows/{{stream_partition.flow_sid}}/Executions/{{stream_partition.execution_sid}}/Steps http_method: GET + error_handler: + type: CompositeErrorHandler + error_handlers: + - type: DefaultErrorHandler + response_filters: + - http_codes: [404] + action: IGNORE + error_message: "The requested resource was not found." record_selector: type: RecordSelector extractor: @@ -552,24 +560,6 @@ definitions: datetime: "{{ now_utc().strftime('%Y-%m-%dT%H:%M:%SZ') }}" datetime_format: "%Y-%m-%dT%H:%M:%SZ" transformations: - - type: AddFields - fields: - - type: AddedFieldDefinition - path: - - date_created - value: '{% set formatted_datetime = format_datetime(record["date_created"], "%Y-%m-%dT%H:%M:%SZ", "%a, %d %b %Y %H:%M:%S %z") %}{{ formatted_datetime }}' - - type: AddFields - fields: - - type: AddedFieldDefinition - path: - - date_updated - value: '{% set formatted_datetime = format_datetime(record["date_updated"], "%Y-%m-%dT%H:%M:%SZ", "%a, %d %b %Y %H:%M:%S %z") %}{{ formatted_datetime }}' - - type: AddFields - fields: - - type: AddedFieldDefinition - path: - - date_generated - value: '{% set formatted_datetime = format_datetime(record["date_generated"], "%Y-%m-%dT%H:%M:%SZ", "%a, %d %b %Y %H:%M:%S %z") %}{{ formatted_datetime }}' - type: AddFields fields: - type: AddedFieldDefinition @@ -1002,19 +992,6 @@ definitions: partition_field: flow_sid stream: $ref: "#/definitions/streams/Flows" - transformations: - - type: AddFields - fields: - - type: AddedFieldDefinition - path: - - date_created - value: '{% set formatted_datetime = format_datetime(record["date_created"], "%Y-%m-%dT%H:%M:%SZ", "%a, %d %b %Y %H:%M:%S %z") %}{{ formatted_datetime }}' - - type: AddFields - fields: - - type: AddedFieldDefinition - path: - - date_updated - value: '{% set formatted_datetime = format_datetime(record["date_updated"], "%Y-%m-%dT%H:%M:%SZ", "%a, %d %b %Y %H:%M:%S %z") %}{{ formatted_datetime }}' schema_loader: type: InlineSchemaLoader schema: From a3bb5d8f890fd1728bef56c0d51436ed71d40ecc Mon Sep 17 00:00:00 2001 From: "cristina.mariscal" Date: Wed, 10 Jul 2024 07:06:31 -0600 Subject: [PATCH 32/37] adding steps --- .../source-twilio/source_twilio/manifest.yaml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/airbyte-integrations/connectors/source-twilio/source_twilio/manifest.yaml b/airbyte-integrations/connectors/source-twilio/source_twilio/manifest.yaml index c94c3849c3d1..b32ed05c3820 100644 --- a/airbyte-integrations/connectors/source-twilio/source_twilio/manifest.yaml +++ b/airbyte-integrations/connectors/source-twilio/source_twilio/manifest.yaml @@ -559,6 +559,8 @@ definitions: type: MinMaxDatetime datetime: "{{ now_utc().strftime('%Y-%m-%dT%H:%M:%SZ') }}" datetime_format: "%Y-%m-%dT%H:%M:%SZ" + cursor_granularity: PT1S + step: P1Y transformations: - type: AddFields fields: @@ -757,6 +759,8 @@ definitions: type: MinMaxDatetime datetime: "{{ now_utc().strftime('%Y-%m-%dT%H:%M:%SZ') }}" datetime_format: "%Y-%m-%dT%H:%M:%SZ" + cursor_granularity: P1D + step: P1Y transformations: - type: AddFields fields: @@ -867,6 +871,8 @@ definitions: type: MinMaxDatetime datetime: "{{ now_utc().strftime('%Y-%m-%dT%H:%M:%SZ') }}" datetime_format: "%Y-%m-%dT%H:%M:%SZ" + cursor_granularity: PT1S + step: P1Y transformations: - type: AddFields fields: @@ -1103,6 +1109,8 @@ definitions: type: MinMaxDatetime datetime: "{{ now_utc().strftime('%Y-%m-%dT%H:%M:%SZ') }}" datetime_format: "%Y-%m-%dT%H:%M:%SZ" + cursor_granularity: PT1S + step: P1Y transformations: - type: AddFields fields: @@ -1421,6 +1429,8 @@ definitions: type: MinMaxDatetime datetime: "{{ now_utc().strftime('%Y-%m-%dT%H:%M:%SZ') }}" datetime_format: "%Y-%m-%dT%H:%M:%SZ" + cursor_granularity: PT1S + step: P1D transformations: - type: AddFields fields: @@ -1539,6 +1549,8 @@ definitions: type: MinMaxDatetime datetime: "{{ now_utc().strftime('%Y-%m-%dT%H:%M:%SZ') }}" datetime_format: "%Y-%m-%dT%H:%M:%SZ" + cursor_granularity: PT1S + step: P1Y transformations: - type: AddFields fields: @@ -1620,6 +1632,8 @@ definitions: type: MinMaxDatetime datetime: "{{ now_utc().strftime('%Y-%m-%dT%H:%M:%SZ') }}" datetime_format: "%Y-%m-%dT%H:%M:%SZ" + cursor_granularity: P1D + step: P1Y schema_loader: type: InlineSchemaLoader schema: From 5ce72d81c7bdeee194ca2043b4e693e20d55a05b Mon Sep 17 00:00:00 2001 From: "cristina.mariscal" Date: Tue, 16 Jul 2024 13:17:22 -0600 Subject: [PATCH 33/37] substreams --- .../connectors/source-twilio/poetry.lock | 20 +++++++++---------- .../connectors/source-twilio/pyproject.toml | 2 +- .../source-twilio/source_twilio/manifest.yaml | 8 ++------ 3 files changed, 13 insertions(+), 17 deletions(-) diff --git a/airbyte-integrations/connectors/source-twilio/poetry.lock b/airbyte-integrations/connectors/source-twilio/poetry.lock index 1e85e2e202ca..f3f554044796 100644 --- a/airbyte-integrations/connectors/source-twilio/poetry.lock +++ b/airbyte-integrations/connectors/source-twilio/poetry.lock @@ -2,13 +2,13 @@ [[package]] name = "airbyte-cdk" -version = "2.4.0" +version = "3.5.3" description = "A framework for writing Airbyte Connectors." optional = false python-versions = "<4.0,>=3.9" files = [ - {file = "airbyte_cdk-2.4.0-py3-none-any.whl", hash = "sha256:39470b2fe97f28959fcecb839d3080a8aba4a64a29dddf54a39f11f93f9f9ef7"}, - {file = "airbyte_cdk-2.4.0.tar.gz", hash = "sha256:f973d2e17a6dd0416c4395139e2761a10b38aafa61e097eaacffebbe6164ef45"}, + {file = "airbyte_cdk-3.5.3-py3-none-any.whl", hash = "sha256:729e9d6f9373f6011a0904b3583c0c9f354c5b48ba006051852f7f8255ae3c31"}, + {file = "airbyte_cdk-3.5.3.tar.gz", hash = "sha256:5213b04d470093f43cc780b72b6bcc8a2bacb12ec5162a2704e50c036b5086a8"}, ] [package.dependencies] @@ -118,13 +118,13 @@ files = [ [[package]] name = "cachetools" -version = "5.3.3" +version = "5.4.0" description = "Extensible memoizing collections and decorators" optional = false python-versions = ">=3.7" files = [ - {file = "cachetools-5.3.3-py3-none-any.whl", hash = "sha256:0abad1021d3f8325b2fc1d2e9c8b9c9d57b04c3932657a72465447332c24d945"}, - {file = "cachetools-5.3.3.tar.gz", hash = "sha256:ba29e2dfa0b8b556606f097407ed1aa62080ee108ab0dc5ec9d6a723a007d105"}, + {file = "cachetools-5.4.0-py3-none-any.whl", hash = "sha256:3ae3b49a3d5e28a77a0be2b37dbcb89005058959cb2323858c2657c4a8cab474"}, + {file = "cachetools-5.4.0.tar.gz", hash = "sha256:b8adc2e7c07f105ced7bc56dbb6dfbe7c4a00acce20e2227b3f355be89bc6827"}, ] [[package]] @@ -591,13 +591,13 @@ extended-testing = ["jinja2 (>=3,<4)"] [[package]] name = "langsmith" -version = "0.1.84" +version = "0.1.86" description = "Client library to connect to the LangSmith LLM Tracing and Evaluation Platform." optional = false python-versions = "<4.0,>=3.8.1" files = [ - {file = "langsmith-0.1.84-py3-none-any.whl", hash = "sha256:01f3c6390dba26c583bac8dd0e551ce3d0509c7f55cad714db0b5c8d36e4c7ff"}, - {file = "langsmith-0.1.84.tar.gz", hash = "sha256:5220c0439838b9a5bd320fd3686be505c5083dcee22d2452006c23891153bea1"}, + {file = "langsmith-0.1.86-py3-none-any.whl", hash = "sha256:55ed80cc6e98f9761f9b3ec3c49e01f6745d13e40bef80d9f831acabfd9a8a1e"}, + {file = "langsmith-0.1.86.tar.gz", hash = "sha256:2e66577817253327b99b727588c3173fbba217fe0ca07ac6b7cdd23fc4894104"}, ] [package.dependencies] @@ -1420,4 +1420,4 @@ files = [ [metadata] lock-version = "2.0" python-versions = "^3.9,<3.12" -content-hash = "5c9e0f9a32f316f08b2fa20279601e968bad24cb2f47815af067b91689933b20" +content-hash = "a2f5a0a03a1e482ede36cf35a5275976f875b4eb3039291332b4dc259a327d56" diff --git a/airbyte-integrations/connectors/source-twilio/pyproject.toml b/airbyte-integrations/connectors/source-twilio/pyproject.toml index 1fec3c6f8979..d8ec37e6df31 100644 --- a/airbyte-integrations/connectors/source-twilio/pyproject.toml +++ b/airbyte-integrations/connectors/source-twilio/pyproject.toml @@ -17,7 +17,7 @@ include = "source_twilio" [tool.poetry.dependencies] python = "^3.9,<3.12" -airbyte-cdk = "^2" +airbyte-cdk = "^3" pendulum = "==2.1.2" requests = "==2.31.0" diff --git a/airbyte-integrations/connectors/source-twilio/source_twilio/manifest.yaml b/airbyte-integrations/connectors/source-twilio/source_twilio/manifest.yaml index b32ed05c3820..99515c568cd1 100644 --- a/airbyte-integrations/connectors/source-twilio/source_twilio/manifest.yaml +++ b/airbyte-integrations/connectors/source-twilio/source_twilio/manifest.yaml @@ -493,18 +493,14 @@ definitions: partition_router: - type: SubstreamPartitionRouter parent_stream_configs: - - type: ParentStreamConfig + - stream: "#/definitions/streams/Flows" parent_key: sid partition_field: flow_sid - stream: - $ref: "#/definitions/streams/Flows" - type: SubstreamPartitionRouter parent_stream_configs: - - type: ParentStreamConfig + - stream: "#/definitions/streams/Executions" parent_key: sid partition_field: execution_sid - stream: - $ref: "#/definitions/streams/Executions" transformations: - type: AddFields fields: From 602ba041a4d3e404b163386f7cbd19114c3c6941 Mon Sep 17 00:00:00 2001 From: "cristina.mariscal" Date: Tue, 16 Jul 2024 14:56:22 -0600 Subject: [PATCH 34/37] updates manifest --- .../source-twilio/source_twilio/manifest.yaml | 192 +++++++----------- 1 file changed, 69 insertions(+), 123 deletions(-) diff --git a/airbyte-integrations/connectors/source-twilio/source_twilio/manifest.yaml b/airbyte-integrations/connectors/source-twilio/source_twilio/manifest.yaml index 99515c568cd1..90c60b972264 100644 --- a/airbyte-integrations/connectors/source-twilio/source_twilio/manifest.yaml +++ b/airbyte-integrations/connectors/source-twilio/source_twilio/manifest.yaml @@ -33,6 +33,20 @@ definitions: type: DpathExtractor field_path: - accounts + paginator: + type: DefaultPaginator + page_token_option: + type: RequestOption + inject_into: request_parameter + field_name: Page + page_size_option: + type: RequestOption + field_name: PageSize + inject_into: request_parameter + pagination_strategy: + type: PageIncrement + page_size: 1000 + start_from_page: 0 transformations: - type: AddFields fields: @@ -158,7 +172,7 @@ definitions: requester: $ref: "#/definitions/base_requester" path: >- - https://api.twilio.com/2010-04-01/Accounts/{{stream_partition.account_sid}}/AvailablePhoneNumbers/{{stream_partition.country_code}}/Local.json + https://api.twilio.com/2010-04-01/Accounts/{{stream_partition.parent_slice.account_sid}}/AvailablePhoneNumbers/{{stream_partition.country_code}}/Local.json http_method: GET record_selector: type: RecordSelector @@ -181,20 +195,13 @@ definitions: page_size: 1000 start_from_page: 0 partition_router: - - type: SubstreamPartitionRouter - parent_stream_configs: - - type: ParentStreamConfig - parent_key: sid - partition_field: account_sid - stream: - $ref: "#/definitions/streams/Accounts" - - type: SubstreamPartitionRouter - parent_stream_configs: - - type: ParentStreamConfig - parent_key: country_code - partition_field: country_code - stream: - $ref: "#/definitions/streams/Available Phone Number Countries" + type: SubstreamPartitionRouter + parent_stream_configs: + - type: ParentStreamConfig + parent_key: country_code + partition_field: country_code + stream: + $ref: "#/definitions/streams/Available Phone Number Countries" schema_loader: type: InlineSchemaLoader schema: @@ -207,7 +214,7 @@ definitions: requester: $ref: "#/definitions/base_requester" path: >- - https://api.twilio.com/2010-04-01/Accounts/{{stream_partition.account_sid}}/AvailablePhoneNumbers/{{stream_partition.country_code}}/Mobile.json + https://api.twilio.com/2010-04-01/Accounts/{{stream_partition.parent_slice.account_sid}}/AvailablePhoneNumbers/{{stream_partition.country_code}}/Mobile.json http_method: GET record_selector: type: RecordSelector @@ -230,20 +237,13 @@ definitions: page_size: 1000 start_from_page: 0 partition_router: - - type: SubstreamPartitionRouter - parent_stream_configs: - - type: ParentStreamConfig - parent_key: sid - partition_field: account_sid - stream: - $ref: "#/definitions/streams/Accounts" - - type: SubstreamPartitionRouter - parent_stream_configs: - - type: ParentStreamConfig - parent_key: country_code - partition_field: country_code - stream: - $ref: "#/definitions/streams/Available Phone Number Countries" + type: SubstreamPartitionRouter + parent_stream_configs: + - type: ParentStreamConfig + parent_key: country_code + partition_field: country_code + stream: + $ref: "#/definitions/streams/Available Phone Number Countries" schema_loader: type: InlineSchemaLoader schema: @@ -256,7 +256,7 @@ definitions: requester: $ref: "#/definitions/base_requester" path: >- - https://api.twilio.com/2010-04-01/Accounts/{{stream_partition.account_sid}}/AvailablePhoneNumbers/{{stream_partition.country_code}}/TollFree.json + https://api.twilio.com/2010-04-01/Accounts/{{stream_partition.parent_slice.account_sid}}/AvailablePhoneNumbers/{{stream_partition.country_code}}/TollFree.json http_method: GET error_handler: type: CompositeErrorHandler @@ -287,20 +287,13 @@ definitions: page_size: 1000 start_from_page: 0 partition_router: - - type: SubstreamPartitionRouter - parent_stream_configs: - - type: ParentStreamConfig - parent_key: sid - partition_field: account_sid - stream: - $ref: "#/definitions/streams/Accounts" - - type: SubstreamPartitionRouter - parent_stream_configs: - - type: ParentStreamConfig - parent_key: country_code - partition_field: country_code - stream: - $ref: "#/definitions/streams/Available Phone Number Countries" + type: SubstreamPartitionRouter + parent_stream_configs: + - type: ParentStreamConfig + parent_key: country_code + partition_field: country_code + stream: + $ref: "#/definitions/streams/Available Phone Number Countries" schema_loader: type: InlineSchemaLoader schema: @@ -460,22 +453,14 @@ definitions: requester: $ref: "#/definitions/base_requester" path: >- - https://studio.twilio.com/v1/Flows/{{stream_partition.flow_sid}}/Executions/{{stream_partition.execution_sid}}/Steps + https://studio.twilio.com/v1/Flows/{{stream_partition.parent_slice.flow_sid}}/Executions/{{stream_partition.execution_sid}}/Steps http_method: GET - error_handler: - type: CompositeErrorHandler - error_handlers: - - type: DefaultErrorHandler - response_filters: - - http_codes: [404] - action: IGNORE - error_message: "The requested resource was not found." record_selector: type: RecordSelector extractor: type: DpathExtractor field_path: - - step + - steps paginator: type: DefaultPaginator page_token_option: @@ -491,29 +476,11 @@ definitions: page_size: 1000 start_from_page: 0 partition_router: - - type: SubstreamPartitionRouter - parent_stream_configs: - - stream: "#/definitions/streams/Flows" - parent_key: sid - partition_field: flow_sid - - type: SubstreamPartitionRouter - parent_stream_configs: - - stream: "#/definitions/streams/Executions" - parent_key: sid - partition_field: execution_sid - transformations: - - type: AddFields - fields: - - type: AddedFieldDefinition - path: - - date_created - value: '{% set formatted_datetime = format_datetime(record["date_created"], "%Y-%m-%dT%H:%M:%SZ", "%a, %d %b %Y %H:%M:%S %z") %}{{ formatted_datetime }}' - - type: AddFields - fields: - - type: AddedFieldDefinition - path: - - date_updated - value: '{% set formatted_datetime = format_datetime(record["date_updated"], "%Y-%m-%dT%H:%M:%SZ", "%a, %d %b %Y %H:%M:%S %z") %}{{ formatted_datetime }}' + type: SubstreamPartitionRouter + parent_stream_configs: + - stream: "#/definitions/streams/Executions" + parent_key: sid + partition_field: execution_sid schema_loader: type: InlineSchemaLoader schema: @@ -637,7 +604,7 @@ definitions: $ref: "#/definitions/base_requester" path: >- https://api.twilio.com/2010-04-01/Accounts/{{ - stream_partition.account_sid }}/Addresses/{{ + stream_partition.parent_slice.account_sid }}/Addresses/{{ stream_partition.address_sid }}/DependentPhoneNumbers.json http_method: GET record_selector: @@ -661,20 +628,13 @@ definitions: page_size: 1000 start_from_page: 0 partition_router: - - type: SubstreamPartitionRouter - parent_stream_configs: - - type: ParentStreamConfig - parent_key: sid - partition_field: account_sid - stream: - $ref: "#/definitions/streams/Accounts" - - type: SubstreamPartitionRouter - parent_stream_configs: - - type: ParentStreamConfig - parent_key: sid - partition_field: address_sid - stream: - $ref: "#/definitions/streams/Addresses" + type: SubstreamPartitionRouter + parent_stream_configs: + - type: ParentStreamConfig + parent_key: sid + partition_field: address_sid + stream: + $ref: "#/definitions/streams/Addresses" transformations: - type: AddFields fields: @@ -898,7 +858,7 @@ definitions: $ref: "#/definitions/base_requester" path: >- https://api.twilio.com/2010-04-01/Accounts/{{ - stream_partition.account_sid }}/Conferences/{{ + stream_partition.parent_slice.account_sid }}/Conferences/{{ stream_partition.conference_sid }}/Participants.json http_method: GET record_selector: @@ -922,20 +882,13 @@ definitions: page_size: 1000 start_from_page: 0 partition_router: - - type: SubstreamPartitionRouter - parent_stream_configs: - - type: ParentStreamConfig - parent_key: sid - partition_field: account_sid - stream: - $ref: "#/definitions/streams/Accounts" - - type: SubstreamPartitionRouter - parent_stream_configs: - - type: ParentStreamConfig - parent_key: sid - partition_field: conference_sid - stream: - $ref: "#/definitions/streams/Conferences" + type: SubstreamPartitionRouter + parent_stream_configs: + - type: ParentStreamConfig + parent_key: sid + partition_field: conference_sid + stream: + $ref: "#/definitions/streams/Conferences" transformations: - type: AddFields fields: @@ -1485,7 +1438,7 @@ definitions: $ref: "#/definitions/base_requester" path: >- https://api.twilio.com/2010-04-01/Accounts/{{ - stream_partition.account_sid }}/Messages/{{ + stream_partition.parent_slice.account_sid }}/Messages/{{ stream_partition.message_sid }}/Media.json http_method: GET record_selector: @@ -1509,20 +1462,13 @@ definitions: page_size: 1000 start_from_page: 0 partition_router: - - type: SubstreamPartitionRouter - parent_stream_configs: - - type: ParentStreamConfig - parent_key: sid - partition_field: account_sid - stream: - $ref: "#/definitions/streams/Accounts" - - type: SubstreamPartitionRouter - parent_stream_configs: - - type: ParentStreamConfig - parent_key: sid - partition_field: message_sid - stream: - $ref: "#/definitions/streams/Messages" + type: SubstreamPartitionRouter + parent_stream_configs: + - type: ParentStreamConfig + parent_key: sid + partition_field: message_sid + stream: + $ref: "#/definitions/streams/Messages" incremental_sync: type: DatetimeBasedCursor cursor_field: date_created From 4950b004da1d2ac2093befc806415e1adf479947 Mon Sep 17 00:00:00 2001 From: "cristina.mariscal" Date: Wed, 17 Jul 2024 09:58:57 -0600 Subject: [PATCH 35/37] updating toml --- .../connectors/source-twilio/poetry.lock | 14 +++++++------- .../connectors/source-twilio/pyproject.toml | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/airbyte-integrations/connectors/source-twilio/poetry.lock b/airbyte-integrations/connectors/source-twilio/poetry.lock index f3f554044796..4691a6d0e4a2 100644 --- a/airbyte-integrations/connectors/source-twilio/poetry.lock +++ b/airbyte-integrations/connectors/source-twilio/poetry.lock @@ -2,13 +2,13 @@ [[package]] name = "airbyte-cdk" -version = "3.5.3" +version = "2.4.0" description = "A framework for writing Airbyte Connectors." optional = false python-versions = "<4.0,>=3.9" files = [ - {file = "airbyte_cdk-3.5.3-py3-none-any.whl", hash = "sha256:729e9d6f9373f6011a0904b3583c0c9f354c5b48ba006051852f7f8255ae3c31"}, - {file = "airbyte_cdk-3.5.3.tar.gz", hash = "sha256:5213b04d470093f43cc780b72b6bcc8a2bacb12ec5162a2704e50c036b5086a8"}, + {file = "airbyte_cdk-2.4.0-py3-none-any.whl", hash = "sha256:39470b2fe97f28959fcecb839d3080a8aba4a64a29dddf54a39f11f93f9f9ef7"}, + {file = "airbyte_cdk-2.4.0.tar.gz", hash = "sha256:f973d2e17a6dd0416c4395139e2761a10b38aafa61e097eaacffebbe6164ef45"}, ] [package.dependencies] @@ -591,13 +591,13 @@ extended-testing = ["jinja2 (>=3,<4)"] [[package]] name = "langsmith" -version = "0.1.86" +version = "0.1.88" description = "Client library to connect to the LangSmith LLM Tracing and Evaluation Platform." optional = false python-versions = "<4.0,>=3.8.1" files = [ - {file = "langsmith-0.1.86-py3-none-any.whl", hash = "sha256:55ed80cc6e98f9761f9b3ec3c49e01f6745d13e40bef80d9f831acabfd9a8a1e"}, - {file = "langsmith-0.1.86.tar.gz", hash = "sha256:2e66577817253327b99b727588c3173fbba217fe0ca07ac6b7cdd23fc4894104"}, + {file = "langsmith-0.1.88-py3-none-any.whl", hash = "sha256:460ebb7de440afd150fcea8f54ca8779821f2228cd59e149e5845c9dbe06db16"}, + {file = "langsmith-0.1.88.tar.gz", hash = "sha256:28a07dec19197f4808aa2628d5a3ccafcbe14cc137aef0e607bbd128e7907821"}, ] [package.dependencies] @@ -1420,4 +1420,4 @@ files = [ [metadata] lock-version = "2.0" python-versions = "^3.9,<3.12" -content-hash = "a2f5a0a03a1e482ede36cf35a5275976f875b4eb3039291332b4dc259a327d56" +content-hash = "5c9e0f9a32f316f08b2fa20279601e968bad24cb2f47815af067b91689933b20" diff --git a/airbyte-integrations/connectors/source-twilio/pyproject.toml b/airbyte-integrations/connectors/source-twilio/pyproject.toml index d8ec37e6df31..1fec3c6f8979 100644 --- a/airbyte-integrations/connectors/source-twilio/pyproject.toml +++ b/airbyte-integrations/connectors/source-twilio/pyproject.toml @@ -17,7 +17,7 @@ include = "source_twilio" [tool.poetry.dependencies] python = "^3.9,<3.12" -airbyte-cdk = "^3" +airbyte-cdk = "^2" pendulum = "==2.1.2" requests = "==2.31.0" From 877e7a0ab30da560370b6f6ee3929216ee781b95 Mon Sep 17 00:00:00 2001 From: "cristina.mariscal" Date: Fri, 19 Jul 2024 09:34:30 -0600 Subject: [PATCH 36/37] changes --- .../source-twilio/source_twilio/manifest.yaml | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/airbyte-integrations/connectors/source-twilio/source_twilio/manifest.yaml b/airbyte-integrations/connectors/source-twilio/source_twilio/manifest.yaml index 90c60b972264..1e1075a1543b 100644 --- a/airbyte-integrations/connectors/source-twilio/source_twilio/manifest.yaml +++ b/airbyte-integrations/connectors/source-twilio/source_twilio/manifest.yaml @@ -258,14 +258,6 @@ definitions: path: >- https://api.twilio.com/2010-04-01/Accounts/{{stream_partition.parent_slice.account_sid}}/AvailablePhoneNumbers/{{stream_partition.country_code}}/TollFree.json http_method: GET - error_handler: - type: CompositeErrorHandler - error_handlers: - - type: DefaultErrorHandler - response_filters: - - http_codes: [404] - action: IGNORE - error_message: "The requested resource was not found." record_selector: type: RecordSelector extractor: @@ -488,6 +480,8 @@ definitions: Alerts: type: DeclarativeStream name: alerts + primary_key: + - sid retriever: type: SimpleRetriever requester: From 0a2317b0e01d3ddf3230f9a27fdcb3b2f63d3baa Mon Sep 17 00:00:00 2001 From: "cristina.mariscal" Date: Sun, 21 Jul 2024 16:30:04 -0600 Subject: [PATCH 37/37] parametrize Available Phone Number --- .../source-twilio/source_twilio/manifest.yaml | 33 ++++++++++++++----- 1 file changed, 24 insertions(+), 9 deletions(-) diff --git a/airbyte-integrations/connectors/source-twilio/source_twilio/manifest.yaml b/airbyte-integrations/connectors/source-twilio/source_twilio/manifest.yaml index 1e1075a1543b..ac2fcefe1a42 100644 --- a/airbyte-integrations/connectors/source-twilio/source_twilio/manifest.yaml +++ b/airbyte-integrations/connectors/source-twilio/source_twilio/manifest.yaml @@ -138,6 +138,8 @@ definitions: type: DpathExtractor field_path: - countries + record_filter: + condition: '{{ not parameters.get("subresource", False) or record["subresource_uris"].get(parameters.get("subresource", ""), False) }}' paginator: type: DefaultPaginator page_token_option: @@ -160,6 +162,13 @@ definitions: partition_field: account_sid stream: $ref: "#/definitions/streams/Accounts" + transformations: + - type: AddFields + fields: + - type: AddedFieldDefinition + path: + - subresource_uri + value: '{{ record["subresource_uris"].get(parameters.get("subresource", ""), "") }}' schema_loader: type: InlineSchemaLoader schema: @@ -172,7 +181,7 @@ definitions: requester: $ref: "#/definitions/base_requester" path: >- - https://api.twilio.com/2010-04-01/Accounts/{{stream_partition.parent_slice.account_sid}}/AvailablePhoneNumbers/{{stream_partition.country_code}}/Local.json + https://api.twilio.com{{stream_partition.subresource_uri}} http_method: GET record_selector: type: RecordSelector @@ -198,10 +207,12 @@ definitions: type: SubstreamPartitionRouter parent_stream_configs: - type: ParentStreamConfig - parent_key: country_code - partition_field: country_code + parent_key: subresource_uri + partition_field: subresource_uri stream: $ref: "#/definitions/streams/Available Phone Number Countries" + $parameters: + subresource: "local" schema_loader: type: InlineSchemaLoader schema: @@ -214,7 +225,7 @@ definitions: requester: $ref: "#/definitions/base_requester" path: >- - https://api.twilio.com/2010-04-01/Accounts/{{stream_partition.parent_slice.account_sid}}/AvailablePhoneNumbers/{{stream_partition.country_code}}/Mobile.json + https://api.twilio.com{{stream_partition.subresource_uri}} http_method: GET record_selector: type: RecordSelector @@ -240,10 +251,12 @@ definitions: type: SubstreamPartitionRouter parent_stream_configs: - type: ParentStreamConfig - parent_key: country_code - partition_field: country_code + parent_key: subresource_uri + partition_field: subresource_uri stream: $ref: "#/definitions/streams/Available Phone Number Countries" + $parameters: + subresource: "mobile" schema_loader: type: InlineSchemaLoader schema: @@ -256,7 +269,7 @@ definitions: requester: $ref: "#/definitions/base_requester" path: >- - https://api.twilio.com/2010-04-01/Accounts/{{stream_partition.parent_slice.account_sid}}/AvailablePhoneNumbers/{{stream_partition.country_code}}/TollFree.json + https://api.twilio.com{{stream_partition.subresource_uri}} http_method: GET record_selector: type: RecordSelector @@ -282,10 +295,12 @@ definitions: type: SubstreamPartitionRouter parent_stream_configs: - type: ParentStreamConfig - parent_key: country_code - partition_field: country_code + parent_key: subresource_uri + partition_field: subresource_uri stream: $ref: "#/definitions/streams/Available Phone Number Countries" + $parameters: + subresource: "toll_free" schema_loader: type: InlineSchemaLoader schema: